From aaf3d7ee0632b0b4a019760a4e8bd61ba8be6056 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 3 Aug 2020 13:31:06 +0200 Subject: [PATCH 01/19] Prepare home directories for docker images in a different stage (#20356) Files included in the home directory of the docker images need some changes on permissions and ownerships after being copied. If this is done as a COPY and a RUN, it creates two layers with all the files, that are included in the final image, increasing its size. Move the preparation of the home directory to a different stage, so in the final image it is done as an only COPY operation. --- .../templates/docker/Dockerfile.tmpl | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 9080b7c534d..9eac254f822 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -2,6 +2,25 @@ {{- $beatBinary := printf "%s/%s" $beatHome .BeatName }} {{- $repoInfo := repo }} +# Prepare home in a different stage to avoid creating additional layers on +# the final image because of permission changes. +FROM {{ .from }} AS home + +COPY beat {{ $beatHome }} + +RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ + chown -R root:root {{ $beatHome }} && \ + find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ + find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ + chmod 0750 {{ $beatBinary }} && \ +{{- if .linux_capabilities }} + setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ +{{- end }} +{{- range $i, $modulesd := .ModulesDirs }} + chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ +{{- end }} + chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs + FROM {{ .from }} RUN yum -y --setopt=tsflags=nodocs update && \ @@ -23,26 +42,13 @@ LABEL \ ENV ELASTIC_CONTAINER "true" ENV PATH={{ $beatHome }}:$PATH -COPY beat {{ $beatHome }} COPY docker-entrypoint /usr/local/bin/docker-entrypoint RUN chmod 755 /usr/local/bin/docker-entrypoint -RUN groupadd --gid 1000 {{ .BeatName }} - -RUN mkdir {{ $beatHome }}/data {{ $beatHome }}/logs && \ - chown -R root:root {{ $beatHome }} && \ - find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \ - find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \ - chmod 0750 {{ $beatBinary }} && \ -{{- if .linux_capabilities }} - setcap {{ .linux_capabilities }} {{ $beatBinary }} && \ -{{- end }} -{{- range $i, $modulesd := .ModulesDirs }} - chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \ -{{- end }} - chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs +COPY --from=home {{ $beatHome }} {{ $beatHome }} {{- if ne .user "root" }} +RUN groupadd --gid 1000 {{ .BeatName }} RUN useradd -M --uid 1000 --gid 1000 --groups 0 --home {{ $beatHome }} {{ .user }} {{- end }} USER {{ .user }} From 77b3b075b2a9733ac7ae05ce836150b6a677c5c6 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 3 Aug 2020 11:17:13 -0400 Subject: [PATCH 02/19] [Elastic Agent] Improve version, restart, enroll CLI commands (#20359) * Add improve version CLI cmd. * Add new restart cmd. Perform restart at end of enroll. * Fix yaml annotations on version struct. * Fix control.Address on Windows. * Fix control.Address on Windows. * Fix windows dialer. * Fix control.Address on Windows. * Add to CHANGELOG. * Review cleanups. * Fix go vet. * Update talking to communicating. --- x-pack/elastic-agent/CHANGELOG.asciidoc | 3 + x-pack/elastic-agent/control.proto | 2 +- .../pkg/agent/application/reexec/manager.go | 18 +-- x-pack/elastic-agent/pkg/agent/cmd/enroll.go | 24 ++- x-pack/elastic-agent/pkg/agent/cmd/run.go | 10 +- .../pkg/agent/control/addr_windows.go | 4 +- .../pkg/agent/control/client/client.go | 22 +-- .../pkg/agent/control/client/dial_windows.go | 2 +- .../pkg/agent/control/control_test.go | 6 +- .../pkg/agent/control/proto/control.pb.go | 142 +++++++++--------- .../pkg/agent/control/server/listener.go | 19 ++- .../agent/control/server/listener_windows.go | 7 +- .../pkg/agent/control/server/server.go | 19 ++- x-pack/elastic-agent/pkg/basecmd/cmd.go | 2 + .../elastic-agent/pkg/basecmd/restart/cmd.go | 37 +++++ .../elastic-agent/pkg/basecmd/version/cmd.go | 87 +++++++++-- .../pkg/basecmd/version/cmd_test.go | 85 ++++++++++- x-pack/elastic-agent/pkg/release/version.go | 35 +++++ 18 files changed, 389 insertions(+), 135 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/basecmd/restart/cmd.go diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index e0e925847aa..f4ad19cc290 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -99,3 +99,6 @@ - Add --staging option to enroll command {pull}20026[20026] - Add `event.dataset` to all events {pull}20076[20076] - Prepare packaging for endpoint and asc files {pull}20186[20186] +- Improved version CLI {pull}20359[20359] +- Enroll CLI now restarts running daemon {pull}20359[20359] +- Add restart CLI cmd {pull}20359[20359] diff --git a/x-pack/elastic-agent/control.proto b/x-pack/elastic-agent/control.proto index a7ff22e5157..0c5645faab9 100644 --- a/x-pack/elastic-agent/control.proto +++ b/x-pack/elastic-agent/control.proto @@ -104,7 +104,7 @@ message StatusResponse { repeated ApplicationStatus applications = 3; } -service ElasticAgent { +service ElasticAgentControl { // Fetches the currently running version of the Elastic Agent. rpc Version(Empty) returns (VersionResponse); diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go index 33de754a27d..4662b1c6230 100644 --- a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go +++ b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go @@ -5,16 +5,9 @@ package reexec import ( - "sync" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -var ( - execSingleton ExecManager - execSingletonOnce sync.Once -) - // ExecManager is the interface that the global reexec manager implements. type ExecManager interface { // ReExec asynchronously re-executes command in the same PID and memory address @@ -30,14 +23,6 @@ type ExecManager interface { ShutdownComplete() } -// Manager returns the global reexec manager. -func Manager(log *logger.Logger, exec string) ExecManager { - execSingletonOnce.Do(func() { - execSingleton = newManager(log, exec) - }) - return execSingleton -} - type manager struct { logger *logger.Logger exec string @@ -46,7 +31,8 @@ type manager struct { complete chan bool } -func newManager(log *logger.Logger, exec string) *manager { +// NewManager returns the reexec manager. +func NewManager(log *logger.Logger, exec string) ExecManager { return &manager{ logger: log, exec: exec, diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index 311ad31e63b..b34e0236782 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -5,11 +5,14 @@ package cmd import ( + "context" "fmt" "math/rand" "os" "time" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client" + "github.com/spf13/cobra" "github.com/elastic/beats/v7/libbeat/common/backoff" @@ -45,6 +48,7 @@ func newEnrollCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStr cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation") cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to Kibana") cmd.Flags().StringP("staging", "", "", "Configures agent to download artifacts from a staging build") + cmd.Flags().Bool("no-restart", false, "Skip restarting the currently running daemon") return cmd } @@ -144,7 +148,25 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args return errors.New(err, "fail to enroll") } - fmt.Fprintln(streams.Out, "Successfully enrolled the Agent.") + fmt.Fprintln(streams.Out, "Successfully enrolled the Elastic Agent.") + + // skip restarting + noRestart, _ := cmd.Flags().GetBool("no-restart") + if noRestart { + return nil + } + + daemon := client.New() + err = daemon.Connect(context.Background()) + if err == nil { + defer daemon.Disconnect() + err = daemon.Restart(context.Background()) + if err == nil { + fmt.Fprintln(streams.Out, "Successfully triggered restart on running Elastic Agent.") + return nil + } + } + fmt.Fprintln(streams.Out, "Elastic Agent might not be running; unable to trigger restart") return nil } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 1c3c81b9ff7..f502ef9cf49 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/server" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" @@ -80,7 +81,14 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { return err } rexLogger := logger.Named("reexec") - rex := reexec.Manager(rexLogger, execPath) + rex := reexec.NewManager(rexLogger, execPath) + + // start the control listener + control := server.New(logger.Named("control"), rex) + if err := control.Start(); err != nil { + return err + } + defer control.Stop() app, err := application.New(logger, pathConfigFile) if err != nil { diff --git a/x-pack/elastic-agent/pkg/agent/control/addr_windows.go b/x-pack/elastic-agent/pkg/agent/control/addr_windows.go index 1123eec941b..bf2e164fbae 100644 --- a/x-pack/elastic-agent/pkg/agent/control/addr_windows.go +++ b/x-pack/elastic-agent/pkg/agent/control/addr_windows.go @@ -15,8 +15,8 @@ import ( // Address returns the address to connect to Elastic Agent daemon. func Address() string { - data = paths.Data() + data := paths.Data() // entire string cannot be longer than 256 characters, this forces the // length to always be 87 characters (but unique per data path) - return fmt.Sprintf(`\\.\pipe\elastic-agent-%s`, sha256.Sum256(data)) + return fmt.Sprintf(`\\.\pipe\elastic-agent-%x`, sha256.Sum256([]byte(data))) } diff --git a/x-pack/elastic-agent/pkg/agent/control/client/client.go b/x-pack/elastic-agent/pkg/agent/control/client/client.go index bcd8eccdb82..5e55fce9349 100644 --- a/x-pack/elastic-agent/pkg/agent/control/client/client.go +++ b/x-pack/elastic-agent/pkg/agent/control/client/client.go @@ -8,11 +8,11 @@ import ( "context" "encoding/json" "fmt" + "sync" "time" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto" ) @@ -62,10 +62,10 @@ type AgentStatus struct { // Client communicates to Elastic Agent through the control protocol. type Client interface { - // Start starts the client. - Start(ctx context.Context) error - // Stop stops the client. - Stop() + // Connect connects to the running Elastic Agent. + Connect(ctx context.Context) error + // Disconnect disconnects from the running Elastic Agent. + Disconnect() // Version returns the current version of the running agent. Version(ctx context.Context) (Version, error) // Status returns the current status of the running agent. @@ -81,7 +81,7 @@ type client struct { ctx context.Context cancel context.CancelFunc wg sync.WaitGroup - client proto.ElasticAgentClient + client proto.ElasticAgentControlClient cfgLock sync.RWMutex obsLock sync.RWMutex } @@ -91,19 +91,19 @@ func New() Client { return &client{} } -// Start starts the connection to Elastic Agent. -func (c *client) Start(ctx context.Context) error { +// Connect connects to the running Elastic Agent. +func (c *client) Connect(ctx context.Context) error { c.ctx, c.cancel = context.WithCancel(ctx) conn, err := dialContext(ctx) if err != nil { return err } - c.client = proto.NewElasticAgentClient(conn) + c.client = proto.NewElasticAgentControlClient(conn) return nil } -// Stop stops the connection to Elastic Agent. -func (c *client) Stop() { +// Disconnect disconnects from the running Elastic Agent. +func (c *client) Disconnect() { if c.cancel != nil { c.cancel() c.wg.Wait() diff --git a/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go b/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go index 58b36c18043..c061753d327 100644 --- a/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go +++ b/x-pack/elastic-agent/pkg/agent/control/client/dial_windows.go @@ -22,5 +22,5 @@ func dialContext(ctx context.Context) (*grpc.ClientConn, error) { } func dialer(ctx context.Context, addr string) (net.Conn, error) { - return npipe.DialContext(arr)(ctx, "", "") + return npipe.DialContext(addr)(ctx, "", "") } diff --git a/x-pack/elastic-agent/pkg/agent/control/control_test.go b/x-pack/elastic-agent/pkg/agent/control/control_test.go index 13d32420258..9454179ae60 100644 --- a/x-pack/elastic-agent/pkg/agent/control/control_test.go +++ b/x-pack/elastic-agent/pkg/agent/control/control_test.go @@ -20,15 +20,15 @@ import ( ) func TestServerClient_Version(t *testing.T) { - srv := server.New(newErrorLogger(t)) + srv := server.New(newErrorLogger(t), nil) err := srv.Start() require.NoError(t, err) defer srv.Stop() c := client.New() - err = c.Start(context.Background()) + err = c.Connect(context.Background()) require.NoError(t, err) - defer c.Stop() + defer c.Disconnect() ver, err := c.Version(context.Background()) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/agent/control/proto/control.pb.go b/x-pack/elastic-agent/pkg/agent/control/proto/control.pb.go index 58df5e28f19..a0e2e710f0c 100644 --- a/x-pack/elastic-agent/pkg/agent/control/proto/control.pb.go +++ b/x-pack/elastic-agent/pkg/agent/control/proto/control.pb.go @@ -662,24 +662,24 @@ var file_control_proto_rawDesc = []byte{ 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x07, 0x2a, 0x28, 0x0a, 0x0c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0b, 0x0a, - 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x32, 0xd9, 0x01, 0x0a, 0x0c, 0x45, - 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x12, 0x2f, 0x0a, 0x07, 0x56, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x06, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x52, - 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x07, - 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x32, 0xe0, 0x01, 0x0a, 0x13, 0x45, + 0x6c, 0x61, 0x73, 0x74, 0x69, 0x63, 0x41, 0x67, 0x65, 0x6e, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x12, 0x2f, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x0c, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x2d, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0c, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x0c, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x15, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x22, 0x5a, 0x1d, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x67, - 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0xf8, 0x01, 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x55, 0x70, + 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x22, 0x5a, + 0x1d, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, + 0x6f, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0xf8, 0x01, + 0x01, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -713,14 +713,14 @@ var file_control_proto_depIdxs = []int32{ 0, // 2: proto.ApplicationStatus.status:type_name -> proto.Status 0, // 3: proto.StatusResponse.status:type_name -> proto.Status 7, // 4: proto.StatusResponse.applications:type_name -> proto.ApplicationStatus - 2, // 5: proto.ElasticAgent.Version:input_type -> proto.Empty - 2, // 6: proto.ElasticAgent.Status:input_type -> proto.Empty - 2, // 7: proto.ElasticAgent.Restart:input_type -> proto.Empty - 5, // 8: proto.ElasticAgent.Upgrade:input_type -> proto.UpgradeRequest - 3, // 9: proto.ElasticAgent.Version:output_type -> proto.VersionResponse - 8, // 10: proto.ElasticAgent.Status:output_type -> proto.StatusResponse - 4, // 11: proto.ElasticAgent.Restart:output_type -> proto.RestartResponse - 6, // 12: proto.ElasticAgent.Upgrade:output_type -> proto.UpgradeResponse + 2, // 5: proto.ElasticAgentControl.Version:input_type -> proto.Empty + 2, // 6: proto.ElasticAgentControl.Status:input_type -> proto.Empty + 2, // 7: proto.ElasticAgentControl.Restart:input_type -> proto.Empty + 5, // 8: proto.ElasticAgentControl.Upgrade:input_type -> proto.UpgradeRequest + 3, // 9: proto.ElasticAgentControl.Version:output_type -> proto.VersionResponse + 8, // 10: proto.ElasticAgentControl.Status:output_type -> proto.StatusResponse + 4, // 11: proto.ElasticAgentControl.Restart:output_type -> proto.RestartResponse + 6, // 12: proto.ElasticAgentControl.Upgrade:output_type -> proto.UpgradeResponse 9, // [9:13] is the sub-list for method output_type 5, // [5:9] is the sub-list for method input_type 5, // [5:5] is the sub-list for extension type_name @@ -848,10 +848,10 @@ var _ grpc.ClientConnInterface // is compatible with the grpc package it is being compiled against. const _ = grpc.SupportPackageIsVersion6 -// ElasticAgentClient is the client API for ElasticAgent service. +// ElasticAgentControlClient is the client API for ElasticAgentControl service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type ElasticAgentClient interface { +type ElasticAgentControlClient interface { // Fetches the currently running version of the Elastic Agent. Version(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*VersionResponse, error) // Fetches the currently status of the Elastic Agent. @@ -862,52 +862,52 @@ type ElasticAgentClient interface { Upgrade(ctx context.Context, in *UpgradeRequest, opts ...grpc.CallOption) (*UpgradeResponse, error) } -type elasticAgentClient struct { +type elasticAgentControlClient struct { cc grpc.ClientConnInterface } -func NewElasticAgentClient(cc grpc.ClientConnInterface) ElasticAgentClient { - return &elasticAgentClient{cc} +func NewElasticAgentControlClient(cc grpc.ClientConnInterface) ElasticAgentControlClient { + return &elasticAgentControlClient{cc} } -func (c *elasticAgentClient) Version(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*VersionResponse, error) { +func (c *elasticAgentControlClient) Version(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*VersionResponse, error) { out := new(VersionResponse) - err := c.cc.Invoke(ctx, "/proto.ElasticAgent/Version", in, out, opts...) + err := c.cc.Invoke(ctx, "/proto.ElasticAgentControl/Version", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *elasticAgentClient) Status(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*StatusResponse, error) { +func (c *elasticAgentControlClient) Status(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*StatusResponse, error) { out := new(StatusResponse) - err := c.cc.Invoke(ctx, "/proto.ElasticAgent/Status", in, out, opts...) + err := c.cc.Invoke(ctx, "/proto.ElasticAgentControl/Status", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *elasticAgentClient) Restart(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*RestartResponse, error) { +func (c *elasticAgentControlClient) Restart(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*RestartResponse, error) { out := new(RestartResponse) - err := c.cc.Invoke(ctx, "/proto.ElasticAgent/Restart", in, out, opts...) + err := c.cc.Invoke(ctx, "/proto.ElasticAgentControl/Restart", in, out, opts...) if err != nil { return nil, err } return out, nil } -func (c *elasticAgentClient) Upgrade(ctx context.Context, in *UpgradeRequest, opts ...grpc.CallOption) (*UpgradeResponse, error) { +func (c *elasticAgentControlClient) Upgrade(ctx context.Context, in *UpgradeRequest, opts ...grpc.CallOption) (*UpgradeResponse, error) { out := new(UpgradeResponse) - err := c.cc.Invoke(ctx, "/proto.ElasticAgent/Upgrade", in, out, opts...) + err := c.cc.Invoke(ctx, "/proto.ElasticAgentControl/Upgrade", in, out, opts...) if err != nil { return nil, err } return out, nil } -// ElasticAgentServer is the server API for ElasticAgent service. -type ElasticAgentServer interface { +// ElasticAgentControlServer is the server API for ElasticAgentControl service. +type ElasticAgentControlServer interface { // Fetches the currently running version of the Elastic Agent. Version(context.Context, *Empty) (*VersionResponse, error) // Fetches the currently status of the Elastic Agent. @@ -918,118 +918,118 @@ type ElasticAgentServer interface { Upgrade(context.Context, *UpgradeRequest) (*UpgradeResponse, error) } -// UnimplementedElasticAgentServer can be embedded to have forward compatible implementations. -type UnimplementedElasticAgentServer struct { +// UnimplementedElasticAgentControlServer can be embedded to have forward compatible implementations. +type UnimplementedElasticAgentControlServer struct { } -func (*UnimplementedElasticAgentServer) Version(context.Context, *Empty) (*VersionResponse, error) { +func (*UnimplementedElasticAgentControlServer) Version(context.Context, *Empty) (*VersionResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Version not implemented") } -func (*UnimplementedElasticAgentServer) Status(context.Context, *Empty) (*StatusResponse, error) { +func (*UnimplementedElasticAgentControlServer) Status(context.Context, *Empty) (*StatusResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Status not implemented") } -func (*UnimplementedElasticAgentServer) Restart(context.Context, *Empty) (*RestartResponse, error) { +func (*UnimplementedElasticAgentControlServer) Restart(context.Context, *Empty) (*RestartResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Restart not implemented") } -func (*UnimplementedElasticAgentServer) Upgrade(context.Context, *UpgradeRequest) (*UpgradeResponse, error) { +func (*UnimplementedElasticAgentControlServer) Upgrade(context.Context, *UpgradeRequest) (*UpgradeResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Upgrade not implemented") } -func RegisterElasticAgentServer(s *grpc.Server, srv ElasticAgentServer) { - s.RegisterService(&_ElasticAgent_serviceDesc, srv) +func RegisterElasticAgentControlServer(s *grpc.Server, srv ElasticAgentControlServer) { + s.RegisterService(&_ElasticAgentControl_serviceDesc, srv) } -func _ElasticAgent_Version_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _ElasticAgentControl_Version_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(ElasticAgentServer).Version(ctx, in) + return srv.(ElasticAgentControlServer).Version(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.ElasticAgent/Version", + FullMethod: "/proto.ElasticAgentControl/Version", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ElasticAgentServer).Version(ctx, req.(*Empty)) + return srv.(ElasticAgentControlServer).Version(ctx, req.(*Empty)) } return interceptor(ctx, in, info, handler) } -func _ElasticAgent_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _ElasticAgentControl_Status_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(ElasticAgentServer).Status(ctx, in) + return srv.(ElasticAgentControlServer).Status(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.ElasticAgent/Status", + FullMethod: "/proto.ElasticAgentControl/Status", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ElasticAgentServer).Status(ctx, req.(*Empty)) + return srv.(ElasticAgentControlServer).Status(ctx, req.(*Empty)) } return interceptor(ctx, in, info, handler) } -func _ElasticAgent_Restart_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _ElasticAgentControl_Restart_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Empty) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(ElasticAgentServer).Restart(ctx, in) + return srv.(ElasticAgentControlServer).Restart(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.ElasticAgent/Restart", + FullMethod: "/proto.ElasticAgentControl/Restart", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ElasticAgentServer).Restart(ctx, req.(*Empty)) + return srv.(ElasticAgentControlServer).Restart(ctx, req.(*Empty)) } return interceptor(ctx, in, info, handler) } -func _ElasticAgent_Upgrade_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { +func _ElasticAgentControl_Upgrade_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(UpgradeRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(ElasticAgentServer).Upgrade(ctx, in) + return srv.(ElasticAgentControlServer).Upgrade(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/proto.ElasticAgent/Upgrade", + FullMethod: "/proto.ElasticAgentControl/Upgrade", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ElasticAgentServer).Upgrade(ctx, req.(*UpgradeRequest)) + return srv.(ElasticAgentControlServer).Upgrade(ctx, req.(*UpgradeRequest)) } return interceptor(ctx, in, info, handler) } -var _ElasticAgent_serviceDesc = grpc.ServiceDesc{ - ServiceName: "proto.ElasticAgent", - HandlerType: (*ElasticAgentServer)(nil), +var _ElasticAgentControl_serviceDesc = grpc.ServiceDesc{ + ServiceName: "proto.ElasticAgentControl", + HandlerType: (*ElasticAgentControlServer)(nil), Methods: []grpc.MethodDesc{ { MethodName: "Version", - Handler: _ElasticAgent_Version_Handler, + Handler: _ElasticAgentControl_Version_Handler, }, { MethodName: "Status", - Handler: _ElasticAgent_Status_Handler, + Handler: _ElasticAgentControl_Status_Handler, }, { MethodName: "Restart", - Handler: _ElasticAgent_Restart_Handler, + Handler: _ElasticAgentControl_Restart_Handler, }, { MethodName: "Upgrade", - Handler: _ElasticAgent_Upgrade_Handler, + Handler: _ElasticAgentControl_Upgrade_Handler, }, }, Streams: []grpc.StreamDesc{}, diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener.go b/x-pack/elastic-agent/pkg/agent/control/server/listener.go index 2dd5d54a46f..bf03f54e2da 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/listener.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/listener.go @@ -7,16 +7,26 @@ package server import ( + "fmt" + "net" "os" "path/filepath" "strings" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -func createListener() (net.Listener, error) { +func createListener(log *logger.Logger) (net.Listener, error) { path := strings.TrimPrefix(control.Address(), "unix://") + if _, err := os.Stat(path); !os.IsNotExist(err) { + err = os.Remove(path) + if err != nil { + log.Errorf("%s", errors.New(err, fmt.Sprintf("Failed to cleanup %s", path), errors.TypeFilesystem, errors.M("path", path))) + } + } dir := filepath.Dir(path) if _, err := os.Stat(dir); os.IsNotExist(err) { err = os.MkdirAll(dir, 0755) @@ -36,3 +46,10 @@ func createListener() (net.Listener, error) { } return lis, err } + +func cleanupListener(log *logger.Logger) { + path := strings.TrimPrefix(control.Address(), "unix://") + if err := os.Remove(path); err != nil { + log.Errorf("%s", errors.New(err, fmt.Sprintf("Failed to cleanup %s", path), errors.TypeFilesystem, errors.M("path", path))) + } +} diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go b/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go index d2d2866b98a..f98c32bcee3 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/listener_windows.go @@ -13,10 +13,11 @@ import ( "github.com/elastic/beats/v7/libbeat/api/npipe" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) // createListener creates a named pipe listener on Windows -func createListener() (net.Listener, error) { +func createListener(_ *logger.Logger) (net.Listener, error) { u, err := user.Current() if err != nil { return nil, err @@ -27,3 +28,7 @@ func createListener() (net.Listener, error) { } return npipe.NewListener(control.Address(), sd) } + +func cleanupListener(_ *logger.Logger) { + // nothing to do on windows +} diff --git a/x-pack/elastic-agent/pkg/agent/control/server/server.go b/x-pack/elastic-agent/pkg/agent/control/server/server.go index c9a750808fc..faa7982c814 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/server.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/server.go @@ -8,26 +8,29 @@ import ( "context" "net" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec" "google.golang.org/grpc" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) // Server is the daemon side of the control protocol. type Server struct { logger *logger.Logger + rex reexec.ExecManager listener net.Listener server *grpc.Server } // New creates a new control protocol server. -func New(log *logger.Logger) *Server { +func New(log *logger.Logger, rex reexec.ExecManager) *Server { return &Server{ logger: log, + rex: rex, } } @@ -38,13 +41,13 @@ func (s *Server) Start() error { return nil } - lis, err := createListener() + lis, err := createListener(s.logger) if err != nil { return err } s.listener = lis s.server = grpc.NewServer() - proto.RegisterElasticAgentServer(s.server, s) + proto.RegisterElasticAgentControlServer(s.server, s) // start serving GRPC connections go func() { @@ -63,6 +66,7 @@ func (s *Server) Stop() { s.server.Stop() s.server = nil s.listener = nil + cleanupListener(s.logger) } } @@ -88,10 +92,9 @@ func (s *Server) Status(_ context.Context, _ *proto.Empty) (*proto.StatusRespons // Restart performs re-exec. func (s *Server) Restart(_ context.Context, _ *proto.Empty) (*proto.RestartResponse, error) { - // not implemented + s.rex.ReExec() return &proto.RestartResponse{ - Status: proto.ActionStatus_FAILURE, - Error: "not implemented", + Status: proto.ActionStatus_SUCCESS, }, nil } diff --git a/x-pack/elastic-agent/pkg/basecmd/cmd.go b/x-pack/elastic-agent/pkg/basecmd/cmd.go index 9b957916fb1..b30b540d472 100644 --- a/x-pack/elastic-agent/pkg/basecmd/cmd.go +++ b/x-pack/elastic-agent/pkg/basecmd/cmd.go @@ -7,6 +7,7 @@ package basecmd import ( "github.com/spf13/cobra" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/basecmd/restart" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/basecmd/version" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" ) @@ -14,6 +15,7 @@ import ( // NewDefaultCommandsWithArgs returns a list of default commands to executes. func NewDefaultCommandsWithArgs(args []string, streams *cli.IOStreams) []*cobra.Command { return []*cobra.Command{ + restart.NewCommandWithArgs(streams), version.NewCommandWithArgs(streams), } } diff --git a/x-pack/elastic-agent/pkg/basecmd/restart/cmd.go b/x-pack/elastic-agent/pkg/basecmd/restart/cmd.go new file mode 100644 index 00000000000..ebb3bf6effd --- /dev/null +++ b/x-pack/elastic-agent/pkg/basecmd/restart/cmd.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package restart + +import ( + "context" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" +) + +// NewCommandWithArgs returns a new version command. +func NewCommandWithArgs(streams *cli.IOStreams) *cobra.Command { + return &cobra.Command{ + Use: "restart", + Short: "Restart the currently running Elastic Agent daemon", + RunE: func(cmd *cobra.Command, _ []string) error { + c := client.New() + err := c.Connect(context.Background()) + if err != nil { + return errors.New(err, "Failed communicating to running daemon", errors.TypeNetwork, errors.M("socket", control.Address())) + } + defer c.Disconnect() + err = c.Restart(context.Background()) + if err != nil { + return errors.New(err, "Failed trigger restart of daemon") + } + return nil + }, + } +} diff --git a/x-pack/elastic-agent/pkg/basecmd/version/cmd.go b/x-pack/elastic-agent/pkg/basecmd/version/cmd.go index 0bf25438e80..b4e602759cb 100644 --- a/x-pack/elastic-agent/pkg/basecmd/version/cmd.go +++ b/x-pack/elastic-agent/pkg/basecmd/version/cmd.go @@ -5,32 +5,95 @@ package version import ( + "context" "fmt" "github.com/spf13/cobra" + "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) +// Output returns the output when `--yaml` is used. +type Output struct { + Binary *release.VersionInfo `yaml:"binary"` + Daemon *release.VersionInfo `yaml:"daemon,omitempty"` +} + // NewCommandWithArgs returns a new version command. func NewCommandWithArgs(streams *cli.IOStreams) *cobra.Command { - return &cobra.Command{ + cmd := &cobra.Command{ Use: "version", Short: "Display the version of the elastic-agent.", - Run: func(_ *cobra.Command, _ []string) { - version := release.Version() - if release.Snapshot() { - version = version + "-SNAPSHOT" + Run: func(cmd *cobra.Command, _ []string) { + var daemon *release.VersionInfo + var daemonError error + + binary := release.Info() + binaryOnly, _ := cmd.Flags().GetBool("binary-only") + if !binaryOnly { + c := client.New() + daemonError = c.Connect(context.Background()) + if daemonError == nil { + defer c.Disconnect() + + var version client.Version + version, daemonError = c.Version(context.Background()) + if daemonError == nil { + daemon = &release.VersionInfo{ + Version: version.Version, + Commit: version.Commit, + BuildTime: version.BuildTime, + Snapshot: version.Snapshot, + } + } + } + } + if daemonError != nil { + fmt.Fprintf(streams.Err, "Failed talking to running daemon: %s\n", daemonError) + } + + outputYaml, _ := cmd.Flags().GetBool("yaml") + if outputYaml { + p := Output{ + Binary: &binary, + Daemon: daemon, + } + out, err := yaml.Marshal(p) + if err != nil { + fmt.Fprintf(streams.Err, "Failed to render YAML: %s\n", err) + } + fmt.Fprintf(streams.Out, "%s", out) + return } - fmt.Fprintf( - streams.Out, - "Agent version is %s (build: %s at %s)\n", - version, - release.Commit(), - release.BuildTime(), - ) + if !binaryOnly { + mismatch := false + str := "" + if daemon != nil { + str = daemon.String() + mismatch = isMismatch(&binary, daemon) + } + if mismatch { + fmt.Fprintf(streams.Err, "WARN: Then running daemon of Elastic Agent does not match this version.\n") + } + fmt.Fprintf(streams.Out, "Daemon: %s\n", str) + } + fmt.Fprintf(streams.Out, "Binary: %s\n", binary.String()) }, } + + cmd.Flags().Bool("binary-only", false, "Version of current binary only") + cmd.Flags().Bool("yaml", false, "Output information in YAML format") + + return cmd +} + +func isMismatch(a *release.VersionInfo, b *release.VersionInfo) bool { + if a.Commit != "unknown" && b.Commit != "unknown" { + return a.Commit != b.Commit + } + return a.Version != b.Version || a.BuildTime != b.BuildTime || a.Snapshot != b.Snapshot } diff --git a/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go b/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go index 111d174608f..119809338d6 100644 --- a/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go +++ b/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go @@ -10,17 +10,90 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" + "github.com/elastic/beats/v7/libbeat/logp" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/server" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) -func TestCmd(t *testing.T) { +func TestCmdBinaryOnly(t *testing.T) { + streams, _, out, _ := cli.NewTestingIOStreams() + cmd := NewCommandWithArgs(streams) + cmd.Flags().Set("binary-only", "true") + cmd.Execute() + version, err := ioutil.ReadAll(out) + + require.NoError(t, err) + assert.True(t, strings.Contains(string(version), "Binary: ")) + assert.False(t, strings.Contains(string(version), "Daemon: ")) +} + +func TestCmdBinaryOnlyYAML(t *testing.T) { + streams, _, out, _ := cli.NewTestingIOStreams() + cmd := NewCommandWithArgs(streams) + cmd.Flags().Set("binary-only", "true") + cmd.Flags().Set("yaml", "true") + cmd.Execute() + version, err := ioutil.ReadAll(out) + + require.NoError(t, err) + + var output Output + err = yaml.Unmarshal(version, &output) + require.NoError(t, err) + + assert.Nil(t, output.Daemon) + assert.Equal(t, release.Info(), *output.Binary) +} + +func TestCmdDaemon(t *testing.T) { + srv := server.New(newErrorLogger(t), nil) + require.NoError(t, srv.Start()) + defer srv.Stop() + + streams, _, out, _ := cli.NewTestingIOStreams() + cmd := NewCommandWithArgs(streams) + cmd.Execute() + version, err := ioutil.ReadAll(out) + + require.NoError(t, err) + assert.True(t, strings.Contains(string(version), "Binary: ")) + assert.True(t, strings.Contains(string(version), "Daemon: ")) +} + +func TestCmdDaemonYAML(t *testing.T) { + srv := server.New(newErrorLogger(t), nil) + require.NoError(t, srv.Start()) + defer srv.Stop() + streams, _, out, _ := cli.NewTestingIOStreams() - NewCommandWithArgs(streams).Execute() + cmd := NewCommandWithArgs(streams) + cmd.Flags().Set("yaml", "true") + cmd.Execute() version, err := ioutil.ReadAll(out) - if !assert.NoError(t, err) { - return - } - assert.True(t, strings.Contains(string(version), "Agent version is")) + require.NoError(t, err) + + var output Output + err = yaml.Unmarshal(version, &output) + require.NoError(t, err) + + assert.Equal(t, release.Info(), *output.Daemon) + assert.Equal(t, release.Info(), *output.Binary) +} + +func newErrorLogger(t *testing.T) *logger.Logger { + t.Helper() + + loggerCfg := logger.DefaultLoggingConfig() + loggerCfg.Level = logp.ErrorLevel + + log, err := logger.NewFromConfig("", loggerCfg) + require.NoError(t, err) + return log } diff --git a/x-pack/elastic-agent/pkg/release/version.go b/x-pack/elastic-agent/pkg/release/version.go index 7c139d943a9..542ea829417 100644 --- a/x-pack/elastic-agent/pkg/release/version.go +++ b/x-pack/elastic-agent/pkg/release/version.go @@ -6,6 +6,7 @@ package release import ( "strconv" + "strings" "time" libbeatVersion "github.com/elastic/beats/v7/libbeat/version" @@ -34,3 +35,37 @@ func Snapshot() bool { val, err := strconv.ParseBool(snapshot) return err == nil && val } + +// VersionInfo is structure used by `version --yaml`. +type VersionInfo struct { + Version string `yaml:"version"` + Commit string `yaml:"commit"` + BuildTime time.Time `yaml:"build_time"` + Snapshot bool `yaml:"snapshot"` +} + +// Info returns current version information. +func Info() VersionInfo { + return VersionInfo{ + Version: Version(), + Commit: Commit(), + BuildTime: BuildTime(), + Snapshot: Snapshot(), + } +} + +// String returns the string format for the version informaiton. +func (v *VersionInfo) String() string { + var sb strings.Builder + + sb.WriteString(v.Version) + if v.Snapshot { + sb.WriteString("-SNAPSHOT") + } + sb.WriteString(" (build: ") + sb.WriteString(v.Commit) + sb.WriteString(" at ") + sb.WriteString(v.BuildTime.Format("2006-01-02 15:04:05 -0700 MST")) + sb.WriteString(")") + return sb.String() +} From b6713399fc768f882b85647db659dcefcb96d07b Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Mon, 3 Aug 2020 11:20:54 -0400 Subject: [PATCH 03/19] Update Suricata dashboards (#20394) This is a followup to #20220 to get the correct dashboard files into place. --- ... => filebeat-suricata-alert-overview.json} | 203 ++++++---- ... => filebeat-suricata-event-overview.json} | 371 +++++++++++++----- x-pack/filebeat/module/suricata/module.yml | 4 +- 3 files changed, 414 insertions(+), 164 deletions(-) rename x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/{Filebeat-Suricata-Alert-Overview.json => filebeat-suricata-alert-overview.json} (71%) rename x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/{Filebeat-Suricata-Event-Overview.json => filebeat-suricata-event-overview.json} (66%) diff --git a/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Alert-Overview.json b/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/filebeat-suricata-alert-overview.json similarity index 71% rename from x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Alert-Overview.json rename to x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/filebeat-suricata-alert-overview.json index 0c26bebbc79..bf71ad88838 100644 --- a/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Alert-Overview.json +++ b/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/filebeat-suricata-alert-overview.json @@ -28,16 +28,16 @@ "i": "1", "w": 23, "x": 0, - "y": 0 + "y": 4 }, "panelIndex": "1", "panelRefName": "panel_0", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, "gridData": { - "h": 22, + "h": 26, "i": "2", "w": 25, "x": 23, @@ -45,7 +45,7 @@ }, "panelIndex": "2", "panelRefName": "panel_1", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -54,11 +54,11 @@ "i": "3", "w": 48, "x": 0, - "y": 37 + "y": 41 }, "panelIndex": "3", "panelRefName": "panel_2", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": { @@ -73,11 +73,11 @@ "i": "4", "w": 23, "x": 0, - "y": 22 + "y": 26 }, "panelIndex": "4", "panelRefName": "panel_3", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": { @@ -92,11 +92,11 @@ "i": "5", "w": 25, "x": 23, - "y": 22 + "y": 26 }, "panelIndex": "5", "panelRefName": "panel_4", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -105,11 +105,11 @@ "i": "7", "w": 12, "x": 11, - "y": 10 + "y": 14 }, "panelIndex": "7", "panelRefName": "panel_5", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -118,15 +118,28 @@ "i": "8", "w": 11, "x": 0, - "y": 10 + "y": 14 }, "panelIndex": "8", "panelRefName": "panel_6", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 4, + "i": "e86b7f30-96da-4f52-9ff0-cefcaadcc914", + "w": 23, + "x": 0, + "y": 0 + }, + "panelIndex": "e86b7f30-96da-4f52-9ff0-cefcaadcc914", + "panelRefName": "panel_7", + "version": "7.9.0-SNAPSHOT" } ], "timeRestore": false, - "title": "[Filebeat Suricata] Alert Overview ECS", + "title": "[Filebeat Suricata] Alert Overview", "version": 1 }, "id": "05268ee0-86d1-11e8-b59d-21efb914e65c-ecs", @@ -171,11 +184,16 @@ "id": "c7b8b8f0-86d8-11e8-b59d-21efb914e65c-ecs", "name": "panel_6", "type": "visualization" + }, + { + "id": "908e8c90-d296-11ea-90e3-8767fe7ccf14", + "name": "panel_7", + "type": "visualization" } ], "type": "dashboard", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY2MCwxXQ==" + "updated_at": "2020-07-30T19:13:51.743Z", + "version": "WzEwMTUsMV0=" }, { "attributes": { @@ -190,7 +208,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Alerting Hosts [Filebeat Suricata] ECS", + "title": "Top Alerting Hosts [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -206,10 +224,17 @@ "enabled": true, "id": "2", "params": { + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-6y", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" @@ -258,6 +283,9 @@ "color": "#eee" } }, + "labels": { + "show": false + }, "legendPosition": "right", "seriesParams": [ { @@ -273,6 +301,13 @@ "valueAxis": "ValueAxis-1" } ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], "type": "histogram", "valueAxes": [ @@ -299,7 +334,7 @@ } ] }, - "title": "Top Alerting Hosts [Filebeat Suricata] ECS", + "title": "Top Alerting Hosts [Filebeat Suricata]", "type": "histogram" } }, @@ -318,8 +353,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1MywxXQ==" + "updated_at": "2020-07-30T19:09:55.677Z", + "version": "WzkwNCwxXQ==" }, { "attributes": { @@ -334,7 +369,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Alert Signatures [Filebeat Suricata] ECS", + "title": "Top Alert Signatures [Filebeat Suricata]", "uiStateJSON": { "vis": { "params": { @@ -360,14 +395,14 @@ "id": "2", "params": { "customLabel": "Alert Signature", - "field": "suricata.eve.alert.signature", + "field": "rule.name", "missingBucket": false, "missingBucketLabel": "Missing", "order": "desc", "orderBy": "1", "otherBucket": false, "otherBucketLabel": "Other", - "size": 10 + "size": 15 }, "schema": "bucket", "type": "terms" @@ -377,7 +412,7 @@ "id": "3", "params": { "customLabel": "Alert Category", - "field": "suricata.eve.alert.category", + "field": "rule.category", "missingBucket": false, "missingBucketLabel": "Missing", "order": "desc", @@ -392,6 +427,7 @@ ], "params": { "perPage": 10, + "percentageCol": "", "showMetricsAtAllLevels": false, "showPartialRows": false, "showTotal": false, @@ -401,7 +437,7 @@ }, "totalFunc": "sum" }, - "title": "Top Alert Signatures [Filebeat Suricata] ECS", + "title": "Top Alert Signatures [Filebeat Suricata]", "type": "table" } }, @@ -420,8 +456,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1NCwxXQ==" + "updated_at": "2020-07-30T19:11:35.746Z", + "version": "Wzk0MywxXQ==" }, { "attributes": { @@ -448,21 +484,16 @@ "alias": null, "disabled": false, "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "key": "suricata.eve.event_type", + "key": "event.kind", "negate": false, "params": { - "query": "alert", - "type": "phrase" + "query": "alert" }, - "type": "phrase", - "value": "alert" + "type": "phrase" }, "query": { - "match": { - "suricata.eve.event_type": { - "query": "alert", - "type": "phrase" - } + "match_phrase": { + "event.kind": "alert" } } }, @@ -477,11 +508,9 @@ "key": "event.module", "negate": false, "params": { - "query": "suricata", - "type": "phrase" + "query": "suricata" }, - "type": "phrase", - "value": "suricata" + "type": "phrase" }, "query": { "match": { @@ -508,7 +537,7 @@ "desc" ] ], - "title": "Alerts [Filebeat Suricata] ECS", + "title": "Alerts [Filebeat Suricata]", "version": 1 }, "id": "1c2bcec0-86d1-11e8-b59d-21efb914e65c-ecs", @@ -536,8 +565,8 @@ } ], "type": "search", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1NSwxXQ==" + "updated_at": "2020-07-30T18:46:18.887Z", + "version": "WzYyNiwxXQ==" }, { "attributes": { @@ -552,7 +581,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Alert - Source Location [Filebeat Suricata] ECS", + "title": "Alert - Source Location [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -571,11 +600,6 @@ "autoPrecision": true, "field": "source.geo.location", "isFilteredByCollar": true, - "mapCenter": [ - 0, - 0 - ], - "mapZoom": 2, "precision": 2, "useGeocentroid": true }, @@ -622,7 +646,7 @@ ] } }, - "title": "Alert - Source Location [Filebeat Suricata] ECS", + "title": "Alert - Source Location [Filebeat Suricata]", "type": "tile_map" } }, @@ -641,8 +665,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1NiwxXQ==" + "updated_at": "2020-07-30T19:13:13.311Z", + "version": "Wzk5MCwxXQ==" }, { "attributes": { @@ -657,7 +681,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Alert - Destination Location [Filebeat Suricata] ECS", + "title": "Alert - Destination Location [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -676,11 +700,6 @@ "autoPrecision": true, "field": "destination.geo.location", "isFilteredByCollar": true, - "mapCenter": [ - 0, - 0 - ], - "mapZoom": 2, "precision": 2, "useGeocentroid": true }, @@ -727,7 +746,7 @@ ] } }, - "title": "Alert - Destination Location [Filebeat Suricata] ECS", + "title": "Alert - Destination Location [Filebeat Suricata]", "type": "tile_map" } }, @@ -746,8 +765,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1NywxXQ==" + "updated_at": "2020-07-30T19:13:34.582Z", + "version": "WzEwMDQsMV0=" }, { "attributes": { @@ -762,7 +781,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Alerts - Top Destination Countries [Filebeat Suricata] ECS", + "title": "Alerts - Top Destination Countries [Filebeat Suricata]", "uiStateJSON": { "vis": { "params": { @@ -803,6 +822,7 @@ ], "params": { "perPage": 5, + "percentageCol": "", "showMetricsAtAllLevels": false, "showPartialRows": false, "showTotal": false, @@ -812,7 +832,7 @@ }, "totalFunc": "sum" }, - "title": "Alerts - Top Destination Countries [Filebeat Suricata] ECS", + "title": "Alerts - Top Destination Countries [Filebeat Suricata]", "type": "table" } }, @@ -831,8 +851,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1OCwxXQ==" + "updated_at": "2020-07-30T19:12:34.381Z", + "version": "Wzk2OSwxXQ==" }, { "attributes": { @@ -847,7 +867,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Alerts - Top Source Countries [Filebeat Suricata] ECS", + "title": "Alerts - Top Source Countries [Filebeat Suricata]", "uiStateJSON": { "vis": { "params": { @@ -888,6 +908,7 @@ ], "params": { "perPage": 5, + "percentageCol": "", "showMetricsAtAllLevels": false, "showPartialRows": false, "showTotal": false, @@ -897,7 +918,7 @@ }, "totalFunc": "sum" }, - "title": "Alerts - Top Source Countries [Filebeat Suricata] ECS", + "title": "Alerts - Top Source Countries [Filebeat Suricata]", "type": "table" } }, @@ -916,8 +937,46 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:12.641Z", - "version": "WzY1OSwxXQ==" + "updated_at": "2020-07-30T19:12:12.735Z", + "version": "Wzk1NCwxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Navigation [Filebeat Suricata]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": 18, + "markdown": "![Hello World](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADwAAAAyCAYAAAAA9rgCAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAACXBIWXMAAJ17AACdewE8n3fEAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpMwidZAAAN5UlEQVRoBe1ZeXCU5Rn/7b25s9nEHJAQAhggCIKCThFpFRXFa+yhTtVSndaZUvWPVjqt1bGHY6cdrVNq66C21UpbT7wwxqMjAh4oBCGQC8KRY7Ob3WQ3e9/9Pe/uh2ukrePM7h/oM2S/73ve87l+z/O+6NIkfIFI/wWSVYn6pcAnu8W/tPCXFj7JNPClS59kBv2UOF84Cxs/pYJ8M1jYHS/udIBOV1idF1ZgqWJ1Ov6jpFlKp1PqrVCCF07grLDJeBT+oX5Ex52wVk9DxYy5muwFeRbGn7LCphIx9G26A933noZ0KoFEyAvHe1uUAgoiLRcpiMCMWiVPYGQAvg/vg7lhFpLRIGL+cQxsuBTO95/PyCuKyTMVxqVFDoZtPBSAjivqjJUY/OfXYWlciXl370bVnMUZMXNiO19yF0bg7O6jXicgGJWKw1y3CG3rNsNcZlOonQtk+RJW5i2MwFnLBYcOQGeSZVNIx4eRSiblo6CU/xhmXIr1ktEwQke2Qm+lfPoixMcnEHQMZIQtQOxqWs27wBoMxYKTFHILLdwMmhdSb4RdR7V9FOyZd4G1EsNosUJfvATppI+opVe4HXFnBS4AWGkazbvAUlkJGYvLYKlZhnR0AqnIXpS2fgeRsV5Ibs4AluYL2tby88y/wLJvFcd6GIpsMNpWwFx9BUzl05Dw9SPgOCI9WIicRAJrohiLK5COuVA0/UyEht7m8xwMvvIA3TxBLxfdaz2VDvLyUxALazk2HvAQuHqRDPtgrVvKdBxFMngMrn3blXCFsHLeBdaOglGfB6HDj9Gl52Dyo9+w2rLSsnEYy5sI2uG8WPNEk+ZdYIlfId+RLlrXRUHLVS6OjOyEscSORGBUAdeJNpcPXt4F1tzZ270NejNFSEUodDOSoR7+uVExbw0q5yxVsun0WhLLh6iZOfNbWop1mZbiLDqCBzczD5cTjUMEqDLEJ/ahePpizLhwbVY68YT8C/yxhbk5FW/KBXPQUr4VL7sveUzlad/aM4u2WvwGRg4h5tpNC7dwbIzDo6qmrllykZo0LTW1WiZnXW05bU55apTLy33PRflP8LWB2cODXLPIFcsn9SsLkHOiKmgqb+o3R2pXN7KUt/f9zMlbXecYlFvrLWQZ+E7SaU/1NeXnBHOfcE9Thv23PkaxlgibjEV4IJ+guxlgKa/iJtjE/Jhg0S+KM1gs0BvNPOEk1EEgw2O5aDQhEQnRVeXkIwqiFq3Fak55jwUmMLFnIwxl4s5+SmmhkfthqroUxqJStU0ZG/W61VhzuZ2K+DjSZLzK0+SZSyvVGmo98YocZYiCDSYzDGY5nfDsHfRlqjjKY+I47c7MKIPGmAeH23/BnPgRuxphbbgM827agAg30f3HFUiFBzDz+u2oXrAcQecg+jZeSOw5iJa178M+dxkOv/QgvJ3rmWJWUVEmVlGNsC26HA1nr4G3fw9io50w10odPcn5TUhFAWv9UqWYycE+HHvpXsTcO5Dw96P+4ifQeN63Zc9UZBDdD11LdO+AseIctN3yCkwsUQeevx/+nk1E+wbOGYLeVMGqrQN1qzdj+ooraZQ4+p74McJHH2Ho6NB2mwPWqlraNg3jeH8nDm5YAWM1PU2MGaN2PBtZFNxHA+mQCg2wUCBfWZC74KBU6KDicWa1sVQ8osamjHsRG3MhVsw09MFGRFx/gKmsCpVn3QH/gXuoDHpK0VJ6ElC14DzlUT33t2bPyPSGY0B0YkjNKT/RSQ9jvwP60oWcdzuiPrcSOObpReRgDywto9yhAQkWMuEuKmjFuBobD/gQGWmHztKGhGc/wuOjSmDZu9657XEY6CmG4jPQdM0WzPkhrbbilxxIPxYPpXfJX677yLVBLk/cX7qbbOdg5k2voXjO92CZ1gxn+62I+VwoaVyI6CCFXPoAXXkB21pR3XY2jr3+VxlGS1Wj9sLHMfv2DpQ2LVKblp+IxyFZjBasQoqGCHtGVFv9+bei5dZ2mCrPZbsHRTO+jzl3vQ7b/OWqXYqcpH+YIFmlvsNj1GSWjOGRfyurVi66EfVnXaLY9nnL1DPq92a6qatj2VqW5FX701hxbsxcjrozL0B583wc+P3DKu+aK+pQVNOE4tb5mH7+Dej7++1ovbkDcl3r2/eg0mPF6T9F06rrtdmVNwmWhJxHxCikpOgeYX6DYWWfm8nbnt3PMgwYHjWzUHfGKumoSBTDWyTukR5Irwo5+hVfagK9zlCiXCrqOaKY6RSvX/iX+eBqstJnIfaTvaUScRTZ6mCpv0G5vfq216Fs/ncJfPR1HhJKamcg5B5h3A1AlrK1natWkL5CGsCEHb1KaWkpVli0hB09ql1iMckwkpCSMElnxwmgCoVGD8mRW5HcsEQcexXwiXb1pXOuUA3+fb/D0LZnM6eWjFozIz7LL/vLEJV3Oam6lqUPEtRpnBjMFXakQ2N0MzeqWldi4MUN8O7fCpHfWsI/IrOQCg1RG+eQg0XEsRu6IqC0hZ7Hu7Dw6B6VNcRSmlKUlvktpNdL7KU5rluFXMms1cSM0xB1PosYUVv1qT/namqK2GxbhqGnvgF31ztcmGr7H5QRLmNRNQlLQhPXMhv1KqUEHUcQHP4Xgoy7dHE1TEVl0M2+EFtumw437Kg5cw2cww70HAQOyUWmiVJlSdO1hFPE+QIsZSsYmyupnFlITbyMOO+yFYmC+SJ/x4lyy1VwxLlTeURl6woYSlpY1QWY9sZUN31pXTMar25HzLGTqeMsDPxtOfzDh1RjbvFwfFK+GKkPMzWulb7BaApDo0D3oXG8tflpPPbzW9D+KvDiw0DJnAweHOjqw9NPAE/+6GKUNbQg0bwczz8J7PigHrHkx3GjVWeOESee+y2wzzUNxpoWDPtnQK7Agh5qSBFTDDeg7SHLVMge92wnCC9ESf1MClyrMkh4bEh14eVSGvXLVqPuskeYjt4nYhpw+On1qlFHF5HMIx6T9RqlUV+AwjGcvJGMfrtGUtjyFLCt413846pv4dieHaD3Yu32t1FT36DmMpkMqOBb1YIraRa6VCwMG79LSnj4133CTqr/6NAgBGpitlNha5iB/lgrHtsE9PQdUe1yQTIeSCPGsFd3B4orSO5AglhrqlqisMJinymYRwA8rHrotaWaV9+EiiU/Y3AnEex7Dp7eXZCLNzOtGScWRDN4ggDd9KVngI4tnCTLE2WIU1a3tGLWNZfAoPOjYv4CnHr64uw2KCMVy3WRYEUn2rOWlCjlxdxjiEdZiQixj3a6Gh3oRzlZRVYTxp0jKCu1giGP4f4+6cmxOuw8mMLmjQwLlyYFFeQ6DBO9z2gtZ/5lGuNaggPh0W41Si8LaOjWdPE6ImkdzKxzIyO9iCWS2NcLvPAg4A5lJqVtYCDyMXUf/48puZ6h0VF/+nKsXrcekT2A640uHOraS26GpIgRUk9OVV5lFxxCiGYM+DKAkqSyZT/JRAJDXZ2wzwC6Nj+MXzdMw+G3noN9NgXu2qXaDQY9TPoUaEx4gh8L/FHnfjhCnHfoTfTePw2B/RtgtFcRyHYwvv3QS5xqtWtxdT30lSvRd4BaGw0hntLhg210HU6qXb9I6kgMkEEyWTN1q7xLFoiFgphJy1ZfuVJZ78C726XpOMm2JL+mGCf22jqUssYQBR7a26n6GMU0pMCkF67OdmJKDWLOYzA20gBDR2GqaYTrvWeoIK86eIiCZIQhGxLRcAi7dnyIN7cCr7yyH4QNTHqHacAapCd3IsKCRD/42qMY+7ADvtFB7Hj5eXS078BLr1GAyjrYaIVTzlgCKfG7dmxFJBhAV1YIK0OjsvoU2R+tolf/ZxPx+5Tl5p5/ubJnz6tPwj+RQVVRrNhYHTbicY6twcyLboGFgb3tT3eic9tbGDjQhd49uzHpGYfvPaIqq4obn9qJOzvduH7TVsRdgwjQMydYvoqvyJyiRA3oJicmEOp9DUbixyW/fRNr/uxGw9pd2NWdwAR5MQ+rr/Ej+/Didatx36omPHnZVXBsH0IjXantjKUqPS3+5s3KbXpefBS/+moN3nnobgS5yPwb1qOGriakeYiG6nPP+ooSzt2xG0f7iG6kFFFGvEBnJBBmC5vzrrsJXnqznNQ2XfM1/KTtNHS9s42bc4FeifJZ52LmvDaU2eyY2bYQRU3zIKXF2MgwfzOKljm1VDbucsK/n3m9zYbZCxbCZrejed4CONKL8PRfBPCOQj9WvgQ7Gac+7otpFPVrVuEHHd2wnVKrJr3g2htw/j13wfvuILwfRjDx7hAWrluLq25dr9rlJ84jJCs8SNEuq89onYcK4pXwOt9oly6q0pFIjU44lUcIb/Zpi7DurTeQjmRypHhSma0KQ33d8PC9vKEJJgEUkrWoCOWNpyrlD3bzpECSI6Ccv+S4KDR8sA9i+7KmxbAWC8Qx7Jg/G2Y1q3773mGNQXdIe6lRn9tNdzOirpGLEJ01N9FQc+hQP/xeL4qIro2zT4WBfaWPtIuLRUMhVlZm2OvqFc8z6kA8FmM/Az1hOibHPQj6/URQE6rYR0+g08b7vRNwO0a4OTNOmd6I4OQkIsQDa3EJbDWZsJHNjztHEY1EYCmykl8LWSPBNSwUTvp5ifhhhp2Z2GKvrZchioQv8xl4yFECaw3aU2rpzMV4Jj40obV2eWqbzeV9nvfctT7PeG3MZ92PElg6U4LMWMlb/JtK6lBBpmqZ0uf4+Bz+VN7U79z5j7cJU1tb9pMznzQd75fl/79vGSOU2+8/QDU5FFmyNvEAAAAASUVORK5CYII=) [Events](/app/dashboards#/view/78289c40-86da-11e8-b59d-21efb914e65c-ecs) | [Alerts](/app/dashboards#/view/05268ee0-86d1-11e8-b59d-21efb914e65c-ecs)", + "openLinksInNewTab": false + }, + "title": "Navigation [Filebeat Suricata]", + "type": "markdown" + } + }, + "id": "908e8c90-d296-11ea-90e3-8767fe7ccf14", + "migrationVersion": { + "visualization": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-07-30T18:57:50.040Z", + "version": "Wzc1MywxXQ==" } ], "version": "7.9.0-SNAPSHOT" diff --git a/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Event-Overview.json b/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/filebeat-suricata-event-overview.json similarity index 66% rename from x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Event-Overview.json rename to x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/filebeat-suricata-event-overview.json index d263bd7e617..908f98394cb 100644 --- a/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/Filebeat-Suricata-Event-Overview.json +++ b/x-pack/filebeat/module/suricata/_meta/kibana/7/dashboard/filebeat-suricata-event-overview.json @@ -28,11 +28,11 @@ "i": "1", "w": 48, "x": 0, - "y": 0 + "y": 4 }, "panelIndex": "1", "panelRefName": "panel_0", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -41,11 +41,11 @@ "i": "2", "w": 9, "x": 0, - "y": 20 + "y": 24 }, "panelIndex": "2", "panelRefName": "panel_1", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -54,11 +54,11 @@ "i": "3", "w": 11, "x": 19, - "y": 20 + "y": 24 }, "panelIndex": "3", "panelRefName": "panel_2", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -67,11 +67,11 @@ "i": "4", "w": 48, "x": 0, - "y": 10 + "y": 14 }, "panelIndex": "4", "panelRefName": "panel_3", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -80,11 +80,11 @@ "i": "5", "w": 48, "x": 0, - "y": 34 + "y": 38 }, "panelIndex": "5", "panelRefName": "panel_4", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -93,11 +93,11 @@ "i": "6", "w": 9, "x": 30, - "y": 20 + "y": 24 }, "panelIndex": "6", "panelRefName": "panel_5", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -106,11 +106,11 @@ "i": "7", "w": 9, "x": 39, - "y": 20 + "y": 24 }, "panelIndex": "7", "panelRefName": "panel_6", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -119,11 +119,11 @@ "i": "8", "w": 10, "x": 9, - "y": 20 + "y": 24 }, "panelIndex": "8", "panelRefName": "panel_7", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" }, { "embeddableConfig": {}, @@ -132,15 +132,43 @@ "i": "9", "w": 48, "x": 0, - "y": 53 + "y": 57 }, "panelIndex": "9", "panelRefName": "panel_8", - "version": "7.3.0" + "version": "7.9.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 4, + "i": "78f64fb8-a6ed-4960-a73b-a8c42c40f799", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "78f64fb8-a6ed-4960-a73b-a8c42c40f799", + "panelRefName": "panel_9", + "version": "7.9.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 4, + "i": "63e14057-b48b-48fe-b3e2-84f7690d60e8", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "63e14057-b48b-48fe-b3e2-84f7690d60e8", + "panelRefName": "panel_10", + "version": "7.9.0-SNAPSHOT" } ], "timeRestore": false, - "title": "[Filebeat Suricata] Events Overview ECS", + "title": "[Filebeat Suricata] Events Overview", "version": 1 }, "id": "78289c40-86da-11e8-b59d-21efb914e65c-ecs", @@ -195,11 +223,21 @@ "id": "d57a2db0-86ca-11e8-b59d-21efb914e65c-ecs", "name": "panel_8", "type": "search" + }, + { + "id": "908e8c90-d296-11ea-90e3-8767fe7ccf14", + "name": "panel_9", + "type": "visualization" + }, + { + "id": "169c0600-d297-11ea-90e3-8767fe7ccf14", + "name": "panel_10", + "type": "visualization" } ], "type": "dashboard", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY3MCwxXQ==" + "updated_at": "2020-07-30T19:08:06.676Z", + "version": "Wzg3MiwxXQ==" }, { "attributes": { @@ -214,7 +252,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Activity Types over Time [Filebeat Suricata] ECS", + "title": "Activity Types over Time [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -230,10 +268,17 @@ "enabled": true, "id": "2", "params": { + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-6y", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" @@ -282,6 +327,9 @@ "color": "#eee" } }, + "labels": { + "show": false + }, "legendPosition": "right", "seriesParams": [ { @@ -297,6 +345,13 @@ "valueAxis": "ValueAxis-1" } ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], "type": "histogram", "valueAxes": [ @@ -323,7 +378,7 @@ } ] }, - "title": "Activity Types over Time [Filebeat Suricata] ECS", + "title": "Activity Types over Time [Filebeat Suricata]", "type": "histogram" } }, @@ -342,8 +397,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2MSwxXQ==" + "updated_at": "2020-07-30T18:59:25.617Z", + "version": "Wzc2OCwxXQ==" }, { "attributes": { @@ -358,7 +413,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Event Types [Filebeat Suricata] ECS", + "title": "Event Types [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -374,7 +429,8 @@ "enabled": true, "id": "2", "params": { - "field": "suricata.eve.event_type", + "customLabel": "ECS Event Type", + "field": "event.type", "missingBucket": false, "missingBucketLabel": "Missing", "order": "desc", @@ -385,6 +441,23 @@ }, "schema": "segment", "type": "terms" + }, + { + "enabled": true, + "id": "3", + "params": { + "customLabel": "Suricata Event Type", + "field": "suricata.eve.event_type", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" } ], "params": { @@ -400,7 +473,7 @@ "legendPosition": "bottom", "type": "pie" }, - "title": "Event Types [Filebeat Suricata] ECS", + "title": "Event Types [Filebeat Suricata]", "type": "pie" } }, @@ -419,8 +492,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2MiwxXQ==" + "updated_at": "2020-07-30T19:06:59.207Z", + "version": "Wzg1OCwxXQ==" }, { "attributes": { @@ -435,7 +508,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Application Protocols [Filebeat Suricata] ECS", + "title": "Top Network Protocols [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -477,7 +550,7 @@ "legendPosition": "bottom", "type": "pie" }, - "title": "Top Application Protocols [Filebeat Suricata] ECS", + "title": "Top Network Protocols [Filebeat Suricata]", "type": "pie" } }, @@ -496,8 +569,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2MywxXQ==" + "updated_at": "2020-07-30T18:49:07.711Z", + "version": "WzY3NSwxXQ==" }, { "attributes": { @@ -512,7 +585,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Hosts Generating Events [Filebeat Suricata] ECS", + "title": "Top Hosts Generating Events [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -528,10 +601,17 @@ "enabled": true, "id": "2", "params": { + "drop_partials": false, "extended_bounds": {}, "field": "@timestamp", "interval": "auto", - "min_doc_count": 1 + "min_doc_count": 1, + "scaleMetricValues": false, + "timeRange": { + "from": "now-6y", + "to": "now" + }, + "useNormalizedEsInterval": true }, "schema": "segment", "type": "date_histogram" @@ -580,6 +660,9 @@ "color": "#eee" } }, + "labels": { + "show": false + }, "legendPosition": "right", "seriesParams": [ { @@ -595,6 +678,13 @@ "valueAxis": "ValueAxis-1" } ], + "thresholdLine": { + "color": "#E7664C", + "show": false, + "style": "full", + "value": 10, + "width": 1 + }, "times": [], "type": "histogram", "valueAxes": [ @@ -621,7 +711,7 @@ } ] }, - "title": "Top Hosts Generating Events [Filebeat Suricata] ECS", + "title": "Top Hosts Generating Events [Filebeat Suricata]", "type": "histogram" } }, @@ -640,14 +730,13 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2NCwxXQ==" + "updated_at": "2020-07-30T18:59:45.518Z", + "version": "Wzc4MCwxXQ==" }, { "attributes": { "columns": [ "host.name", - "suricata.eve.event_type", "suricata.eve.flow_id", "network.transport", "source.ip", @@ -670,21 +759,16 @@ "alias": null, "disabled": false, "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "key": "suricata.eve.event_type", - "negate": true, + "key": "event.kind", + "negate": false, "params": { - "query": "stats", - "type": "phrase" + "query": "event" }, - "type": "phrase", - "value": "stats" + "type": "phrase" }, "query": { - "match": { - "suricata.eve.event_type": { - "query": "stats", - "type": "phrase" - } + "match_phrase": { + "event.kind": "event" } } }, @@ -699,11 +783,9 @@ "key": "event.module", "negate": false, "params": { - "query": "suricata", - "type": "phrase" + "query": "suricata" }, - "type": "phrase", - "value": "suricata" + "type": "phrase" }, "query": { "match": { @@ -730,7 +812,7 @@ "desc" ] ], - "title": "Events [Filebeat Suricata] ECS", + "title": "Events [Filebeat Suricata]", "version": 1 }, "id": "13dd22f0-86cc-11e8-b59d-21efb914e65c-ecs", @@ -758,8 +840,8 @@ } ], "type": "search", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2NSwxXQ==" + "updated_at": "2020-07-30T18:45:13.363Z", + "version": "WzYyMCwxXQ==" }, { "attributes": { @@ -774,7 +856,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Connection Source Countries [Filebeat Suricata] ECS", + "title": "Top Connection Source Countries [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -809,9 +891,9 @@ "minFontSize": 18, "orientation": "single", "scale": "linear", - "showLabel": true + "showLabel": false }, - "title": "Top Connection Source Countries [Filebeat Suricata] ECS", + "title": "Top Connection Source Countries [Filebeat Suricata]", "type": "tagcloud" } }, @@ -830,8 +912,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2NiwxXQ==" + "updated_at": "2020-07-30T18:49:36.842Z", + "version": "WzY4OCwxXQ==" }, { "attributes": { @@ -846,7 +928,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Connection Destination Countries [Filebeat Suricata] ECS", + "title": "Top Connection Destination Countries [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -881,9 +963,9 @@ "minFontSize": 18, "orientation": "single", "scale": "linear", - "showLabel": true + "showLabel": false }, - "title": "Top Connection Destination Countries [Filebeat Suricata] ECS", + "title": "Top Connection Destination Countries [Filebeat Suricata]", "type": "tagcloud" } }, @@ -902,8 +984,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2NywxXQ==" + "updated_at": "2020-07-30T18:50:04.448Z", + "version": "WzcwNSwxXQ==" }, { "attributes": { @@ -918,7 +1000,7 @@ } }, "savedSearchRefName": "search_0", - "title": "Top Network Protocols [Filebeat Suricata] ECS", + "title": "Top Transport Protocols [Filebeat Suricata]", "uiStateJSON": {}, "version": 1, "visState": { @@ -960,7 +1042,7 @@ "legendPosition": "bottom", "type": "pie" }, - "title": "Top Network Protocols [Filebeat Suricata] ECS", + "title": "Top Transport Protocols [Filebeat Suricata]", "type": "pie" } }, @@ -979,8 +1061,8 @@ } ], "type": "visualization", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2OCwxXQ==" + "updated_at": "2020-07-30T18:48:19.957Z", + "version": "WzY0NiwxXQ==" }, { "attributes": { @@ -1006,21 +1088,16 @@ "alias": null, "disabled": false, "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "key": "suricata.eve.event_type", + "key": "event.kind", "negate": false, "params": { - "query": "stats", - "type": "phrase" + "query": "metric" }, - "type": "phrase", - "value": "stats" + "type": "phrase" }, "query": { - "match": { - "suricata.eve.event_type": { - "query": "stats", - "type": "phrase" - } + "match_phrase": { + "event.kind": "metric" } } }, @@ -1035,11 +1112,9 @@ "key": "event.module", "negate": false, "params": { - "query": "suricata", - "type": "phrase" + "query": "suricata" }, - "type": "phrase", - "value": "suricata" + "type": "phrase" }, "query": { "match": { @@ -1066,7 +1141,7 @@ "desc" ] ], - "title": "Host Stats [Filebeat Suricata] ECS", + "title": "Host Stats [Filebeat Suricata]", "version": 1 }, "id": "d57a2db0-86ca-11e8-b59d-21efb914e65c-ecs", @@ -1094,8 +1169,124 @@ } ], "type": "search", - "updated_at": "2020-07-23T17:51:13.671Z", - "version": "WzY2OSwxXQ==" + "updated_at": "2020-07-30T18:45:50.678Z", + "version": "WzYyMywxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Navigation [Filebeat Suricata]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "fontSize": 18, + "markdown": "![Hello World](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADwAAAAyCAYAAAAA9rgCAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAACXBIWXMAAJ17AACdewE8n3fEAAABWWlUWHRYTUw6Y29tLmFkb2JlLnhtcAAAAAAAPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUgNS40LjAiPgogICA8cmRmOlJERiB4bWxuczpyZGY9Imh0dHA6Ly93d3cudzMub3JnLzE5OTkvMDIvMjItcmRmLXN5bnRheC1ucyMiPgogICAgICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDx0aWZmOk9yaWVudGF0aW9uPjE8L3RpZmY6T3JpZW50YXRpb24+CiAgICAgIDwvcmRmOkRlc2NyaXB0aW9uPgogICA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgpMwidZAAAN5UlEQVRoBe1ZeXCU5Rn/7b25s9nEHJAQAhggCIKCThFpFRXFa+yhTtVSndaZUvWPVjqt1bGHY6cdrVNq66C21UpbT7wwxqMjAh4oBCGQC8KRY7Ob3WQ3e9/9Pe/uh2ukrePM7h/oM2S/73ve87l+z/O+6NIkfIFI/wWSVYn6pcAnu8W/tPCXFj7JNPClS59kBv2UOF84Cxs/pYJ8M1jYHS/udIBOV1idF1ZgqWJ1Ov6jpFlKp1PqrVCCF07grLDJeBT+oX5Ex52wVk9DxYy5muwFeRbGn7LCphIx9G26A933noZ0KoFEyAvHe1uUAgoiLRcpiMCMWiVPYGQAvg/vg7lhFpLRIGL+cQxsuBTO95/PyCuKyTMVxqVFDoZtPBSAjivqjJUY/OfXYWlciXl370bVnMUZMXNiO19yF0bg7O6jXicgGJWKw1y3CG3rNsNcZlOonQtk+RJW5i2MwFnLBYcOQGeSZVNIx4eRSiblo6CU/xhmXIr1ktEwQke2Qm+lfPoixMcnEHQMZIQtQOxqWs27wBoMxYKTFHILLdwMmhdSb4RdR7V9FOyZd4G1EsNosUJfvATppI+opVe4HXFnBS4AWGkazbvAUlkJGYvLYKlZhnR0AqnIXpS2fgeRsV5Ibs4AluYL2tby88y/wLJvFcd6GIpsMNpWwFx9BUzl05Dw9SPgOCI9WIicRAJrohiLK5COuVA0/UyEht7m8xwMvvIA3TxBLxfdaz2VDvLyUxALazk2HvAQuHqRDPtgrVvKdBxFMngMrn3blXCFsHLeBdaOglGfB6HDj9Gl52Dyo9+w2rLSsnEYy5sI2uG8WPNEk+ZdYIlfId+RLlrXRUHLVS6OjOyEscSORGBUAdeJNpcPXt4F1tzZ270NejNFSEUodDOSoR7+uVExbw0q5yxVsun0WhLLh6iZOfNbWop1mZbiLDqCBzczD5cTjUMEqDLEJ/ahePpizLhwbVY68YT8C/yxhbk5FW/KBXPQUr4VL7sveUzlad/aM4u2WvwGRg4h5tpNC7dwbIzDo6qmrllykZo0LTW1WiZnXW05bU55apTLy33PRflP8LWB2cODXLPIFcsn9SsLkHOiKmgqb+o3R2pXN7KUt/f9zMlbXecYlFvrLWQZ+E7SaU/1NeXnBHOfcE9Thv23PkaxlgibjEV4IJ+guxlgKa/iJtjE/Jhg0S+KM1gs0BvNPOEk1EEgw2O5aDQhEQnRVeXkIwqiFq3Fak55jwUmMLFnIwxl4s5+SmmhkfthqroUxqJStU0ZG/W61VhzuZ2K+DjSZLzK0+SZSyvVGmo98YocZYiCDSYzDGY5nfDsHfRlqjjKY+I47c7MKIPGmAeH23/BnPgRuxphbbgM827agAg30f3HFUiFBzDz+u2oXrAcQecg+jZeSOw5iJa178M+dxkOv/QgvJ3rmWJWUVEmVlGNsC26HA1nr4G3fw9io50w10odPcn5TUhFAWv9UqWYycE+HHvpXsTcO5Dw96P+4ifQeN63Zc9UZBDdD11LdO+AseIctN3yCkwsUQeevx/+nk1E+wbOGYLeVMGqrQN1qzdj+ooraZQ4+p74McJHH2Ho6NB2mwPWqlraNg3jeH8nDm5YAWM1PU2MGaN2PBtZFNxHA+mQCg2wUCBfWZC74KBU6KDicWa1sVQ8osamjHsRG3MhVsw09MFGRFx/gKmsCpVn3QH/gXuoDHpK0VJ6ElC14DzlUT33t2bPyPSGY0B0YkjNKT/RSQ9jvwP60oWcdzuiPrcSOObpReRgDywto9yhAQkWMuEuKmjFuBobD/gQGWmHztKGhGc/wuOjSmDZu9657XEY6CmG4jPQdM0WzPkhrbbilxxIPxYPpXfJX677yLVBLk/cX7qbbOdg5k2voXjO92CZ1gxn+62I+VwoaVyI6CCFXPoAXXkB21pR3XY2jr3+VxlGS1Wj9sLHMfv2DpQ2LVKblp+IxyFZjBasQoqGCHtGVFv9+bei5dZ2mCrPZbsHRTO+jzl3vQ7b/OWqXYqcpH+YIFmlvsNj1GSWjOGRfyurVi66EfVnXaLY9nnL1DPq92a6qatj2VqW5FX701hxbsxcjrozL0B583wc+P3DKu+aK+pQVNOE4tb5mH7+Dej7++1ovbkDcl3r2/eg0mPF6T9F06rrtdmVNwmWhJxHxCikpOgeYX6DYWWfm8nbnt3PMgwYHjWzUHfGKumoSBTDWyTukR5Irwo5+hVfagK9zlCiXCrqOaKY6RSvX/iX+eBqstJnIfaTvaUScRTZ6mCpv0G5vfq216Fs/ncJfPR1HhJKamcg5B5h3A1AlrK1natWkL5CGsCEHb1KaWkpVli0hB09ql1iMckwkpCSMElnxwmgCoVGD8mRW5HcsEQcexXwiXb1pXOuUA3+fb/D0LZnM6eWjFozIz7LL/vLEJV3Oam6lqUPEtRpnBjMFXakQ2N0MzeqWldi4MUN8O7fCpHfWsI/IrOQCg1RG+eQg0XEsRu6IqC0hZ7Hu7Dw6B6VNcRSmlKUlvktpNdL7KU5rluFXMms1cSM0xB1PosYUVv1qT/namqK2GxbhqGnvgF31ztcmGr7H5QRLmNRNQlLQhPXMhv1KqUEHUcQHP4Xgoy7dHE1TEVl0M2+EFtumw437Kg5cw2cww70HAQOyUWmiVJlSdO1hFPE+QIsZSsYmyupnFlITbyMOO+yFYmC+SJ/x4lyy1VwxLlTeURl6woYSlpY1QWY9sZUN31pXTMar25HzLGTqeMsDPxtOfzDh1RjbvFwfFK+GKkPMzWulb7BaApDo0D3oXG8tflpPPbzW9D+KvDiw0DJnAweHOjqw9NPAE/+6GKUNbQg0bwczz8J7PigHrHkx3GjVWeOESee+y2wzzUNxpoWDPtnQK7Agh5qSBFTDDeg7SHLVMge92wnCC9ESf1MClyrMkh4bEh14eVSGvXLVqPuskeYjt4nYhpw+On1qlFHF5HMIx6T9RqlUV+AwjGcvJGMfrtGUtjyFLCt413846pv4dieHaD3Yu32t1FT36DmMpkMqOBb1YIraRa6VCwMG79LSnj4133CTqr/6NAgBGpitlNha5iB/lgrHtsE9PQdUe1yQTIeSCPGsFd3B4orSO5AglhrqlqisMJinymYRwA8rHrotaWaV9+EiiU/Y3AnEex7Dp7eXZCLNzOtGScWRDN4ggDd9KVngI4tnCTLE2WIU1a3tGLWNZfAoPOjYv4CnHr64uw2KCMVy3WRYEUn2rOWlCjlxdxjiEdZiQixj3a6Gh3oRzlZRVYTxp0jKCu1giGP4f4+6cmxOuw8mMLmjQwLlyYFFeQ6DBO9z2gtZ/5lGuNaggPh0W41Si8LaOjWdPE6ImkdzKxzIyO9iCWS2NcLvPAg4A5lJqVtYCDyMXUf/48puZ6h0VF/+nKsXrcekT2A640uHOraS26GpIgRUk9OVV5lFxxCiGYM+DKAkqSyZT/JRAJDXZ2wzwC6Nj+MXzdMw+G3noN9NgXu2qXaDQY9TPoUaEx4gh8L/FHnfjhCnHfoTfTePw2B/RtgtFcRyHYwvv3QS5xqtWtxdT30lSvRd4BaGw0hntLhg210HU6qXb9I6kgMkEEyWTN1q7xLFoiFgphJy1ZfuVJZ78C726XpOMm2JL+mGCf22jqUssYQBR7a26n6GMU0pMCkF67OdmJKDWLOYzA20gBDR2GqaYTrvWeoIK86eIiCZIQhGxLRcAi7dnyIN7cCr7yyH4QNTHqHacAapCd3IsKCRD/42qMY+7ADvtFB7Hj5eXS078BLr1GAyjrYaIVTzlgCKfG7dmxFJBhAV1YIK0OjsvoU2R+tolf/ZxPx+5Tl5p5/ubJnz6tPwj+RQVVRrNhYHTbicY6twcyLboGFgb3tT3eic9tbGDjQhd49uzHpGYfvPaIqq4obn9qJOzvduH7TVsRdgwjQMydYvoqvyJyiRA3oJicmEOp9DUbixyW/fRNr/uxGw9pd2NWdwAR5MQ+rr/Ej+/Didatx36omPHnZVXBsH0IjXantjKUqPS3+5s3KbXpefBS/+moN3nnobgS5yPwb1qOGriakeYiG6nPP+ooSzt2xG0f7iG6kFFFGvEBnJBBmC5vzrrsJXnqznNQ2XfM1/KTtNHS9s42bc4FeifJZ52LmvDaU2eyY2bYQRU3zIKXF2MgwfzOKljm1VDbucsK/n3m9zYbZCxbCZrejed4CONKL8PRfBPCOQj9WvgQ7Gac+7otpFPVrVuEHHd2wnVKrJr3g2htw/j13wfvuILwfRjDx7hAWrluLq25dr9rlJ84jJCs8SNEuq89onYcK4pXwOt9oly6q0pFIjU44lUcIb/Zpi7DurTeQjmRypHhSma0KQ33d8PC9vKEJJgEUkrWoCOWNpyrlD3bzpECSI6Ccv+S4KDR8sA9i+7KmxbAWC8Qx7Jg/G2Y1q3773mGNQXdIe6lRn9tNdzOirpGLEJ01N9FQc+hQP/xeL4qIro2zT4WBfaWPtIuLRUMhVlZm2OvqFc8z6kA8FmM/Az1hOibHPQj6/URQE6rYR0+g08b7vRNwO0a4OTNOmd6I4OQkIsQDa3EJbDWZsJHNjztHEY1EYCmykl8LWSPBNSwUTvp5ifhhhp2Z2GKvrZchioQv8xl4yFECaw3aU2rpzMV4Jj40obV2eWqbzeV9nvfctT7PeG3MZ92PElg6U4LMWMlb/JtK6lBBpmqZ0uf4+Bz+VN7U79z5j7cJU1tb9pMznzQd75fl/79vGSOU2+8/QDU5FFmyNvEAAAAASUVORK5CYII=) [Events](/app/dashboards#/view/78289c40-86da-11e8-b59d-21efb914e65c-ecs) | [Alerts](/app/dashboards#/view/05268ee0-86d1-11e8-b59d-21efb914e65c-ecs)", + "openLinksInNewTab": false + }, + "title": "Navigation [Filebeat Suricata]", + "type": "markdown" + } + }, + "id": "908e8c90-d296-11ea-90e3-8767fe7ccf14", + "migrationVersion": { + "visualization": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "visualization", + "updated_at": "2020-07-30T18:57:50.040Z", + "version": "Wzc1MywxXQ==" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "savedSearchRefName": "search_0", + "title": "Event Count [Filebeat Suricata]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Events" + }, + "schema": "metric", + "type": "count" + } + ], + "params": { + "addLegend": false, + "addTooltip": true, + "metric": { + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 10000 + } + ], + "invertColors": false, + "labels": { + "show": true + }, + "metricColorMode": "None", + "percentageMode": false, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": 30, + "labelColor": false, + "subText": "" + }, + "useRanges": false + }, + "type": "metric" + }, + "title": "Event Count [Filebeat Suricata]", + "type": "metric" + } + }, + "id": "169c0600-d297-11ea-90e3-8767fe7ccf14", + "migrationVersion": { + "visualization": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "13dd22f0-86cc-11e8-b59d-21efb914e65c-ecs", + "name": "search_0", + "type": "search" + } + ], + "type": "visualization", + "updated_at": "2020-07-30T19:02:39.133Z", + "version": "WzgyNCwxXQ==" } ], "version": "7.9.0-SNAPSHOT" diff --git a/x-pack/filebeat/module/suricata/module.yml b/x-pack/filebeat/module/suricata/module.yml index 9975054c9c1..d3747be1f4d 100644 --- a/x-pack/filebeat/module/suricata/module.yml +++ b/x-pack/filebeat/module/suricata/module.yml @@ -1,5 +1,5 @@ dashboards: - id: 78289c40-86da-11e8-b59d-21efb914e65c-ecs - file: Filebeat-Suricata-Overview.json + file: filebeat-suricata-event-overview.json - id: 05268ee0-86d1-11e8-b59d-21efb914e65c-ecs - file: Filebeat-Suricata-Alert-Overview.json + file: filebeat-suricata-alert-overview.json From 778a92fa78cbdeab0377525dd83b18a048e94ff8 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 3 Aug 2020 19:11:25 +0200 Subject: [PATCH 04/19] [Filebeat][SophosXG Module] Renaming module and fileset (#20396) renaming sophosxg module to sophos, and renaming fileset to better support future filesets --- filebeat/docs/fields.asciidoc | 386 ++++---- .../{sophosxg.asciidoc => sophos.asciidoc} | 31 +- filebeat/docs/modules_list.asciidoc | 4 +- x-pack/filebeat/filebeat.reference.yml | 10 +- x-pack/filebeat/include/list.go | 2 +- .../{sophosxg => sophos}/_meta/config.yml | 8 +- .../{sophosxg => sophos}/_meta/docs.asciidoc | 27 +- .../{sophosxg => sophos}/_meta/fields.yml | 8 +- x-pack/filebeat/module/sophos/fields.go | 23 + .../module/{sophosxg => sophos}/module.yml | 0 .../firewall => sophos/xg}/_meta/fields.yml | 79 +- .../xg/config/config.yml} | 0 .../xg}/ingest/antispam.yml | 52 +- .../xg}/ingest/antivirus.yml | 110 +-- .../firewall => sophos/xg}/ingest/atp.yml | 50 +- .../firewall => sophos/xg}/ingest/cfilter.yml | 74 +- .../firewall => sophos/xg}/ingest/event.yml | 76 +- .../xg}/ingest/firewall.yml | 128 +-- .../firewall => sophos/xg}/ingest/idp.yml | 54 +- .../xg}/ingest/pipeline.yml | 74 +- .../xg}/ingest/sandstorm.yml | 54 +- .../firewall => sophos/xg}/ingest/waf.yml | 80 +- .../firewall => sophos/xg}/ingest/wifi.yml | 2 +- .../firewall => sophos/xg}/manifest.yml | 5 +- .../firewall => sophos/xg}/test/anti-spam.log | 0 .../xg}/test/anti-spam.log-expected.json | 476 ++++----- .../xg}/test/anti-virus.log | 0 .../xg}/test/anti-virus.log-expected.json | 284 +++--- .../firewall => sophos/xg}/test/atp.log | 0 .../xg}/test/atp.log-expected.json | 112 +-- .../firewall => sophos/xg}/test/cfilter.log | 0 .../xg}/test/cfilter.log-expected.json | 334 +++---- .../firewall => sophos/xg}/test/event.log | 0 .../xg}/test/event.log-expected.json | 446 ++++----- .../firewall => sophos/xg}/test/firewall.log | 0 .../xg}/test/firewall.log-expected.json | 908 ++++++++++-------- .../firewall => sophos/xg}/test/idp.log | 0 .../xg}/test/idp.log-expected.json | 200 ++-- .../firewall => sophos/xg}/test/sandbox.log | 0 .../xg}/test/sandbox.log-expected.json | 172 ++-- .../firewall => sophos/xg}/test/waf.log | 0 .../xg}/test/waf.log-expected.json | 178 ++-- .../firewall => sophos/xg}/test/wifi.log | 0 .../xg}/test/wifi.log-expected.json | 60 +- x-pack/filebeat/module/sophosxg/fields.go | 23 - .../sophosxg/firewall/ingest/systemhealth.yml | 158 --- ...hosxg.yml.disabled => sophos.yml.disabled} | 12 +- 47 files changed, 2307 insertions(+), 2393 deletions(-) rename filebeat/docs/modules/{sophosxg.asciidoc => sophos.asciidoc} (85%) rename x-pack/filebeat/module/{sophosxg => sophos}/_meta/config.yml (88%) rename x-pack/filebeat/module/{sophosxg => sophos}/_meta/docs.asciidoc (86%) rename x-pack/filebeat/module/{sophosxg => sophos}/_meta/fields.yml (50%) create mode 100644 x-pack/filebeat/module/sophos/fields.go rename x-pack/filebeat/module/{sophosxg => sophos}/module.yml (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/_meta/fields.yml (98%) rename x-pack/filebeat/module/{sophosxg/firewall/config/firewall.yml => sophos/xg/config/config.yml} (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/antispam.yml (82%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/antivirus.yml (71%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/atp.yml (81%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/cfilter.yml (75%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/event.yml (72%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/firewall.yml (77%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/idp.yml (80%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/pipeline.yml (73%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/sandstorm.yml (62%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/waf.yml (74%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/ingest/wifi.yml (84%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/manifest.yml (88%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/anti-spam.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/anti-spam.log-expected.json (68%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/anti-virus.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/anti-virus.log-expected.json (74%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/atp.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/atp.log-expected.json (78%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/cfilter.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/cfilter.log-expected.json (73%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/event.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/event.log-expected.json (68%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/firewall.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/firewall.log-expected.json (74%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/idp.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/idp.log-expected.json (69%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/sandbox.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/sandbox.log-expected.json (70%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/waf.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/waf.log-expected.json (72%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/wifi.log (100%) rename x-pack/filebeat/module/{sophosxg/firewall => sophos/xg}/test/wifi.log-expected.json (56%) delete mode 100644 x-pack/filebeat/module/sophosxg/fields.go delete mode 100644 x-pack/filebeat/module/sophosxg/firewall/ingest/systemhealth.yml rename x-pack/filebeat/modules.d/{sophosxg.yml.disabled => sophos.yml.disabled} (84%) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 5cad750548f..25dd11e74f3 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -76,7 +76,7 @@ grouped in the following categories: * <> * <> * <> -* <> +* <> * <> * <> * <> @@ -122135,27 +122135,27 @@ type: keyword -- -[[exported-fields-sophosxg]] -== sophosxg fields +[[exported-fields-sophos]] +== sophos fields -sophosxg Module +sophos Module [float] -=== sophosxg +=== sophos [float] -=== firewall +=== xg Module for parsing sophosxg syslog. -*`sophosxg.firewall.device`*:: +*`sophos.xg.device`*:: + -- device @@ -122165,7 +122165,7 @@ type: keyword -- -*`sophosxg.firewall.date`*:: +*`sophos.xg.date`*:: + -- Date (yyyy-mm-dd) when the event occurred @@ -122175,7 +122175,7 @@ type: date -- -*`sophosxg.firewall.timezone`*:: +*`sophos.xg.timezone`*:: + -- Time (hh:mm:ss) when the event occurred @@ -122185,7 +122185,7 @@ type: keyword -- -*`sophosxg.firewall.device_name`*:: +*`sophos.xg.device_name`*:: + -- Model number of the device @@ -122195,7 +122195,7 @@ type: keyword -- -*`sophosxg.firewall.device_id`*:: +*`sophos.xg.device_id`*:: + -- Serial number of the device @@ -122205,7 +122205,7 @@ type: keyword -- -*`sophosxg.firewall.log_id`*:: +*`sophos.xg.log_id`*:: + -- Unique 12 characters code (0101011) @@ -122215,7 +122215,7 @@ type: keyword -- -*`sophosxg.firewall.log_type`*:: +*`sophos.xg.log_type`*:: + -- Type of event e.g. firewall event @@ -122225,7 +122225,7 @@ type: keyword -- -*`sophosxg.firewall.log_component`*:: +*`sophos.xg.log_component`*:: + -- Component responsible for logging e.g. Firewall rule @@ -122235,7 +122235,7 @@ type: keyword -- -*`sophosxg.firewall.log_subtype`*:: +*`sophos.xg.log_subtype`*:: + -- Sub type of event @@ -122245,7 +122245,7 @@ type: keyword -- -*`sophosxg.firewall.hb_health`*:: +*`sophos.xg.hb_health`*:: + -- Heartbeat status @@ -122255,7 +122255,7 @@ type: keyword -- -*`sophosxg.firewall.priority`*:: +*`sophos.xg.priority`*:: + -- Severity level of traffic @@ -122265,7 +122265,7 @@ type: keyword -- -*`sophosxg.firewall.status`*:: +*`sophos.xg.status`*:: + -- Ultimate status of traffic – Allowed or Denied @@ -122275,7 +122275,7 @@ type: keyword -- -*`sophosxg.firewall.duration`*:: +*`sophos.xg.duration`*:: + -- Durability of traffic (seconds) @@ -122285,7 +122285,7 @@ type: long -- -*`sophosxg.firewall.fw_rule_id`*:: +*`sophos.xg.fw_rule_id`*:: + -- Firewall Rule ID which is applied on the traffic @@ -122295,7 +122295,7 @@ type: integer -- -*`sophosxg.firewall.user_name`*:: +*`sophos.xg.user_name`*:: + -- user_name @@ -122305,7 +122305,7 @@ type: keyword -- -*`sophosxg.firewall.user_group`*:: +*`sophos.xg.user_group`*:: + -- Group name to which the user belongs @@ -122315,7 +122315,7 @@ type: keyword -- -*`sophosxg.firewall.iap`*:: +*`sophos.xg.iap`*:: + -- Internet Access policy ID applied on the traffic @@ -122325,7 +122325,7 @@ type: keyword -- -*`sophosxg.firewall.ips_policy_id`*:: +*`sophos.xg.ips_policy_id`*:: + -- IPS policy ID applied on the traffic @@ -122335,7 +122335,7 @@ type: integer -- -*`sophosxg.firewall.policy_type`*:: +*`sophos.xg.policy_type`*:: + -- Policy type applied to the traffic @@ -122345,7 +122345,7 @@ type: keyword -- -*`sophosxg.firewall.appfilter_policy_id`*:: +*`sophos.xg.appfilter_policy_id`*:: + -- Application Filter policy applied on the traffic @@ -122355,7 +122355,7 @@ type: integer -- -*`sophosxg.firewall.application_filter_policy`*:: +*`sophos.xg.application_filter_policy`*:: + -- Application Filter policy applied on the traffic @@ -122365,7 +122365,7 @@ type: integer -- -*`sophosxg.firewall.application`*:: +*`sophos.xg.application`*:: + -- Application name @@ -122375,7 +122375,7 @@ type: keyword -- -*`sophosxg.firewall.application_name`*:: +*`sophos.xg.application_name`*:: + -- Application name @@ -122385,7 +122385,7 @@ type: keyword -- -*`sophosxg.firewall.application_risk`*:: +*`sophos.xg.application_risk`*:: + -- Risk level assigned to the application @@ -122395,7 +122395,7 @@ type: keyword -- -*`sophosxg.firewall.application_technology`*:: +*`sophos.xg.application_technology`*:: + -- Technology of the application @@ -122405,7 +122405,7 @@ type: keyword -- -*`sophosxg.firewall.application_category`*:: +*`sophos.xg.application_category`*:: + -- Application is resolved by signature or synchronized application @@ -122415,7 +122415,7 @@ type: keyword -- -*`sophosxg.firewall.appresolvedby`*:: +*`sophos.xg.appresolvedby`*:: + -- Technology of the application @@ -122425,7 +122425,7 @@ type: keyword -- -*`sophosxg.firewall.app_is_cloud`*:: +*`sophos.xg.app_is_cloud`*:: + -- Application is Cloud @@ -122435,7 +122435,7 @@ type: keyword -- -*`sophosxg.firewall.in_interface`*:: +*`sophos.xg.in_interface`*:: + -- Interface for incoming traffic, e.g., Port A @@ -122445,7 +122445,7 @@ type: keyword -- -*`sophosxg.firewall.out_interface`*:: +*`sophos.xg.out_interface`*:: + -- Interface for outgoing traffic, e.g., Port B @@ -122455,7 +122455,7 @@ type: keyword -- -*`sophosxg.firewall.src_ip`*:: +*`sophos.xg.src_ip`*:: + -- Original source IP address of traffic @@ -122465,7 +122465,7 @@ type: ip -- -*`sophosxg.firewall.src_mac`*:: +*`sophos.xg.src_mac`*:: + -- Original source MAC address of traffic @@ -122475,7 +122475,7 @@ type: keyword -- -*`sophosxg.firewall.src_country_code`*:: +*`sophos.xg.src_country_code`*:: + -- Code of the country to which the source IP belongs @@ -122485,7 +122485,7 @@ type: keyword -- -*`sophosxg.firewall.dst_ip`*:: +*`sophos.xg.dst_ip`*:: + -- Original destination IP address of traffic @@ -122495,7 +122495,7 @@ type: ip -- -*`sophosxg.firewall.dst_country_code`*:: +*`sophos.xg.dst_country_code`*:: + -- Code of the country to which the destination IP belongs @@ -122505,7 +122505,7 @@ type: keyword -- -*`sophosxg.firewall.protocol`*:: +*`sophos.xg.protocol`*:: + -- Protocol number of traffic @@ -122515,7 +122515,7 @@ type: keyword -- -*`sophosxg.firewall.src_port`*:: +*`sophos.xg.src_port`*:: + -- Original source port of TCP and UDP traffic @@ -122525,7 +122525,7 @@ type: integer -- -*`sophosxg.firewall.dst_port`*:: +*`sophos.xg.dst_port`*:: + -- Original destination port of TCP and UDP traffic @@ -122535,7 +122535,7 @@ type: integer -- -*`sophosxg.firewall.icmp_type`*:: +*`sophos.xg.icmp_type`*:: + -- ICMP type of ICMP traffic @@ -122545,7 +122545,7 @@ type: keyword -- -*`sophosxg.firewall.icmp_code`*:: +*`sophos.xg.icmp_code`*:: + -- ICMP code of ICMP traffic @@ -122555,7 +122555,7 @@ type: keyword -- -*`sophosxg.firewall.sent_pkts`*:: +*`sophos.xg.sent_pkts`*:: + -- Total number of packets sent @@ -122565,7 +122565,7 @@ type: long -- -*`sophosxg.firewall.received_pkts`*:: +*`sophos.xg.received_pkts`*:: + -- Total number of packets received @@ -122575,7 +122575,7 @@ type: long -- -*`sophosxg.firewall.sent_bytes`*:: +*`sophos.xg.sent_bytes`*:: + -- Total number of bytes sent @@ -122585,7 +122585,7 @@ type: long -- -*`sophosxg.firewall.recv_bytes`*:: +*`sophos.xg.recv_bytes`*:: + -- Total number of bytes received @@ -122595,7 +122595,7 @@ type: long -- -*`sophosxg.firewall.trans_src_ ip`*:: +*`sophos.xg.trans_src_ ip`*:: + -- Translated source IP address for outgoing traffic @@ -122605,7 +122605,7 @@ type: ip -- -*`sophosxg.firewall.trans_src_port`*:: +*`sophos.xg.trans_src_port`*:: + -- Translated source port for outgoing traffic @@ -122615,7 +122615,7 @@ type: integer -- -*`sophosxg.firewall.trans_dst_ip`*:: +*`sophos.xg.trans_dst_ip`*:: + -- Translated destination IP address for outgoing traffic @@ -122625,7 +122625,7 @@ type: ip -- -*`sophosxg.firewall.trans_dst_port`*:: +*`sophos.xg.trans_dst_port`*:: + -- Translated destination port for outgoing traffic @@ -122635,7 +122635,7 @@ type: integer -- -*`sophosxg.firewall.srczonetype`*:: +*`sophos.xg.srczonetype`*:: + -- Type of source zone, e.g., LAN @@ -122645,7 +122645,7 @@ type: keyword -- -*`sophosxg.firewall.srczone`*:: +*`sophos.xg.srczone`*:: + -- Name of source zone @@ -122655,7 +122655,7 @@ type: keyword -- -*`sophosxg.firewall.dstzonetype`*:: +*`sophos.xg.dstzonetype`*:: + -- Type of destination zone, e.g., WAN @@ -122665,7 +122665,7 @@ type: keyword -- -*`sophosxg.firewall.dstzone`*:: +*`sophos.xg.dstzone`*:: + -- Name of destination zone @@ -122675,7 +122675,7 @@ type: keyword -- -*`sophosxg.firewall.dir_disp`*:: +*`sophos.xg.dir_disp`*:: + -- TPacket direction. Possible values:“org”, “reply”, “” @@ -122685,7 +122685,7 @@ type: keyword -- -*`sophosxg.firewall.connevent`*:: +*`sophos.xg.connevent`*:: + -- Event on which this log is generated @@ -122695,7 +122695,7 @@ type: keyword -- -*`sophosxg.firewall.conn_id`*:: +*`sophos.xg.conn_id`*:: + -- Unique identifier of connection @@ -122705,7 +122705,7 @@ type: integer -- -*`sophosxg.firewall.vconn_id`*:: +*`sophos.xg.vconn_id`*:: + -- Connection ID of the master connection @@ -122715,7 +122715,7 @@ type: integer -- -*`sophosxg.firewall.idp_policy_id`*:: +*`sophos.xg.idp_policy_id`*:: + -- IPS policy ID which is applied on the traffic @@ -122725,7 +122725,7 @@ type: integer -- -*`sophosxg.firewall.idp_policy_name`*:: +*`sophos.xg.idp_policy_name`*:: + -- IPS policy name i.e. IPS policy name which is applied on the traffic @@ -122735,7 +122735,7 @@ type: keyword -- -*`sophosxg.firewall.signature_id`*:: +*`sophos.xg.signature_id`*:: + -- Signature ID @@ -122745,7 +122745,7 @@ type: keyword -- -*`sophosxg.firewall.signature_msg`*:: +*`sophos.xg.signature_msg`*:: + -- Signature messsage @@ -122755,7 +122755,7 @@ type: keyword -- -*`sophosxg.firewall.classification`*:: +*`sophos.xg.classification`*:: + -- Signature classification @@ -122765,7 +122765,7 @@ type: keyword -- -*`sophosxg.firewall.rule_priority`*:: +*`sophos.xg.rule_priority`*:: + -- Priority of IPS policy @@ -122775,7 +122775,7 @@ type: keyword -- -*`sophosxg.firewall.platform`*:: +*`sophos.xg.platform`*:: + -- Platform of the traffic. @@ -122785,7 +122785,7 @@ type: keyword -- -*`sophosxg.firewall.category`*:: +*`sophos.xg.category`*:: + -- IPS signature category. @@ -122795,7 +122795,7 @@ type: keyword -- -*`sophosxg.firewall.target`*:: +*`sophos.xg.target`*:: + -- Platform of the traffic. @@ -122805,7 +122805,7 @@ type: keyword -- -*`sophosxg.firewall.eventid`*:: +*`sophos.xg.eventid`*:: + -- ATP Evenet ID @@ -122815,7 +122815,7 @@ type: keyword -- -*`sophosxg.firewall.ep_uuid`*:: +*`sophos.xg.ep_uuid`*:: + -- Endpoint UUID @@ -122825,7 +122825,7 @@ type: keyword -- -*`sophosxg.firewall.threatname`*:: +*`sophos.xg.threatname`*:: + -- ATP threatname @@ -122835,7 +122835,7 @@ type: keyword -- -*`sophosxg.firewall.sourceip`*:: +*`sophos.xg.sourceip`*:: + -- Original source IP address of traffic @@ -122845,7 +122845,7 @@ type: ip -- -*`sophosxg.firewall.destinationip`*:: +*`sophos.xg.destinationip`*:: + -- Original destination IP address of traffic @@ -122855,7 +122855,7 @@ type: ip -- -*`sophosxg.firewall.login_user`*:: +*`sophos.xg.login_user`*:: + -- ATP login user @@ -122865,7 +122865,7 @@ type: keyword -- -*`sophosxg.firewall.eventtype`*:: +*`sophos.xg.eventtype`*:: + -- ATP event type @@ -122875,7 +122875,7 @@ type: keyword -- -*`sophosxg.firewall.execution_path`*:: +*`sophos.xg.execution_path`*:: + -- ATP execution path @@ -122885,7 +122885,7 @@ type: keyword -- -*`sophosxg.firewall.av_policy_name`*:: +*`sophos.xg.av_policy_name`*:: + -- Malware scanning policy name which is applied on the traffic @@ -122895,7 +122895,7 @@ type: keyword -- -*`sophosxg.firewall.from_email_address`*:: +*`sophos.xg.from_email_address`*:: + -- Sender email address @@ -122905,7 +122905,7 @@ type: keyword -- -*`sophosxg.firewall.to_email_address`*:: +*`sophos.xg.to_email_address`*:: + -- Receipeint email address @@ -122915,7 +122915,7 @@ type: keyword -- -*`sophosxg.firewall.subject`*:: +*`sophos.xg.subject`*:: + -- Email subject @@ -122925,7 +122925,7 @@ type: keyword -- -*`sophosxg.firewall.mailsize`*:: +*`sophos.xg.mailsize`*:: + -- mailsize @@ -122935,7 +122935,7 @@ type: integer -- -*`sophosxg.firewall.virus`*:: +*`sophos.xg.virus`*:: + -- virus name @@ -122945,7 +122945,7 @@ type: keyword -- -*`sophosxg.firewall.FTP_url`*:: +*`sophos.xg.FTP_url`*:: + -- FTP URL from which virus was downloaded @@ -122955,7 +122955,7 @@ type: keyword -- -*`sophosxg.firewall.FTP_direction`*:: +*`sophos.xg.FTP_direction`*:: + -- Direction of FTP transfer: Upload or Download @@ -122965,7 +122965,7 @@ type: keyword -- -*`sophosxg.firewall.filesize`*:: +*`sophos.xg.filesize`*:: + -- Size of the file that contained virus @@ -122975,7 +122975,7 @@ type: integer -- -*`sophosxg.firewall.filepath`*:: +*`sophos.xg.filepath`*:: + -- Path of the file containing virus @@ -122985,7 +122985,7 @@ type: keyword -- -*`sophosxg.firewall.filename`*:: +*`sophos.xg.filename`*:: + -- File name associated with the event @@ -122995,7 +122995,7 @@ type: keyword -- -*`sophosxg.firewall.ftpcommand`*:: +*`sophos.xg.ftpcommand`*:: + -- FTP command used when virus was found @@ -123005,7 +123005,7 @@ type: keyword -- -*`sophosxg.firewall.url`*:: +*`sophos.xg.url`*:: + -- URL from which virus was downloaded @@ -123015,7 +123015,7 @@ type: keyword -- -*`sophosxg.firewall.domainname`*:: +*`sophos.xg.domainname`*:: + -- Domain from which virus was downloaded @@ -123025,7 +123025,7 @@ type: keyword -- -*`sophosxg.firewall.quarantine`*:: +*`sophos.xg.quarantine`*:: + -- Path and filename of the file quarantined @@ -123035,7 +123035,7 @@ type: keyword -- -*`sophosxg.firewall.src_domainname`*:: +*`sophos.xg.src_domainname`*:: + -- Sender domain name @@ -123045,7 +123045,7 @@ type: keyword -- -*`sophosxg.firewall.dst_domainname`*:: +*`sophos.xg.dst_domainname`*:: + -- Receiver domain name @@ -123055,7 +123055,7 @@ type: keyword -- -*`sophosxg.firewall.reason`*:: +*`sophos.xg.reason`*:: + -- Reason why the record was detected as spam/malicious @@ -123065,7 +123065,7 @@ type: keyword -- -*`sophosxg.firewall.referer`*:: +*`sophos.xg.referer`*:: + -- Referer @@ -123075,7 +123075,7 @@ type: keyword -- -*`sophosxg.firewall.spamaction`*:: +*`sophos.xg.spamaction`*:: + -- Spam Action @@ -123085,7 +123085,7 @@ type: keyword -- -*`sophosxg.firewall.mailid`*:: +*`sophos.xg.mailid`*:: + -- mailid @@ -123095,7 +123095,7 @@ type: keyword -- -*`sophosxg.firewall.quarantine_reason`*:: +*`sophos.xg.quarantine_reason`*:: + -- Quarantine reason @@ -123105,7 +123105,7 @@ type: keyword -- -*`sophosxg.firewall.status_code`*:: +*`sophos.xg.status_code`*:: + -- Status code @@ -123115,7 +123115,7 @@ type: keyword -- -*`sophosxg.firewall.override_token`*:: +*`sophos.xg.override_token`*:: + -- Override token @@ -123125,7 +123125,7 @@ type: keyword -- -*`sophosxg.firewall.con_id`*:: +*`sophos.xg.con_id`*:: + -- Unique identifier of connection @@ -123135,7 +123135,7 @@ type: integer -- -*`sophosxg.firewall.override_authorizer`*:: +*`sophos.xg.override_authorizer`*:: + -- Override authorizer @@ -123145,7 +123145,7 @@ type: keyword -- -*`sophosxg.firewall.transactionid`*:: +*`sophos.xg.transactionid`*:: + -- Transaction ID of the AV scan. @@ -123155,7 +123155,7 @@ type: keyword -- -*`sophosxg.firewall.upload_file_type`*:: +*`sophos.xg.upload_file_type`*:: + -- Upload file type @@ -123165,7 +123165,7 @@ type: keyword -- -*`sophosxg.firewall.upload_file_name`*:: +*`sophos.xg.upload_file_name`*:: + -- Upload file name @@ -123175,7 +123175,7 @@ type: keyword -- -*`sophosxg.firewall.httpresponsecode`*:: +*`sophos.xg.httpresponsecode`*:: + -- code of HTTP response @@ -123185,7 +123185,7 @@ type: long -- -*`sophosxg.firewall.user_gp`*:: +*`sophos.xg.user_gp`*:: + -- Group name to which the user belongs. @@ -123195,7 +123195,7 @@ type: keyword -- -*`sophosxg.firewall.category_type`*:: +*`sophos.xg.category_type`*:: + -- Type of category under which website falls @@ -123205,7 +123205,7 @@ type: keyword -- -*`sophosxg.firewall.download_file_type`*:: +*`sophos.xg.download_file_type`*:: + -- Download file type @@ -123215,7 +123215,7 @@ type: keyword -- -*`sophosxg.firewall.exceptions`*:: +*`sophos.xg.exceptions`*:: + -- List of the checks excluded by web exceptions. @@ -123225,7 +123225,7 @@ type: keyword -- -*`sophosxg.firewall.contenttype`*:: +*`sophos.xg.contenttype`*:: + -- Type of the content @@ -123235,7 +123235,7 @@ type: keyword -- -*`sophosxg.firewall.override_name`*:: +*`sophos.xg.override_name`*:: + -- Override name @@ -123245,7 +123245,7 @@ type: keyword -- -*`sophosxg.firewall.activityname`*:: +*`sophos.xg.activityname`*:: + -- Web policy activity that matched and caused the policy result. @@ -123255,7 +123255,7 @@ type: keyword -- -*`sophosxg.firewall.download_file_name`*:: +*`sophos.xg.download_file_name`*:: + -- Download file name @@ -123265,7 +123265,7 @@ type: keyword -- -*`sophosxg.firewall.sha1sum`*:: +*`sophos.xg.sha1sum`*:: + -- SHA1 checksum of the item being analyzed @@ -123275,7 +123275,7 @@ type: keyword -- -*`sophosxg.firewall.message_id`*:: +*`sophos.xg.message_id`*:: + -- Message ID @@ -123285,7 +123285,7 @@ type: keyword -- -*`sophosxg.firewall.connid`*:: +*`sophos.xg.connid`*:: + -- Connection ID @@ -123295,7 +123295,7 @@ type: keyword -- -*`sophosxg.firewall.message`*:: +*`sophos.xg.message`*:: + -- Message @@ -123305,7 +123305,7 @@ type: keyword -- -*`sophosxg.firewall.email_subject`*:: +*`sophos.xg.email_subject`*:: + -- Email Subject @@ -123315,7 +123315,7 @@ type: keyword -- -*`sophosxg.firewall.file_path`*:: +*`sophos.xg.file_path`*:: + -- File path @@ -123325,7 +123325,7 @@ type: keyword -- -*`sophosxg.firewall.dstdomain`*:: +*`sophos.xg.dstdomain`*:: + -- Destination Domain @@ -123335,7 +123335,7 @@ type: keyword -- -*`sophosxg.firewall.file_size`*:: +*`sophos.xg.file_size`*:: + -- File Size @@ -123345,7 +123345,7 @@ type: integer -- -*`sophosxg.firewall.transaction_id`*:: +*`sophos.xg.transaction_id`*:: + -- Transaction ID @@ -123355,7 +123355,7 @@ type: keyword -- -*`sophosxg.firewall.website`*:: +*`sophos.xg.website`*:: + -- Website @@ -123365,7 +123365,7 @@ type: keyword -- -*`sophosxg.firewall.file_name`*:: +*`sophos.xg.file_name`*:: + -- Filename @@ -123375,7 +123375,7 @@ type: keyword -- -*`sophosxg.firewall.context_prefix`*:: +*`sophos.xg.context_prefix`*:: + -- Content Prefix @@ -123385,7 +123385,7 @@ type: keyword -- -*`sophosxg.firewall.site_category`*:: +*`sophos.xg.site_category`*:: + -- Site Category @@ -123395,7 +123395,7 @@ type: keyword -- -*`sophosxg.firewall.context_suffix`*:: +*`sophos.xg.context_suffix`*:: + -- Context Suffix @@ -123405,7 +123405,7 @@ type: keyword -- -*`sophosxg.firewall.dictionary_name`*:: +*`sophos.xg.dictionary_name`*:: + -- Dictionary Name @@ -123415,7 +123415,7 @@ type: keyword -- -*`sophosxg.firewall.action`*:: +*`sophos.xg.action`*:: + -- Event Action @@ -123425,7 +123425,7 @@ type: keyword -- -*`sophosxg.firewall.user`*:: +*`sophos.xg.user`*:: + -- User @@ -123435,17 +123435,17 @@ type: keyword -- -*`sophosxg.firewall.context_match`*:: +*`sophos.xg.context_match`*:: + -- -Context Match +Context Match type: keyword -- -*`sophosxg.firewall.direction`*:: +*`sophos.xg.direction`*:: + -- Direction @@ -123455,7 +123455,7 @@ type: keyword -- -*`sophosxg.firewall.auth_client`*:: +*`sophos.xg.auth_client`*:: + -- Auth Client @@ -123465,7 +123465,7 @@ type: keyword -- -*`sophosxg.firewall.auth_mechanism`*:: +*`sophos.xg.auth_mechanism`*:: + -- Auth mechanism @@ -123475,7 +123475,7 @@ type: keyword -- -*`sophosxg.firewall.connectionname`*:: +*`sophos.xg.connectionname`*:: + -- Connectionname @@ -123485,7 +123485,7 @@ type: keyword -- -*`sophosxg.firewall.remotenetwork`*:: +*`sophos.xg.remotenetwork`*:: + -- remotenetwork @@ -123495,7 +123495,7 @@ type: keyword -- -*`sophosxg.firewall.localgateway`*:: +*`sophos.xg.localgateway`*:: + -- Localgateway @@ -123505,7 +123505,7 @@ type: keyword -- -*`sophosxg.firewall.localnetwork`*:: +*`sophos.xg.localnetwork`*:: + -- Localnetwork @@ -123515,7 +123515,7 @@ type: keyword -- -*`sophosxg.firewall.connectiontype`*:: +*`sophos.xg.connectiontype`*:: + -- Connectiontype @@ -123525,7 +123525,7 @@ type: keyword -- -*`sophosxg.firewall.oldversion`*:: +*`sophos.xg.oldversion`*:: + -- Oldversion @@ -123535,7 +123535,7 @@ type: keyword -- -*`sophosxg.firewall.newversion`*:: +*`sophos.xg.newversion`*:: + -- Newversion @@ -123545,7 +123545,7 @@ type: keyword -- -*`sophosxg.firewall.ipaddress`*:: +*`sophos.xg.ipaddress`*:: + -- Ipaddress @@ -123555,7 +123555,7 @@ type: keyword -- -*`sophosxg.firewall.client_physical_address`*:: +*`sophos.xg.client_physical_address`*:: + -- Client physical address @@ -123565,7 +123565,7 @@ type: keyword -- -*`sophosxg.firewall.client_host_name`*:: +*`sophos.xg.client_host_name`*:: + -- Client host name @@ -123575,7 +123575,7 @@ type: keyword -- -*`sophosxg.firewall.raw_data`*:: +*`sophos.xg.raw_data`*:: + -- Raw data @@ -123585,7 +123585,7 @@ type: keyword -- -*`sophosxg.firewall.Mode`*:: +*`sophos.xg.Mode`*:: + -- Mode @@ -123595,7 +123595,7 @@ type: keyword -- -*`sophosxg.firewall.sessionid`*:: +*`sophos.xg.sessionid`*:: + -- Sessionid @@ -123605,7 +123605,7 @@ type: keyword -- -*`sophosxg.firewall.starttime`*:: +*`sophos.xg.starttime`*:: + -- Starttime @@ -123615,7 +123615,7 @@ type: date -- -*`sophosxg.firewall.remote_ip`*:: +*`sophos.xg.remote_ip`*:: + -- Remote IP @@ -123625,7 +123625,7 @@ type: ip -- -*`sophosxg.firewall.timestamp`*:: +*`sophos.xg.timestamp`*:: + -- timestamp @@ -123635,7 +123635,7 @@ type: date -- -*`sophosxg.firewall.SysLog_SERVER_NAME`*:: +*`sophos.xg.SysLog_SERVER_NAME`*:: + -- SysLog SERVER NAME @@ -123645,7 +123645,7 @@ type: keyword -- -*`sophosxg.firewall.backup_mode`*:: +*`sophos.xg.backup_mode`*:: + -- Backup mode @@ -123655,7 +123655,7 @@ type: keyword -- -*`sophosxg.firewall.source`*:: +*`sophos.xg.source`*:: + -- Source @@ -123665,7 +123665,7 @@ type: keyword -- -*`sophosxg.firewall.server`*:: +*`sophos.xg.server`*:: + -- Server @@ -123675,7 +123675,7 @@ type: keyword -- -*`sophosxg.firewall.host`*:: +*`sophos.xg.host`*:: + -- Host @@ -123685,7 +123685,7 @@ type: keyword -- -*`sophosxg.firewall.responsetime`*:: +*`sophos.xg.responsetime`*:: + -- Responsetime @@ -123695,7 +123695,7 @@ type: long -- -*`sophosxg.firewall.cookie`*:: +*`sophos.xg.cookie`*:: + -- cookie @@ -123705,7 +123705,7 @@ type: keyword -- -*`sophosxg.firewall.querystring`*:: +*`sophos.xg.querystring`*:: + -- querystring @@ -123715,7 +123715,7 @@ type: keyword -- -*`sophosxg.firewall.extra`*:: +*`sophos.xg.extra`*:: + -- extra @@ -123725,7 +123725,7 @@ type: keyword -- -*`sophosxg.firewall.PHPSESSID`*:: +*`sophos.xg.PHPSESSID`*:: + -- PHPSESSID @@ -123735,7 +123735,7 @@ type: keyword -- -*`sophosxg.firewall.start_time`*:: +*`sophos.xg.start_time`*:: + -- Start time @@ -123745,7 +123745,7 @@ type: date -- -*`sophosxg.firewall.eventtime`*:: +*`sophos.xg.eventtime`*:: + -- Event time @@ -123755,7 +123755,7 @@ type: date -- -*`sophosxg.firewall.red_id`*:: +*`sophos.xg.red_id`*:: + -- RED ID @@ -123765,7 +123765,7 @@ type: keyword -- -*`sophosxg.firewall.branch_name`*:: +*`sophos.xg.branch_name`*:: + -- Branch Name @@ -123775,7 +123775,7 @@ type: keyword -- -*`sophosxg.firewall.updatedip`*:: +*`sophos.xg.updatedip`*:: + -- updatedip @@ -123785,7 +123785,7 @@ type: ip -- -*`sophosxg.firewall.idle_cpu`*:: +*`sophos.xg.idle_cpu`*:: + -- idle ## @@ -123795,7 +123795,7 @@ type: float -- -*`sophosxg.firewall.system_cpu`*:: +*`sophos.xg.system_cpu`*:: + -- system @@ -123805,7 +123805,7 @@ type: float -- -*`sophosxg.firewall.user_cpu`*:: +*`sophos.xg.user_cpu`*:: + -- system @@ -123815,7 +123815,7 @@ type: float -- -*`sophosxg.firewall.used`*:: +*`sophos.xg.used`*:: + -- used @@ -123825,7 +123825,7 @@ type: integer -- -*`sophosxg.firewall.unit`*:: +*`sophos.xg.unit`*:: + -- unit @@ -123835,7 +123835,7 @@ type: keyword -- -*`sophosxg.firewall.total_memory`*:: +*`sophos.xg.total_memory`*:: + -- Total Memory @@ -123845,7 +123845,7 @@ type: integer -- -*`sophosxg.firewall.free`*:: +*`sophos.xg.free`*:: + -- free @@ -123855,7 +123855,7 @@ type: integer -- -*`sophosxg.firewall.transmittederrors`*:: +*`sophos.xg.transmittederrors`*:: + -- transmitted errors @@ -123865,7 +123865,7 @@ type: keyword -- -*`sophosxg.firewall.receivederrors`*:: +*`sophos.xg.receivederrors`*:: + -- received errors @@ -123875,7 +123875,7 @@ type: keyword -- -*`sophosxg.firewall.receivedkbits`*:: +*`sophos.xg.receivedkbits`*:: + -- received kbits @@ -123885,7 +123885,7 @@ type: long -- -*`sophosxg.firewall.transmittedkbits`*:: +*`sophos.xg.transmittedkbits`*:: + -- transmitted kbits @@ -123895,7 +123895,7 @@ type: long -- -*`sophosxg.firewall.transmitteddrops`*:: +*`sophos.xg.transmitteddrops`*:: + -- transmitted drops @@ -123905,7 +123905,7 @@ type: long -- -*`sophosxg.firewall.receiveddrops`*:: +*`sophos.xg.receiveddrops`*:: + -- received drops @@ -123915,7 +123915,7 @@ type: long -- -*`sophosxg.firewall.collisions`*:: +*`sophos.xg.collisions`*:: + -- collisions @@ -123925,7 +123925,7 @@ type: long -- -*`sophosxg.firewall.interface`*:: +*`sophos.xg.interface`*:: + -- interface @@ -123935,7 +123935,7 @@ type: keyword -- -*`sophosxg.firewall.Configuration`*:: +*`sophos.xg.Configuration`*:: + -- Configuration @@ -123945,7 +123945,7 @@ type: float -- -*`sophosxg.firewall.Reports`*:: +*`sophos.xg.Reports`*:: + -- Reports @@ -123955,7 +123955,7 @@ type: float -- -*`sophosxg.firewall.Signature`*:: +*`sophos.xg.Signature`*:: + -- Signature @@ -123965,7 +123965,7 @@ type: float -- -*`sophosxg.firewall.Temp`*:: +*`sophos.xg.Temp`*:: + -- Temp @@ -123975,7 +123975,7 @@ type: float -- -*`sophosxg.firewall.users`*:: +*`sophos.xg.users`*:: + -- users @@ -123985,7 +123985,7 @@ type: keyword -- -*`sophosxg.firewall.ssid`*:: +*`sophos.xg.ssid`*:: + -- ssid @@ -123995,7 +123995,7 @@ type: keyword -- -*`sophosxg.firewall.ap`*:: +*`sophos.xg.ap`*:: + -- ap @@ -124005,7 +124005,7 @@ type: keyword -- -*`sophosxg.firewall.clients_conn_ssid`*:: +*`sophos.xg.clients_conn_ssid`*:: + -- clients connection ssid diff --git a/filebeat/docs/modules/sophosxg.asciidoc b/filebeat/docs/modules/sophos.asciidoc similarity index 85% rename from filebeat/docs/modules/sophosxg.asciidoc rename to filebeat/docs/modules/sophos.asciidoc index c276cba4f82..74aacf0df0f 100644 --- a/filebeat/docs/modules/sophosxg.asciidoc +++ b/filebeat/docs/modules/sophos.asciidoc @@ -2,15 +2,15 @@ This file is generated! See scripts/docs_collector.py //// -[[filebeat-module-sophosxg]] +[[filebeat-module-sophos]] [role="xpack"] -:modulename: sophosxg +:modulename: sophos :has-dashboards: false -== SophosXG module +== Sophos module -This is a module for SophosXG SFOS logs sent in the syslog format. +This is a module for Sophos Products, currently it supports XG SFOS logs sent in the syslog format. To configure a remote syslog destination, please reference the https://community.sophos.com/kb/en-us/123184[SophosXG/SFOS Documentation]. @@ -21,27 +21,34 @@ include::../include/gs-link.asciidoc[] [float] === Compatibility -This module has been tested against SFOS version 17.5.x and 18.0.x. +This module has been tested against SFOS version 17.5.x and 18.0.x. Versions above this are expected to work but have not been tested. include::../include/configuring-intro.asciidoc[] -:fileset_ex: firewall +:fileset_ex: xg include::../include/config-option-intro.asciidoc[] [float] -==== `firewall` fileset settings +==== `xg` fileset settings + +The Sophos XG firewalls do not include hostname in either the syslog header or body, and the only unique identifier for each firewall is the related serial number. + +Below you will see an example configuration file, that sets the default hostname (if no serial number is included in the config file), and example on how to map serial numbers to a hostname [source,yaml] ---- -- module: sophosxg - firewall: +- module: sophos + xg: enabled: true var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9005 - var.host_name: firewall.localgroup.local + var.default_host_name: firewall.localgroup.local + var.known_devices: + "1234567890123457": "a.host.local" + "1234234590678557": "b.host.local" ---- include::../include/var-paths.asciidoc[] @@ -68,7 +75,7 @@ Default to `firewall.localgroup.local` [float] ==== SophosXG ECS fields -This is a list of FortiOS fields that are mapped to ECS. +This is a list of SophosXG fields that are mapped to ECS. [options="header"] |============================================================== @@ -139,5 +146,5 @@ This is a list of FortiOS fields that are mapped to ECS. === Fields For a description of each field in the module, see the -<> section. +<> section. diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index cf898fde975..f4c8f1d84ba 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -55,7 +55,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> - * <> + * <> * <> * <> * <> @@ -121,7 +121,7 @@ include::modules/rapid7.asciidoc[] include::modules/redis.asciidoc[] include::modules/santa.asciidoc[] include::modules/sonicwall.asciidoc[] -include::modules/sophosxg.asciidoc[] +include::modules/sophos.asciidoc[] include::modules/squid.asciidoc[] include::modules/suricata.asciidoc[] include::modules/system.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f5d235404bf..4ce72f66813 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1340,9 +1340,9 @@ filebeat.modules: # "+02:00" for GMT+02:00 # var.tz_offset: local -#------------------------------- Sophosxg Module ------------------------------- -- module: sophosxg - firewall: +#-------------------------------- Sophos Module -------------------------------- +- module: sophos + xg: enabled: true # Set which input to use between tcp, udp (default) or file. @@ -1355,9 +1355,9 @@ filebeat.modules: # The port to listen for syslog traffic. Defaults to 9004. #var.syslog_port: 9005 - # firewall default hostanme + # firewall default hostname #var.default_host_name: firewall.localgroup.local - + # known firewalls #var.known_devices: # "device1_serialnumber": "a.host.local" diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 6f79780a2e1..3cc9adb51d0 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -49,7 +49,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/radware" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/rapid7" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/sonicwall" - _ "github.com/elastic/beats/v7/x-pack/filebeat/module/sophosxg" + _ "github.com/elastic/beats/v7/x-pack/filebeat/module/sophos" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/squid" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/suricata" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/tenable" diff --git a/x-pack/filebeat/module/sophosxg/_meta/config.yml b/x-pack/filebeat/module/sophos/_meta/config.yml similarity index 88% rename from x-pack/filebeat/module/sophosxg/_meta/config.yml rename to x-pack/filebeat/module/sophos/_meta/config.yml index 6d605b852e1..c7c5add7422 100644 --- a/x-pack/filebeat/module/sophosxg/_meta/config.yml +++ b/x-pack/filebeat/module/sophos/_meta/config.yml @@ -1,5 +1,5 @@ -- module: sophosxg - firewall: +- module: sophos + xg: enabled: true # Set which input to use between tcp, udp (default) or file. @@ -12,9 +12,9 @@ # The port to listen for syslog traffic. Defaults to 9004. #var.syslog_port: 9005 - # firewall default hostanme + # firewall default hostname #var.default_host_name: firewall.localgroup.local - + # known firewalls #var.known_devices: # "device1_serialnumber": "a.host.local" diff --git a/x-pack/filebeat/module/sophosxg/_meta/docs.asciidoc b/x-pack/filebeat/module/sophos/_meta/docs.asciidoc similarity index 86% rename from x-pack/filebeat/module/sophosxg/_meta/docs.asciidoc rename to x-pack/filebeat/module/sophos/_meta/docs.asciidoc index 304b2ca88a3..28035328083 100644 --- a/x-pack/filebeat/module/sophosxg/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/sophos/_meta/docs.asciidoc @@ -1,11 +1,11 @@ [role="xpack"] -:modulename: sophosxg +:modulename: sophos :has-dashboards: false -== SophosXG module +== Sophos module -This is a module for SophosXG SFOS logs sent in the syslog format. +This is a module for Sophos Products, currently it supports XG SFOS logs sent in the syslog format. To configure a remote syslog destination, please reference the https://community.sophos.com/kb/en-us/123184[SophosXG/SFOS Documentation]. @@ -16,27 +16,34 @@ include::../include/gs-link.asciidoc[] [float] === Compatibility -This module has been tested against SFOS version 17.5.x and 18.0.x. +This module has been tested against SFOS version 17.5.x and 18.0.x. Versions above this are expected to work but have not been tested. include::../include/configuring-intro.asciidoc[] -:fileset_ex: firewall +:fileset_ex: xg include::../include/config-option-intro.asciidoc[] [float] -==== `firewall` fileset settings +==== `xg` fileset settings + +The Sophos XG firewalls do not include hostname in either the syslog header or body, and the only unique identifier for each firewall is the related serial number. + +Below you will see an example configuration file, that sets the default hostname (if no serial number is included in the config file), and example on how to map serial numbers to a hostname [source,yaml] ---- -- module: sophosxg - firewall: +- module: sophos + xg: enabled: true var.input: udp var.syslog_host: 0.0.0.0 var.syslog_port: 9005 - var.host_name: firewall.localgroup.local + var.default_host_name: firewall.localgroup.local + var.known_devices: + "1234567890123457": "a.host.local" + "1234234590678557": "b.host.local" ---- include::../include/var-paths.asciidoc[] @@ -63,7 +70,7 @@ Default to `firewall.localgroup.local` [float] ==== SophosXG ECS fields -This is a list of FortiOS fields that are mapped to ECS. +This is a list of SophosXG fields that are mapped to ECS. [options="header"] |============================================================== diff --git a/x-pack/filebeat/module/sophosxg/_meta/fields.yml b/x-pack/filebeat/module/sophos/_meta/fields.yml similarity index 50% rename from x-pack/filebeat/module/sophosxg/_meta/fields.yml rename to x-pack/filebeat/module/sophos/_meta/fields.yml index 63386abd814..ea0412ba5ca 100644 --- a/x-pack/filebeat/module/sophosxg/_meta/fields.yml +++ b/x-pack/filebeat/module/sophos/_meta/fields.yml @@ -1,9 +1,9 @@ -- key: sophosxg - title: "sophosxg" +- key: sophos + title: "sophos" description: > - sophosxg Module + sophos Module fields: - - name: sophosxg + - name: sophos type: group description: > fields: diff --git a/x-pack/filebeat/module/sophos/fields.go b/x-pack/filebeat/module/sophos/fields.go new file mode 100644 index 00000000000..11b91b9dd6d --- /dev/null +++ b/x-pack/filebeat/module/sophos/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package sophos + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "sophos", asset.ModuleFieldsPri, AssetSophos); err != nil { + panic(err) + } +} + +// AssetSophos returns asset data. +// This is the base64 encoded gzipped contents of module/sophos. +func AssetSophos() string { + return "eJzUXMFy4zYSvc9XoJLLpCrj3dmjD1vl2J4dV40nWklOjiwIbJFYgwADgJLl0/zDXpOfmy/ZAkjKpAQKtNRKZj0nUZrXj92N7kY3yHfkETaXxKgyV+YNIZZbAZfku/rCd28IScEwzUvLlbwk/3xDCGl+Te5VWgl4Q8iSg0jNpf/uHZG0gA6i+7ObEi5JplVVNlcCqH2cLtZTtr0UwnJ/GgRQA5dkAZZ2rqewpJWwiYe+JEsqDPS+DtCo/+q7I0ulSUm14TJr7ukpI2ZjhMouOr/fpd6ln8KKM+h91d7GI2zWSqc73x1gVX/dwwvLpDYsMfBFRNwNtUDebjabzbuieJemP5B1DpLYHAisQFqiGKu0hvQwI8sLeFYSUQ9zXgB5m+eXRXFpzJG0amUm7gMes3uVgiCyKhagiVp6UqOsVpPhu+JOoDIDzekxXITKUIk8SP5bBeT9PwjLqabMgjaEqRTI27+/d//e/xAn5OQjOtCmBKeS2l/gIrsgS65hTYWor8UJMVWUSnZ/ejKr6xaSaDClkoYvmjgkVJa5OOSZfmiZ6joIR4iaaoGrvFm18P99q8DDHPJFkgMVNsdj8BGotguglhhLbWUOEyg1V5rbDebSWoFDJAJWIPza0nS55OwwkR2yJ9N4EJYXLkbXwB0e5OuX/5IrIdQaUqI0uQHJo/Gw0tRJCvITSmavTB6VpgsunJY6vN4aYEqmJrLil+vEOfdQGOLSQgb6dYS2y2bqsvvdDVnnnOWEG0LLUnCnqDqJjLJlZUAj5459yGHJu1XQiaL/5eA8PrGqUYxThZNFFuCsH1lknCLSuZMWtARLrhgDY0ipBGcbZ7NjTMVLk9QIqP50N5mdSKwhhRudJzUlH6BbUlaNJ0XLcsmFBX0OnV05QsyHGfLBS2lVeIz+6Ata0uP8f0AYz+BdivHo0dUZbvg6nofm5hGPx5SbxyYxU2N4Jl8WQEj/UXYWWC6VUBliATHfYraF+VHUGLWQKY1IrGtEblwdqsQKUrLYEKdKaisNrqAwG8lyrSR/hvRV3FvExTejzYSbhAlVIe54drR43UMP5yiZuPiklxSzX3DXQvpNBJdMFW4X0cSqH/124kcyUdqSq8P8VGXPT1BVNlNDBH+K1NaaJTxcguxdjpD6WfOMSyqIUZVmQO4mhKapdoXI6Epfs6SgDE9Ru5zur66PIsVUJa3eJG77jblxTaFdeo2Efh35oslRxWRqLLoxUzCWy3pNHmFRR+kvUt4O81EaLLWyiimBWFc2iN2G0livK5UOt0mOqsl214JDd3Tm1xNCZUoebibjbXomal2bHc2Ps6JE3h3cXd9Pts2b+sNoJrhu74WzxvfHMzEgbVI+2nAb5fVtirmyvSZpSdkjWOPlHGaigQFfQfpnsGlljdDNYmPhXHQ89jjVrP4EIuO0YjWVJnFhaD95HJdT5g5RUAtpoEQIFTJjCeIGo32aPhIdSxAxKXeYDaTlU0ieTYt7Qf31LI1mz0rCeaYZjZWdgLZ0/nT1eRQhPDKfabFLJpqFz6eSrsm6evk1ppeGFb5edhlFaHCdpNwg9lbnE59THDIw96MLMlGmHjWtqKjAXH798rvS2dcvf/xIvn75XUMpNu2Hr1/+OEyYKSn7Q6GTGd/WQ125LYm5IUJlbmedgQTt1macFGofsxlp8hSk5UteJyV/6yzec1ih07neSiZ3N+1eoqDGgh7Niqfl+bvkJ41cOgxxO5cdjn4Ewi/gYu/iSdS3vTPMwfps25C7uxkrvjC7ldcpY9AtgQKMMTSLBDMmqDF8id4Bf+ExICFcpFYCEvzZ8KRB9NuarQ8dplIKapdKF4gsGsQ2GDReehExEHo/2WngpXHc4kdoWKozQEwgxynDZzHMczBX84nPZWCjCxbKpKowZd/KtFRcWvLwEJNtcw3UIs+G5pMQbjhY+cLx2+nldgq2b6wnKVTGZVKZvZx8oqU8LunhDq8Q3Lrdia8PZfVww+KfgFV+DlZSzNNFnkOLTXrYQR50dZ6y5J6KNdVADKNSuh0mWjmy1KpIoKBcJI3PISZjkClo4tHJLno44KhzcZmCCyTg4t4r+Jhq8R9gmBsYL3sXNijb/dLw57AXHVWA7yGG9yRcY55M83AjTgF8mE+SSiNOKT7MJ+Rh+sl7eLNGai5rakiq1lIomsa2i47VdnuMx+2mhXQB3hH1Lasl6EvyUDpe/qRewzGygLkAXDeZ8eftFMqhE5tT6zaPlnIJ6Y6DDJLCjcQTavMeqYaPC4YjCeGG5A+OhI+/1BjFuG8LrrnNX06fRyjZkqmioBKxtnOu1IC6pJ3Wx+Ff3H6pKhnxJ9Q1ePL6S1VBucS13I3HPI3XbxXVVFqO2Rb0Lu4s1zprz99fJMZGP5ol51Bbk81r6BEBPTX2LDym9ajnFUw0UIMZvacej6zzjTePBqZ0WrsPWGAuEFBDTEmLvxVUcMZVLDxpWILGrN2nO4BhTylpQZEz26ykBbka0WR0xQjmtnYHL7JqE2yf+PcWetfdwqr3x/SR5+iz+ux/DzQoXa1Aa55CYtUjICrh5waX9HGDHJj6hvrwW4XQyuZK82fMtbjVSgA8vAtyxWC9MBFXyPwFtTMauPrFbyojDbDKF6WJS0TIp1CacreuM6P7/C4P3KzS5RHPKLm1ZfNgFgyu4defbWjPwHyczyftg1+jHgH5k5//GNk7PtOTei08qXxBUlNcw8JwC2RJhYgdKWxqvHO4c7tnG+vQ8MTAg+HtuD9xY7eHCHNgj8ZJEVVan91ew6IjNWZJJS16Y6+1Y33K0QsYGZ1xl/w2Lo94RIFZvuJ2g0vgV1hsHyFpBNR77YJalrsqUqaEUb+Rc8pqfqvBVMJGLNd3cuwNVNfJR7Txc/reVIgjrdnHq/eNb1fbUQ63UJAFcJkRKqnYPMc2SgX4QSXqA873NWZ0quMKE0y5vbH/qNtGv+dIoPNt3TP1U2dj+ql+IeA2pXwPKD4VSI2t96qIS7AzLrrpYw/fPG6b0N/8LNpO7hSzqCutX84e5tAUB6iRuwc4rHP8pmM83vqk+mSTUsOSP6EGGZesyaSPG4753MIZHkqbuRrvehf2oBZMtcTXwpMlsz5ueOVz76BUYw8Eb7bA/jRhvHrB7HXUp/DGNHlwB9IP0TF0a3NfQOGb/L4HO2Dx8w2LIlaubJ4wwVEPXF5VNifXfdBh6QWwnEpuEEs9T2Afd7CoqjWFu9iuw7hBDhoKZUGCXSuN+CRzGDbIQChGRUYtrCli4P0UQh2Wj66ATyHUiBfgbluvw7hBDkqkK9AGNQz8vI8ZlC1hjS778z5mUDYv0Q9u3O1Bhq3uY1RS5hvDGT3DAZI6CJJWwLjzIw2rXBmLXAA0dBzymAkYXScp7b0U70T5U7omPcSg3HvU+cZ9dLBhwBjcrjmZ7UGGJVuqreUDJn79C/dme3gH0g3Ww1hTj0buJoeFOlrG0iIs9PX3uo8XFDvbmE8qS2a3019up8nnq/tbRCt7bFJjkx52kMuCsseqTApU//7Jg5Ii7ub+BCvi3ffxBpaWXmEW87M+XlCmi254Ej920QZWUz1yGVzHrx/pTEOQA3WLeuSIRt3BC8r8rQK9MVbzvds6QXAINCgdnqxGTEl9uKDEycfJ7HY26/SLTpa6DzmcHxLsBEHiblWfzMaTW2/+x2SmFLXfN729ifb5FppKliNXWj950BFtlqp02kyRcvE+WrjiTgUkrKyCMpdC0d0AGhHr8Mj330eceWMsFJhia8SIfg3ov0Am4hmVHlpYnuSIGa+HFi7llKUiKaAY6tIe96C8f0fDfR81KH+pAXEk0UML36+m0hTcWkhBa6URN4gdaLKDPRAg61dWYPNocV9F4nHB0d5csiXQB43ZA5NC1xavY5FqVZ6DRR/3oC0wKWxtMUI+U0JwM3gW5ZjjTHuA4RyG/za5fcig5Gsllzw79LbjI/JKGDQon0yhVHrA7Y8QvYcXlLp9SBpL7D5gUO4cBloGR4jsYQ2WC4hhtQ8XLosMZsHbQwvKw3zPMo1os+5kmsS/sAL3RhvoTuO+vvf/BQAA//8gM69j" +} diff --git a/x-pack/filebeat/module/sophosxg/module.yml b/x-pack/filebeat/module/sophos/module.yml similarity index 100% rename from x-pack/filebeat/module/sophosxg/module.yml rename to x-pack/filebeat/module/sophos/module.yml diff --git a/x-pack/filebeat/module/sophosxg/firewall/_meta/fields.yml b/x-pack/filebeat/module/sophos/xg/_meta/fields.yml similarity index 98% rename from x-pack/filebeat/module/sophosxg/firewall/_meta/fields.yml rename to x-pack/filebeat/module/sophos/xg/_meta/fields.yml index 69d2796ca57..efb17a6a7b8 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/_meta/fields.yml +++ b/x-pack/filebeat/module/sophos/xg/_meta/fields.yml @@ -1,4 +1,4 @@ -- name: firewall +- name: xg type: group release: beta default_field: false @@ -9,7 +9,7 @@ type: keyword description: > device - + - name: date type: date description: > @@ -19,12 +19,12 @@ type: keyword description: > Time (hh:mm:ss) when the event occurred - + - name: device_name type: keyword description: > Model number of the device - + - name: device_id type: keyword description: > @@ -34,17 +34,17 @@ type: keyword description: > Unique 12 characters code (0101011) - + - name: log_type type: keyword description: > Type of event e.g. firewall event - + - name: log_component type: keyword description: > Component responsible for logging e.g. Firewall rule - + - name: log_subtype type: keyword description: > @@ -59,7 +59,7 @@ type: keyword description: > Severity level of traffic - + - name: status type: keyword description: > @@ -69,17 +69,17 @@ type: long description: > Durability of traffic (seconds) - + - name: fw_rule_id type: integer description: > Firewall Rule ID which is applied on the traffic - + - name: user_name type: keyword description: > user_name - + - name: user_group type: keyword description: > @@ -124,7 +124,7 @@ type: keyword description: > Risk level assigned to the application - + - name: application_technology type: keyword description: > @@ -154,7 +154,7 @@ type: keyword description: > Interface for outgoing traffic, e.g., Port B - + - name: src_ip type: ip description: > @@ -164,17 +164,17 @@ type: keyword description: > Original source MAC address of traffic - + - name: src_country_code type: keyword description: > Code of the country to which the source IP belongs - + - name: dst_ip type: ip description: > Original destination IP address of traffic - + - name: dst_country_code type: keyword description: > @@ -194,7 +194,7 @@ type: integer description: > Original destination port of TCP and UDP traffic - + - name: icmp_type type: keyword description: > @@ -204,17 +204,17 @@ type: keyword description: > ICMP code of ICMP traffic - + - name: sent_pkts type: long description: > Total number of packets sent - + - name: received_pkts type: long description: > Total number of packets received - + - name: sent_bytes type: long description: > @@ -234,7 +234,7 @@ type: integer description: > Translated source port for outgoing traffic - + - name: trans_dst_ip type: ip description: > @@ -244,17 +244,17 @@ type: integer description: > Translated destination port for outgoing traffic - + - name: srczonetype type: keyword description: > Type of source zone, e.g., LAN - + - name: srczone type: keyword description: > Name of source zone - + - name: dstzonetype type: keyword description: > @@ -269,12 +269,12 @@ type: keyword description: > TPacket direction. Possible values:“org”, “reply”, “” - + - name: connevent type: keyword description: > Event on which this log is generated - + - name: conn_id type: integer description: > @@ -289,7 +289,7 @@ type: integer description: > IPS policy ID which is applied on the traffic - + - name: idp_policy_name type: keyword description: > @@ -304,12 +304,12 @@ type: keyword description: > Signature messsage - + - name: classification type: keyword description: > Signature classification - + - name: rule_priority type: keyword description: > @@ -334,12 +334,12 @@ type: keyword description: > ATP Evenet ID - + - name: ep_uuid type: keyword description: > Endpoint UUID - + - name: threatname type: keyword description: > @@ -374,12 +374,12 @@ type: keyword description: > Malware scanning policy name which is applied on the traffic - + - name: from_email_address type: keyword description: > Sender email address - + - name: to_email_address type: keyword description: > @@ -414,7 +414,7 @@ type: integer description: > Size of the file that contained virus - + - name: filepath type: keyword description: > @@ -424,7 +424,7 @@ type: keyword description: > File name associated with the event - + - name: ftpcommand type: keyword description: > @@ -484,12 +484,12 @@ type: keyword description: > Status code - + - name: override_token type: keyword description: > Override token - + - name: con_id type: integer description: > @@ -648,7 +648,7 @@ - name: context_match type: keyword description: > - Context Match + Context Match - name: direction type: keyword @@ -669,7 +669,7 @@ type: keyword description: > Connectionname - + - name: remotenetwork type: keyword description: > @@ -934,4 +934,3 @@ type: keyword description: > clients connection ssid - \ No newline at end of file diff --git a/x-pack/filebeat/module/sophosxg/firewall/config/firewall.yml b/x-pack/filebeat/module/sophos/xg/config/config.yml similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/config/firewall.yml rename to x-pack/filebeat/module/sophos/xg/config/config.yml diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/antispam.yml b/x-pack/filebeat/module/sophos/xg/ingest/antispam.yml similarity index 82% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/antispam.yml rename to x-pack/filebeat/module/sophos/xg/ingest/antispam.yml index 63d984d868c..dc58149d7c7 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/antispam.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/antispam.yml @@ -8,7 +8,7 @@ processors: value: event - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" + value: "{{sophos.xg.log_subtype}}" ignore_empty_value: true - set: field: event.outcome @@ -17,15 +17,15 @@ processors: - set: field: event.kind value: alert - if: '["13001", "13002", "13004", "13005", "13006", "13009", "13012", "13014", "14001", "14002", "15001", "15002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["13001", "13002", "13004", "13005", "13006", "13009", "13012", "13014", "14001", "14002", "15001", "15002"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.category value: malware - if: '["13001", "13002", "13004", "13005", "13006", "13009", "13014", "14001", "14002", "15001", "15002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["13001", "13002", "13004", "13005", "13006", "13009", "13014", "14001", "14002", "15001", "15002"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.category value: intrusion_detection - if: "ctx.sophosxg?.firewall?.message_id == '13012'" + if: "ctx.sophos?.xg?.message_id == '13012'" - append: field: event.category value: network @@ -34,34 +34,34 @@ processors: value: - allowed - connection - if: '["13003", "13007", "13008", "13010", "13013", "14003", "15003", "18035"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["13003", "13007", "13008", "13010", "13013", "14003", "15003", "18035"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.type value: - info - denied - connection - if: '["13001", "13002", "13004", "13005", "13006", "13009", "13012", "13014", "14001", "14002", "15001", "15002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["13001", "13002", "13004", "13005", "13006", "13009", "13012", "13014", "14001", "14002", "15001", "15002"].contains(ctx.sophos?.xg?.message_id)' #################################### ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.dst_ip + field: sophos.xg.dst_ip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_ip != null" + if: "ctx.sophos?.xg?.dst_ip != null" - set: field: server.ip value: '{{destination.ip}}' ignore_empty_value: true - convert: - field: sophosxg.firewall.dst_port + field: sophos.xg.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_port != null" + if: "ctx.sophos?.xg?.dst_port != null" - set: field: server.port value: '{{destination.port}}' @@ -72,12 +72,12 @@ processors: ignore_failure: true ignore_missing: true - convert: - field: sophosxg.firewall.recv_bytes + field: sophos.xg.recv_bytes target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.recv_bytes != null" + if: "ctx.sophos?.xg?.recv_bytes != null" - set: field: server.bytes value: '{{destination.bytes}}' @@ -89,7 +89,7 @@ processors: ignore_missing: true if: "ctx.server?.bytes != null" - rename: - field: sophosxg.firewall.to_email_address + field: sophos.xg.to_email_address target_field: destination.user.email ignore_missing: true @@ -97,7 +97,7 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - set: @@ -105,12 +105,12 @@ processors: value: '{{source.ip}}' ignore_empty_value: true - convert: - field: sophosxg.firewall.src_port + field: sophos.xg.src_port target_field: source.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_port != null" + if: "ctx.sophos?.xg?.src_port != null" - set: field: client.port value: '{{source.port}}' @@ -121,12 +121,12 @@ processors: ignore_failure: true ignore_missing: true - convert: - field: sophosxg.firewall.sent_bytes + field: sophos.xg.sent_bytes target_field: source.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.sent_bytes != null" + if: "ctx.sophos?.xg?.sent_bytes != null" - set: field: client.bytes value: '{{source.bytes}}' @@ -138,11 +138,11 @@ processors: ignore_missing: true if: "ctx.client?.bytes != null" - rename: - field: sophosxg.firewall.from_email_address + field: sophos.xg.from_email_address target_field: source.user.email ignore_missing: true - rename: - field: sophosxg.firewall.src_domainname + field: sophos.xg.src_domainname target_field: source.domain ignore_missing: true @@ -150,7 +150,7 @@ processors: ## ECS Network/Geo Mapping ## ############################# - rename: - field: sophosxg.firewall.protocol + field: sophos.xg.protocol target_field: network.transport ignore_missing: true - geoip: @@ -229,12 +229,12 @@ processors: ############# - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.recv_bytes - - sophosxg.firewall.src_port - - sophosxg.firewall.sent_bytes + - sophos.xg.dst_port + - sophos.xg.recv_bytes + - sophos.xg.src_port + - sophos.xg.sent_bytes ignore_missing: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/antivirus.yml b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml similarity index 71% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/antivirus.yml rename to x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml index 54747b7a89c..bb2548bf941 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/antivirus.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml @@ -8,41 +8,41 @@ processors: value: alert - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" - if: "ctx.sophosxg?.firewall?.log_subtype != null" + value: "{{sophos.xg.log_subtype}}" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.log_subtype != null" + if: "ctx.sophos?.xg?.log_subtype != null" - append: field: event.category value: - malware - network - if: "ctx.sophosxg?.firewall?.log_subtype == 'Virus'" + if: "ctx.sophos?.xg?.log_subtype == 'Virus'" - append: field: event.type value: - info - denied - connection - if: "ctx.sophosxg?.firewall?.log_subtype == 'Virus'" + if: "ctx.sophos?.xg?.log_subtype == 'Virus'" - set: field: event.kind value: event - if: '["09002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["09002"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.type value: - allowed - connection - if: '["09002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["09002"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.category value: network - if: '["09002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["09002"].contains(ctx.sophos?.xg?.message_id)' - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true if: "ctx.event?.code == null" @@ -51,21 +51,21 @@ processors: ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.dst_ip + field: sophos.xg.dst_ip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_ip != null" + if: "ctx.sophos?.xg?.dst_ip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - convert: - field: sophosxg.firewall.dst_port + field: sophos.xg.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_port != null" + if: "ctx.sophos?.xg?.dst_port != null" - set: field: server.port value: '{{destination.port}}' @@ -78,12 +78,12 @@ processors: ignore_missing: true if: "ctx.server?.port != null" - convert: - field: sophosxg.firewall.recv_bytes + field: sophos.xg.recv_bytes target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.recv_bytes != null" + if: "ctx.sophos?.xg?.recv_bytes != null" - set: field: server.bytes value: '{{destination.bytes}}' @@ -96,30 +96,30 @@ processors: ignore_missing: true if: "ctx.server?.bytes != null" - rename: - field: sophosxg.firewall.to_email_address + field: sophos.xg.to_email_address target_field: destination.user.email ignore_missing: true - if: "ctx.sophosxg?.firewall?.to_email_address != null" + if: "ctx.sophos?.xg?.to_email_address != null" ############################### ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - convert: - field: sophosxg.firewall.src_port + field: sophos.xg.src_port target_field: source.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_port != null" + if: "ctx.sophos?.xg?.src_port != null" - set: field: client.port value: '{{source.port}}' @@ -132,12 +132,12 @@ processors: ignore_missing: true if: "ctx.client?.port != null" - convert: - field: sophosxg.firewall.sent_bytes + field: sophos.xg.sent_bytes target_field: source.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.sent_bytes != null" + if: "ctx.sophos?.xg?.sent_bytes != null" - set: field: client.bytes value: '{{source.bytes}}' @@ -150,21 +150,21 @@ processors: ignore_missing: true if: "ctx.client?.bytes != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" - rename: - field: sophosxg.firewall.from_email_address + field: sophos.xg.from_email_address target_field: source.user.email ignore_missing: true - if: "ctx.sophosxg?.firewall?.from_email_address != null" + if: "ctx.sophos?.xg?.from_email_address != null" ###################### ## ECS Rule Mapping ## ###################### - rename: - field: sophosxg.firewall.fw_rule_id + field: sophos.xg.fw_rule_id target_field: rule.id ignore_missing: true if: "ctx.rule?.id == null" @@ -173,71 +173,71 @@ processors: ## ECS URL Mapping ## ##################### - rename: - field: sophosxg.firewall.url + field: sophos.xg.url target_field: url.original ignore_missing: true - if: "ctx.sophosxg?.firewall?.url != null" + if: "ctx.sophos?.xg?.url != null" - rename: - field: sophosxg.firewall.domainname + field: sophos.xg.domainname target_field: url.domain ignore_missing: true - if: "ctx.sophosxg?.firewall?.domainname != null" + if: "ctx.sophos?.xg?.domainname != null" - rename: - field: sophosxg.firewall.dst_domainname + field: sophos.xg.dst_domainname target_field: url.domain ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_domainname != null && ctx?.url?.domain == null" + if: "ctx.sophos?.xg?.dst_domainname != null && ctx?.url?.domain == null" - rename: - field: sophosxg.firewall.src_domainname + field: sophos.xg.src_domainname target_field: url.domain ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_domainname != null" + if: "ctx.sophos?.xg?.src_domainname != null" ############################ ## ECS User Agent Mapping ## ############################ - rename: - field: sophosxg.firewall.user_agent + field: sophos.xg.user_agent target_field: user_agent.original ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_agent != null" + if: "ctx.sophos?.xg?.user_agent != null" - convert: - field: sophosxg.firewall.status_code + field: sophos.xg.status_code target_field: http.response.status_code type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.status_code != null" + if: "ctx.sophos?.xg?.status_code != null" ###################### ## ECS File Mapping ## ###################### - rename: - field: sophosxg.firewall.filename + field: sophos.xg.filename target_field: file.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.filename != null" + if: "ctx.sophos?.xg?.filename != null" - convert: - field: sophosxg.firewall.file_size + field: sophos.xg.file_size target_field: file.size type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.file_size != null" + if: "ctx.sophos?.xg?.file_size != null" - rename: - field: sophosxg.firewall.file_path + field: sophos.xg.file_path target_field: file.directory ignore_missing: true - if: "ctx.sophosxg?.firewall?.file_path != null" + if: "ctx.sophos?.xg?.file_path != null" ############################# ## ECS Network/Geo Mapping ## ############################# - rename: - field: sophosxg.firewall.protocol + field: sophos.xg.protocol target_field: network.transport ignore_missing: true - if: "ctx.sophosxg?.firewall?.protocol != null" + if: "ctx.sophos?.xg?.protocol != null" - geoip: field: source.ip target_field: source.geo @@ -333,14 +333,14 @@ processors: ignore_failure: true - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.src_port - - sophosxg.firewall.recv_bytes - - sophosxg.firewall.sent_bytes - - sophosxg.firewall.status_code - - sophosxg.firewall.file_size + - sophos.xg.dst_port + - sophos.xg.src_port + - sophos.xg.recv_bytes + - sophos.xg.sent_bytes + - sophos.xg.status_code + - sophos.xg.file_size ignore_missing: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/atp.yml b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml similarity index 81% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/atp.yml rename to x-pack/filebeat/module/sophos/xg/ingest/atp.yml index 0083725aec9..df6ed8b35ca 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/atp.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml @@ -8,54 +8,54 @@ processors: value: alert - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" - if: "ctx.sophosxg?.firewall?.log_subtype != null" + value: "{{sophos.xg.log_subtype}}" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.log_subtype != null" + if: "ctx.sophos?.xg?.log_subtype != null" - append: field: event.category value: - intrusion_detection - network - if: '["18009", "18010"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["18009", "18010"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.type value: - denied - connection - if: '["18009", "18010"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["18009", "18010"].contains(ctx.sophos?.xg?.message_id)' - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true if: "ctx.event?.code == null" - rename: - field: sophosxg.firewall.eventid + field: sophos.xg.eventid target_field: event.id ignore_missing: true - if: "ctx.sophosxg?.firewall?.eventid != null" + if: "ctx.sophos?.xg?.eventid != null" #################################### ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.destinationip + field: sophos.xg.destinationip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.destinationip != null" + if: "ctx.sophos?.xg?.destinationip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - convert: - field: sophosxg.firewall.dst_port + field: sophos.xg.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_port != null" + if: "ctx.sophos?.xg?.dst_port != null" - set: field: server.port value: '{{destination.port}}' @@ -72,30 +72,30 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.sourceip + field: sophos.xg.sourceip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.sourceip != null" + if: "ctx.sophos?.xg?.sourceip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - convert: - field: sophosxg.firewall.src_port + field: sophos.xg.src_port target_field: source.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_port != null" + if: "ctx.sophos?.xg?.src_port != null" - set: field: client.port value: '{{source.port}}' @@ -108,7 +108,7 @@ processors: ignore_missing: true if: "ctx.client?.port != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true @@ -116,19 +116,19 @@ processors: ## ECS URL Mapping ## ##################### - rename: - field: sophosxg.firewall.url + field: sophos.xg.url target_field: url.original ignore_missing: true - if: "ctx.sophosxg?.firewall?.url != null" + if: "ctx.sophos?.xg?.url != null" ############################# ## ECS Network/Geo Mapping ## ############################# - rename: - field: sophosxg.firewall.protocol + field: sophos.xg.protocol target_field: network.transport ignore_missing: true - if: "ctx.sophosxg?.firewall?.protocol != null" + if: "ctx.sophos?.xg?.protocol != null" - geoip: field: source.ip target_field: source.geo @@ -233,8 +233,8 @@ processors: ignore_failure: true - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.src_port + - sophos.xg.dst_port + - sophos.xg.src_port ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/cfilter.yml b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml similarity index 75% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/cfilter.yml rename to x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml index 634e8deb11b..a9dedb4070f 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/cfilter.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml @@ -8,41 +8,41 @@ processors: value: event - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" - if: "ctx.sophosxg?.firewall?.log_subtype != null" + value: "{{sophos.xg.log_subtype}}" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.log_subtype != null" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.kind value: alert - if: 'ctx.sophosxg?.firewall?.log_subtype == "Denied"' + if: 'ctx.sophos?.xg?.log_subtype == "Denied"' - append: field: event.category value: - malware - network - if: 'ctx.sophosxg?.firewall?.log_subtype == "Denied"' + if: 'ctx.sophos?.xg?.log_subtype == "Denied"' - append: field: event.category value: network - if: "ctx.sophosxg?.firewall?.log_subtype != 'Denied'" + if: "ctx.sophos?.xg?.log_subtype != 'Denied'" - append: field: event.type value: - allowed - connection - if: '["Allowed", "Warned"].contains(ctx.sophosxg?.firewall?.log_subtype)' + if: '["Allowed", "Warned"].contains(ctx.sophos?.xg?.log_subtype)' - append: field: event.type value: - info - denied - connection - if: "ctx.sophosxg?.firewall?.log_subtype == 'Denied'" + if: "ctx.sophos?.xg?.log_subtype == 'Denied'" - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true if: "ctx.event?.code == null" @@ -51,21 +51,21 @@ processors: ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.dst_ip + field: sophos.xg.dst_ip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_ip != null" + if: "ctx.sophos?.xg?.dst_ip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - convert: - field: sophosxg.firewall.dst_port + field: sophos.xg.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_port != null" + if: "ctx.sophos?.xg?.dst_port != null" - set: field: server.port value: '{{destination.port}}' @@ -82,21 +82,21 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - convert: - field: sophosxg.firewall.src_port + field: sophos.xg.src_port target_field: source.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_port != null" + if: "ctx.sophos?.xg?.src_port != null" - set: field: client.port value: '{{source.port}}' @@ -109,57 +109,57 @@ processors: ignore_missing: true if: "ctx.client?.port != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" - rename: - field: sophosxg.firewall.user_gp + field: sophos.xg.user_gp target_field: source.user.group.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_gp != null" + if: "ctx.sophos?.xg?.user_gp != null" ##################### ## ECS URL Mapping ## ##################### - rename: - field: sophosxg.firewall.url + field: sophos.xg.url target_field: url.full ignore_missing: true - if: "ctx.sophosxg?.firewall?.url != null" + if: "ctx.sophos?.xg?.url != null" - rename: - field: sophosxg.firewall.domain + field: sophos.xg.domain target_field: url.domain ignore_missing: true - if: "ctx.sophosxg?.firewall?.domain != null" + if: "ctx.sophos?.xg?.domain != null" ############################ ## ECS User Agent Mapping ## ############################ - rename: - field: sophosxg.firewall.referer + field: sophos.xg.referer target_field: http.request.referrer ignore_missing: true - if: "ctx.sophosxg?.firewall?.referer != null" + if: "ctx.sophos?.xg?.referer != null" - rename: - field: sophosxg.firewall.status_code + field: sophos.xg.status_code target_field: http.response.status_code ignore_missing: true - if: "ctx.sophosxg?.firewall?.status_code != null" + if: "ctx.sophos?.xg?.status_code != null" - rename: - field: sophosxg.firewall.user_agent + field: sophos.xg.user_agent target_field: user_agent.original ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_agent != null" + if: "ctx.sophos?.xg?.user_agent != null" ############################# ## ECS Network/Geo Mapping ## ############################# - rename: - field: sophosxg.firewall.protocol + field: sophos.xg.protocol target_field: network.transport ignore_missing: true - if: "ctx.sophosxg?.firewall?.protocol != null" + if: "ctx.sophos?.xg?.protocol != null" - geoip: field: source.ip target_field: source.geo @@ -261,10 +261,10 @@ processors: ignore_failure: true - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.src_port - - sophosxg.firewall.recv_bytes - - sophosxg.firewall.sent_bytes + - sophos.xg.dst_port + - sophos.xg.src_port + - sophos.xg.recv_bytes + - sophos.xg.sent_bytes ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/event.yml b/x-pack/filebeat/module/sophos/xg/ingest/event.yml similarity index 72% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/event.yml rename to x-pack/filebeat/module/sophos/xg/ingest/event.yml index d172166967d..2565434a6f0 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/event.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/event.yml @@ -9,55 +9,55 @@ processors: - set: field: event.outcome value: success - if: 'ctx?.sophosxg?.firewall?.log_subtype == "Authentication" && ctx?.sophosxg?.firewall?.status == "Successful"' + if: 'ctx?.sophos?.xg?.log_subtype == "Authentication" && ctx?.sophos?.xg?.status == "Successful"' - set: field: event.outcome value: failure - if: 'ctx?.sophosxg?.firewall?.log_subtype == "Authentication" && ctx?.sophosxg?.firewall?.status == "Failed"' + if: 'ctx?.sophos?.xg?.log_subtype == "Authentication" && ctx?.sophos?.xg?.status == "Failed"' - set: field: event.outcome value: success - if: 'ctx?.sophosxg?.firewall?.log_subtype == "Admin" && ctx?.sophosxg?.firewall?.status == "Successful" && ctx?.sophosxg?.firewall?.message_id == "17507"' + if: 'ctx?.sophos?.xg?.log_subtype == "Admin" && ctx?.sophos?.xg?.status == "Successful" && ctx?.sophos?.xg?.message_id == "17507"' - set: field: event.outcome value: failure - if: 'ctx?.sophosxg?.firewall?.log_subtype == "Admin" && ctx?.sophosxg?.firewall?.status == "Failed" && ctx?.sophosxg?.firewall?.message_id == "17507"' + if: 'ctx?.sophos?.xg?.log_subtype == "Admin" && ctx?.sophos?.xg?.status == "Failed" && ctx?.sophos?.xg?.message_id == "17507"' - append: field: event.type value: - user - start - if: "['17701', '17704', '17707', '17710', '17713'].contains(ctx.sophosxg?.firewall?.message_id)" + if: "['17701', '17704', '17707', '17710', '17713'].contains(ctx.sophos?.xg?.message_id)" - append: field: event.type value: - user - end - if: "['17703', '17706', '17709', '17712', '17715'].contains(ctx.sophosxg?.firewall?.message_id)" + if: "['17703', '17706', '17709', '17712', '17715'].contains(ctx.sophos?.xg?.message_id)" - append: field: event.type value: connection - if: "['SSLVPN', 'IPSec', 'Thin Client', 'Radius SSO'].contains(ctx.sophosxg?.firewall?.auth_client)" + if: "['SSLVPN', 'IPSec', 'Thin Client', 'Radius SSO'].contains(ctx.sophos?.xg?.auth_client)" - append: field: event.category value: network - if: "['SSLVPN', 'IPSec', 'Thin Client', 'Radius SSO'].contains(ctx.sophosxg?.firewall?.auth_client)" + if: "['SSLVPN', 'IPSec', 'Thin Client', 'Radius SSO'].contains(ctx.sophos?.xg?.auth_client)" - append: field: event.category value: authentication - if: 'ctx?.sophosxg?.firewall?.log_subtype == "Authentication"' + if: 'ctx?.sophos?.xg?.log_subtype == "Authentication"' - append: field: event.type value: info - if: 'ctx?.sophosxg?.firewall?.message_id == "17819"' + if: 'ctx?.sophos?.xg?.message_id == "17819"' - append: field: event.category value: - host - malware - if: 'ctx?.sophosxg?.firewall?.message_id == "17819"' + if: 'ctx?.sophos?.xg?.message_id == "17819"' - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true if: "ctx.event?.code == null" @@ -66,26 +66,26 @@ processors: ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.dst_ip + field: sophos.xg.dst_ip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_ip != null" + if: "ctx.sophos?.xg?.dst_ip != null" - rename: - field: sophosxg.firewall.localinterfaceip + field: sophos.xg.localinterfaceip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.localinterfaceip != null" + if: "ctx.sophos?.xg?.localinterfaceip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - convert: - field: sophosxg.firewall.recv_bytes + field: sophos.xg.recv_bytes target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.recv_bytes != null" + if: "ctx.sophos?.xg?.recv_bytes != null" - set: field: server.bytes value: '{{destination.bytes}}' @@ -102,35 +102,35 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - rename: - field: sophosxg.firewall.remoteinterfaceip + field: sophos.xg.remoteinterfaceip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.remoteinterfaceip != null" + if: "ctx.sophos?.xg?.remoteinterfaceip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - rename: - field: sophosxg.firewall.src_mac + field: sophos.xg.src_mac target_field: source.mac ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_mac != null" + if: "ctx.sophos?.xg?.src_mac != null" - set: field: client.mac value: '{{source.mac}}' if: "ctx.source?.mac != null" - convert: - field: sophosxg.firewall.sent_bytes + field: sophos.xg.sent_bytes target_field: source.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.sent_bytes != null" + if: "ctx.sophos?.xg?.sent_bytes != null" - set: field: client.bytes value: '{{source.bytes}}' @@ -143,25 +143,25 @@ processors: ignore_missing: true if: "ctx.client?.bytes != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" - set: field: source.user.name - value: '{{sophosxg.firewall.name}}' - if: "ctx.sophosxg?.firewall?.name != null" + value: '{{sophos.xg.name}}' + if: "ctx.sophos?.xg?.name != null" - rename: - field: sophosxg.firewall.usergroupname + field: sophos.xg.usergroupname target_field: source.user.group.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.usergroupname != null" + if: "ctx.sophos?.xg?.usergroupname != null" ######################### ## ECS Message Mapping ## ######################### - rename: - field: sophosxg.firewall.message + field: sophos.xg.message target_field: message ignore_missing: true @@ -260,11 +260,11 @@ processors: ############# - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.recv_bytes - - sophosxg.firewall.src_port - - sophosxg.firewall.sent_bytes - - sophosxg.firewall.name + - sophos.xg.dst_port + - sophos.xg.recv_bytes + - sophos.xg.src_port + - sophos.xg.sent_bytes + - sophos.xg.name ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/firewall.yml b/x-pack/filebeat/module/sophos/xg/ingest/firewall.yml similarity index 77% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/firewall.yml rename to x-pack/filebeat/module/sophos/xg/ingest/firewall.yml index fb82e326a77..193af05b836 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/firewall.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/firewall.yml @@ -8,45 +8,45 @@ processors: value: event - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" - if: "ctx.sophosxg?.firewall?.log_subtype != null" + value: "{{sophos.xg.log_subtype}}" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.log_subtype != null" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.kind value: alert - if: '["03001", "05001", "05151", "00003", "00004"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["03001", "05001", "05151", "00003", "00004"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.category value: intrusion_detection - if: '["03001", "05001", "05151", "00003", "00004"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["03001", "05001", "05151", "00003", "00004"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.category value: network - append: field: event.type - value: + value: - start - allowed - connection - if: "['Start', 'Interim'].contains(ctx.sophosxg?.firewall?.connevent)" + if: "['Start', 'Interim'].contains(ctx.sophos?.xg?.connevent)" - append: field: event.type - value: + value: - end - allowed - connection - if: "ctx.sophosxg?.firewall?.connevent == 'Stop'" + if: "ctx.sophos?.xg?.connevent == 'Stop'" - append: field: event.type value: - denied - connection - if: "ctx.sophosxg?.firewall?.status == 'Deny'" + if: "ctx.sophos?.xg?.status == 'Deny'" - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true if: "ctx.event?.code == null" @@ -55,31 +55,31 @@ processors: ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.dst_ip + field: sophos.xg.dst_ip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_ip != null" + if: "ctx.sophos?.xg?.dst_ip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - rename: - field: sophosxg.firewall.tran_dst_ip + field: sophos.xg.tran_dst_ip target_field: destination.nat.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.tran_dst_ip != null" + if: "ctx.sophos?.xg?.tran_dst_ip != null" - rename: - field: sophosxg.firewall.destinationip + field: sophos.xg.destinationip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.destinationip !=null" + if: "ctx.sophos?.xg?.destinationip !=null" - convert: - field: sophosxg.firewall.dst_port + field: sophos.xg.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_port != null" + if: "ctx.sophos?.xg?.dst_port != null" - set: field: server.port value: '{{destination.port}}' @@ -92,12 +92,12 @@ processors: ignore_missing: true if: "ctx.server?.port != null" - convert: - field: sophosxg.firewall.tran_dst_port + field: sophos.xg.tran_dst_port target_field: destination.nat.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.tran_dst_port != null" + if: "ctx.sophos?.xg?.tran_dst_port != null" - set: field: server.nat.port value: '{{destination.nat.port}}' @@ -110,21 +110,21 @@ processors: ignore_missing: true if: "ctx.server?.nat?.port != null" - rename: - field: sophosxg.firewall.dst_mac + field: sophos.xg.dst_mac target_field: destination.mac ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_mac != null" + if: "ctx.sophos?.xg?.dst_mac != null" - set: field: server.mac value: '{{destination.mac}}' if: "ctx.destination?.mac != null" - convert: - field: sophosxg.firewall.recv_bytes + field: sophos.xg.recv_bytes target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.recv_bytes != null" + if: "ctx.sophos?.xg?.recv_bytes != null" - set: field: server.bytes value: '{{destination.bytes}}' @@ -137,12 +137,12 @@ processors: ignore_missing: true if: "ctx.server?.bytes != null" - convert: - field: sophosxg.firewall.recv_pkts + field: sophos.xg.recv_pkts target_field: destination.packets type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.recv_pkts !=null" + if: "ctx.sophos?.xg?.recv_pkts !=null" - set: field: server.packets value: '{{destination.packets}}' @@ -159,31 +159,31 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - rename: - field: sophosxg.firewall.tran_src_ip + field: sophos.xg.tran_src_ip target_field: source.nat.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.tran_src_ip != null" + if: "ctx.sophos?.xg?.tran_src_ip != null" - rename: - field: sophosxg.firewall.sourceip + field: sophos.xg.sourceip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.sourceip != null" + if: "ctx.sophos?.xg?.sourceip != null" - convert: - field: sophosxg.firewall.src_port + field: sophos.xg.src_port target_field: source.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_port != null" + if: "ctx.sophos?.xg?.src_port != null" - set: field: client.port value: '{{source.port}}' @@ -196,12 +196,12 @@ processors: ignore_missing: true if: "ctx.client?.port != null" - convert: - field: sophosxg.firewall.tran_src_port + field: sophos.xg.tran_src_port target_field: source.nat.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.tran_src_port != null" + if: "ctx.sophos?.xg?.tran_src_port != null" - set: field: client.nat.port value: '{{source.nat.port}}' @@ -212,23 +212,23 @@ processors: type: long ignore_failure: true ignore_missing: true - if: "ctx.client?.nat?.port != null" + if: "ctx.client?.nat?.port != null" - rename: - field: sophosxg.firewall.src_mac + field: sophos.xg.src_mac target_field: source.mac ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_mac != null" + if: "ctx.sophos?.xg?.src_mac != null" - set: field: client.mac value: '{{source.mac}}' if: "ctx.source?.mac != null" - convert: - field: sophosxg.firewall.sent_bytes + field: sophos.xg.sent_bytes target_field: source.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.sent_bytes != null" + if: "ctx.sophos?.xg?.sent_bytes != null" - set: field: client.bytes value: '{{source.bytes}}' @@ -241,14 +241,14 @@ processors: ignore_missing: true if: "ctx.client?.bytes != null" - trim: - field: sophosxg.firewall.sent_pkts + field: sophos.xg.sent_pkts - convert: - field: sophosxg.firewall.sent_pkts + field: sophos.xg.sent_pkts target_field: source.packets type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.sent_pkts != null" + if: "ctx.sophos?.xg?.sent_pkts != null" - set: field: client.packets value: '{{source.packets}}' @@ -261,43 +261,43 @@ processors: ignore_missing: true if: "ctx.client?.packets != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" - rename: - field: sophosxg.firewall.user_gp + field: sophos.xg.user_gp target_field: source.user.group.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_gp != null" + if: "ctx.sophos?.xg?.user_gp != null" ###################### ## ECS Rule Mapping ## ###################### - rename: - field: sophosxg.firewall.fw_rule_id + field: sophos.xg.fw_rule_id target_field: rule.id ignore_missing: true if: "ctx.rule?.id == null" - rename: - field: sophosxg.firewall.policy_type + field: sophos.xg.policy_type target_field: rule.ruleset ignore_missing: true - if: "ctx.sophosxg?.firewall?.policy_type != null" + if: "ctx.sophos?.xg?.policy_type != null" ############################# ## ECS Network/Geo Mapping ## ############################# - rename: - field: sophosxg.firewall.application + field: sophos.xg.application target_field: network.protocol ignore_missing: true - if: "ctx.sophosxg?.firewall?.application != null" + if: "ctx.sophos?.xg?.application != null" - rename: - field: sophosxg.firewall.protocol + field: sophos.xg.protocol target_field: network.transport ignore_missing: true - if: "ctx.sophosxg?.firewall?.protocol != null" + if: "ctx.sophos?.xg?.protocol != null" - geoip: field: source.ip target_field: source.geo @@ -429,14 +429,14 @@ processors: ignore_failure: true - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.tran_dst_port - - sophosxg.firewall.recv_bytes - - sophosxg.firewall.recv_pkts - - sophosxg.firewall.src_port - - sophosxg.firewall.tran_src_port - - sophosxg.firewall.sent_bytes - - sophosxg.firewall.sent_pkts + - sophos.xg.dst_port + - sophos.xg.tran_dst_port + - sophos.xg.recv_bytes + - sophos.xg.recv_pkts + - sophos.xg.src_port + - sophos.xg.tran_src_port + - sophos.xg.sent_bytes + - sophos.xg.sent_pkts ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/idp.yml b/x-pack/filebeat/module/sophos/xg/ingest/idp.yml similarity index 80% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/idp.yml rename to x-pack/filebeat/module/sophos/xg/ingest/idp.yml index dbbc4b424ae..f10f964eb13 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/idp.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/idp.yml @@ -8,49 +8,49 @@ processors: value: alert - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" - if: "ctx.sophosxg?.firewall?.log_subtype != null" + value: "{{sophos.xg.log_subtype}}" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.log_subtype != null" + if: "ctx.sophos?.xg?.log_subtype != null" - append: field: event.category value: - intrusion_detection - network - if: '["06001", "06002", "07001", "07002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["06001", "06002", "07001", "07002"].contains(ctx.sophos?.xg?.message_id)' - append: field: event.type value: - denied - connection - if: '["06001", "06002", "07001", "07002"].contains(ctx.sophosxg?.firewall?.message_id)' + if: '["06001", "06002", "07001", "07002"].contains(ctx.sophos?.xg?.message_id)' - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true - if: "ctx.sophosxg?.firewall?.log_id != null" + if: "ctx.sophos?.xg?.log_id != null" #################################### ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.dst_ip + field: sophos.xg.dst_ip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_ip != null" + if: "ctx.sophos?.xg?.dst_ip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - convert: - field: sophosxg.firewall.dst_port + field: sophos.xg.dst_port target_field: destination.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.dst_port != null" + if: "ctx.sophos?.xg?.dst_port != null" - set: field: server.port value: '{{destination.port}}' @@ -67,21 +67,21 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - convert: - field: sophosxg.firewall.src_port + field: sophos.xg.src_port target_field: source.port type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_port != null" + if: "ctx.sophos?.xg?.src_port != null" - set: field: client.port value: '{{source.port}}' @@ -94,38 +94,38 @@ processors: ignore_missing: true if: "ctx.client?.port != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" ###################### ## ECS Rule Mapping ## ###################### - rename: - field: sophosxg.firewall.signature_id + field: sophos.xg.signature_id target_field: rule.id ignore_missing: true - if: "ctx.sophosxg?.firewall?.signature_id != null" + if: "ctx.sophos?.xg?.signature_id != null" - rename: - field: sophosxg.firewall.signature_msg + field: sophos.xg.signature_msg target_field: rule.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.signature_msg != null" + if: "ctx.sophos?.xg?.signature_msg != null" - rename: - field: sophosxg.firewall.classification + field: sophos.xg.classification target_field: rule.category ignore_missing: true - if: "ctx.sophosxg?.firewall?.classification != null" + if: "ctx.sophos?.xg?.classification != null" ############################# ## ECS Network/Geo Mapping ## ############################# - rename: - field: sophosxg.firewall.protocol + field: sophos.xg.protocol target_field: network.transport ignore_missing: true - if: "ctx.sophosxg?.firewall?.protocol != null" + if: "ctx.sophos?.xg?.protocol != null" - geoip: field: source.ip target_field: source.geo @@ -229,8 +229,8 @@ processors: ignore_failure: true - remove: field: - - sophosxg.firewall.dst_port - - sophosxg.firewall.src_port + - sophos.xg.dst_port + - sophos.xg.src_port ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml similarity index 73% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/pipeline.yml rename to x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml index b3cc5ccbae1..f408b6f01cd 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml @@ -16,7 +16,7 @@ processors: field: log.original field_split: " (?=[a-z0-9\\_\\-]+=)" value_split: "=" - prefix: "sophosxg.firewall." + prefix: "sophos.xg." ignore_missing: true ignore_failure: false trim_value: "\"" @@ -24,7 +24,7 @@ processors: # Parse the date - set: field: _temp_.time - value: "{{sophosxg.firewall.date}} {{sophosxg.firewall.time}}" + value: "{{sophos.xg.date}} {{sophos.xg.time}}" - date: if: "ctx.event.timezone == null" field: _temp_.time @@ -48,9 +48,9 @@ processors: # Sets starts, end and duration when start and duration is known - script: lang: painless - if: ctx?.soposxg?.firewall?.duration != null + if: ctx?.sophos?.xg?.duration != null source: >- - ctx.event.duration = Integer.parseInt(ctx.sophosxg.firewall.duration) * 1000000000L; + ctx.event.duration = Integer.parseInt(ctx.sophos.xg.duration) * 1000000000L; ctx.event.start = ctx['@timestamp']; ZonedDateTime start = ZonedDateTime.parse(ctx.event.start); ctx.event.end = start.plus(ctx.event.duration, ChronoUnit.NANOS); @@ -64,20 +64,20 @@ processors: - "-" - "N/A" source: >- - ctx?.sophosxg?.firewall.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); + ctx?.sophos?.xg.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); ####################### ## ECS Event Mapping ## ####################### - set: field: event.module - value: sophosxg + value: sophos - set: field: event.dataset - value: sophosxg.firewall + value: sophos.xg - set: field: event.severity - value: "{{sophosxg.firewall.log_id}}" + value: "{{sophos.xg.log_id}}" - rename: field: log.original target_field: event.original @@ -91,7 +91,7 @@ processors: - gsub: field: event.severity pattern: "(.{1,5}$)" - replacement: "" + replacement: "" ##################### ## ECS Log Mapping ## @@ -142,38 +142,38 @@ processors: field: observer.type value: firewall - rename: - field: sophosxg.firewall.device_id + field: sophos.xg.device_id target_field: observer.serial_number ignore_missing: true - rename: - field: sophosxg.firewall.out_interface + field: sophos.xg.out_interface target_field: observer.egress.interface.name ignore_missing: true - rename: - field: sophosxg.firewall.in_interface + field: sophos.xg.in_interface target_field: observer.ingress.interface.name ignore_missing: true - rename: - field: sophosxg.firewall.srczonetype + field: sophos.xg.srczonetype target_field: observer.ingress.zone ignore_missing: true - rename: - field: sophosxg.firewall.dstzonetype + field: sophos.xg.dstzonetype target_field: observer.egress.zone ignore_missing: true -# extract from log_id the new field "sophosxg.firewall.message_id" +# extract from log_id the new field "sophos.xg.message_id" - set: - field: sophosxg.firewall.message_id - value: "{{sophosxg.firewall.log_id}}" + field: sophos.xg.message_id + value: "{{sophos.xg.log_id}}" ignore_empty_value: true - gsub: - field: sophosxg.firewall.message_id + field: sophos.xg.message_id pattern: "(^.{1,7})" replacement: "" ignore_failure: true - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true @@ -204,13 +204,13 @@ processors: - message - _temp_ - _conf - - sophosxg.firewall.date - - sophosxg.firewall.time - - sophosxg.firewall.duration - - sophosxg.firewall.timezone - - sophosxg.firewall.dir_disp - - sophosxg.firewall.srczone - - sophosxg.firewall.dstzone + - sophos.xg.date + - sophos.xg.time + - sophos.xg.duration + - sophos.xg.timezone + - sophos.xg.dir_disp + - sophos.xg.srczone + - sophos.xg.dstzone - syslog5424_pri ignore_missing: true @@ -219,37 +219,37 @@ processors: ############################### - pipeline: name: '{< IngestPipeline "firewall" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Firewall'" + if: "ctx.sophos?.xg?.log_type == 'Firewall'" - pipeline: name: '{< IngestPipeline "idp" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'IDP'" + if: "ctx.sophos?.xg?.log_type == 'IDP'" - pipeline: name: '{< IngestPipeline "atp" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'ATP'" + if: "ctx.sophos?.xg?.log_type == 'ATP'" - pipeline: name: '{< IngestPipeline "antivirus" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Anti-Virus'" + if: "ctx.sophos?.xg?.log_type == 'Anti-Virus'" - pipeline: name: '{< IngestPipeline "sandstorm" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Sandbox'" + if: "ctx.sophos?.xg?.log_type == 'Sandbox'" - pipeline: name: '{< IngestPipeline "cfilter" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Content Filtering'" + if: "ctx.sophos?.xg?.log_type == 'Content Filtering'" - pipeline: name: '{< IngestPipeline "event" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Event'" + if: "ctx.sophos?.xg?.log_type == 'Event'" - pipeline: name: '{< IngestPipeline "waf" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'WAF'" + if: "ctx.sophos?.xg?.log_type == 'WAF'" - pipeline: name: '{< IngestPipeline "antispam" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Anti-Spam'" + if: "ctx.sophos?.xg?.log_type == 'Anti-Spam'" - pipeline: name: '{< IngestPipeline "systemhealth" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'System Health'" + if: "ctx.sophos?.xg?.log_type == 'System Health'" - pipeline: name: '{< IngestPipeline "wifi" >}' - if: "ctx.sophosxg?.firewall?.log_type == 'Wireless Protection'" + if: "ctx.sophos?.xg?.log_type == 'Wireless Protection'" on_failure: - set: diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/sandstorm.yml b/x-pack/filebeat/module/sophos/xg/ingest/sandstorm.yml similarity index 62% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/sandstorm.yml rename to x-pack/filebeat/module/sophos/xg/ingest/sandstorm.yml index b92da564ae1..dce06fd1776 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/sandstorm.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/sandstorm.yml @@ -8,50 +8,50 @@ processors: value: event - set: field: event.action - value: "{{sophosxg.firewall.log_subtype}}" - if: "ctx.sophosxg?.firewall?.log_subtype != null" + value: "{{sophos.xg.log_subtype}}" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.log_subtype != null" + if: "ctx.sophos?.xg?.log_subtype != null" - set: field: event.kind value: alert - if: 'ctx.sophosxg?.firewall?.log_subtype == "Denied"' + if: 'ctx.sophos?.xg?.log_subtype == "Denied"' - append: field: event.category value: - malware - network - if: 'ctx.sophosxg?.firewall?.log_subtype == "Denied"' + if: 'ctx.sophos?.xg?.log_subtype == "Denied"' - append: field: event.category value: network - if: "ctx.sophosxg?.firewall?.log_subtype != 'Denied'" + if: "ctx.sophos?.xg?.log_subtype != 'Denied'" - append: field: event.type value: allowed - if: "['Allowed'].contains(ctx.sophosxg?.firewall?.log_subtype)" + if: "['Allowed'].contains(ctx.sophos?.xg?.log_subtype)" - append: field: event.type - value: + value: - start - connection - if: "['pending'].contains(ctx.sophosxg?.firewall?.reason)" + if: "['pending'].contains(ctx.sophos?.xg?.reason)" - append: field: event.type - value: + value: - end - connection - if: "ctx.sophosxg?.firewall?.reason == 'eligible'" + if: "ctx.sophos?.xg?.reason == 'eligible'" - append: field: event.type value: - denied - connection - if: "ctx.sophosxg?.firewall?.log_subtype == 'Denied'" + if: "ctx.sophos?.xg?.log_subtype == 'Denied'" - rename: - field: sophosxg.firewall.log_id + field: sophos.xg.log_id target_field: event.code ignore_missing: true if: "ctx.event?.code == null" @@ -60,45 +60,45 @@ processors: ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.src_ip + field: sophos.xg.src_ip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.src_ip != null" + if: "ctx.sophos?.xg?.src_ip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" ###################### ## ECS File Mapping ## ###################### - rename: - field: sophoxg.firewall.filename + field: sophos.xg.filename target_field: file.name ignore_missing: true - if: "ctx.sophoxg?.firewall?.filename != null" + if: "ctx.sopho?.xg?.filename != null" - convert: - field: sophosxg.firewall.filesize + field: sophos.xg.filesize target_field: file.size type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.filesize != null" + if: "ctx.sophos?.xg?.filesize != null" - rename: - field: sophosxg.firewall.filetype + field: sophos.xg.filetype target_field: file.mime_type ignore_missing: true - if: "ctx.sophosxg?.firewall?.filetype != null" + if: "ctx.sophos?.xg?.filetype != null" - rename: - field: sophosxg.firewall.sha1sum + field: sophos.xg.sha1sum target_field: file.hash.sha1 ignore_missing: true - if: "ctx.sophosxg?.firewall?.sha1sum != null" + if: "ctx.sophos?.xg?.sha1sum != null" ######################### ## ECS Related Mapping ## @@ -117,7 +117,7 @@ processors: if: "ctx.file?.hash?.sha1 != null" - remove: field: - - sophosxg.firewall.filesize + - sophos.xg.filesize ignore_missing: true ############# ## Cleanup ## @@ -125,4 +125,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/waf.yml b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml similarity index 74% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/waf.yml rename to x-pack/filebeat/module/sophos/xg/ingest/waf.yml index 35424bd3377..3cbf1383467 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/waf.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml @@ -9,90 +9,90 @@ processors: - set: field: event.action value: allowed - if: 'ctx.sophosxg?.firewall?.reason == "-"' + if: 'ctx.sophos?.xg?.reason == "-"' - set: field: event.action value: denied - if: 'ctx.sophosxg?.firewall?.reason != "-"' + if: 'ctx.sophos?.xg?.reason != "-"' - set: field: event.outcome value: success - if: "ctx.sophosxg?.firewall?.reason != null" + if: "ctx.sophos?.xg?.reason != null" - set: field: event.kind value: alert - if: 'ctx.sophosxg?.firewall?.reason != "-"' + if: 'ctx.sophos?.xg?.reason != "-"' - append: field: event.category value: - malware - network - if: 'ctx.sophosxg?.firewall?.reason == "Antivirus"' + if: 'ctx.sophos?.xg?.reason == "Antivirus"' - append: field: event.category value: - intrusion_detection - network - if: "ctx.sophosxg?.firewall?.reason != 'Antivirus' && ctx.sophosxg?.firewall?.reason != '-'" + if: "ctx.sophos?.xg?.reason != 'Antivirus' && ctx.sophos?.xg?.reason != '-'" - append: field: event.type value: - allowed - connection - if: 'ctx.sophosxg?.firewall?.reason == "-"' + if: 'ctx.sophos?.xg?.reason == "-"' - append: field: event.type value: - denied - connection - if: 'ctx.sophosxg?.firewall?.reason != "-"' + if: 'ctx.sophos?.xg?.reason != "-"' #################################### ## ECS Server/Destination Mapping ## #################################### - rename: - field: sophosxg.firewall.localip + field: sophos.xg.localip target_field: destination.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.localip != null" + if: "ctx.sophos?.xg?.localip != null" - set: field: server.ip value: '{{destination.ip}}' if: "ctx.destination?.ip != null" - convert: - field: sophosxg.firewall.bytessent + field: sophos.xg.bytessent target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.bytessent != null" + if: "ctx.sophos?.xg?.bytessent != null" - convert: - field: sophosxg.firewall.bytessent + field: sophos.xg.bytessent target_field: server.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.bytessent != null" + if: "ctx.sophos?.xg?.bytessent != null" ############################### ## ECS Client/Source Mapping ## ############################### - rename: - field: sophosxg.firewall.sourceip + field: sophos.xg.sourceip target_field: source.ip ignore_missing: true - if: "ctx.sophosxg?.firewall?.sourceip != null" + if: "ctx.sophos?.xg?.sourceip != null" - set: field: client.ip value: '{{source.ip}}' if: "ctx.source?.ip != null" - convert: - field: sophosxg.firewall.bytesrcv + field: sophos.xg.bytesrcv target_field: source.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.bytesrcv != null" + if: "ctx.sophos?.xg?.bytesrcv != null" - set: field: client.bytes value: '{{source.bytes}}' @@ -105,60 +105,60 @@ processors: ignore_missing: true if: "ctx.client?.bytes != null" - rename: - field: sophosxg.firewall.user_name + field: sophos.xg.user_name target_field: source.user.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_name != null" + if: "ctx.sophos?.xg?.user_name != null" - rename: - field: sophosxg.firewall.user_gp + field: sophos.xg.user_gp target_field: source.user.group.name ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_gp != null" + if: "ctx.sophos?.xg?.user_gp != null" ##################### ## ECS URL Mapping ## ##################### - rename: - field: sophosxg.firewall.url + field: sophos.xg.url target_field: url.full ignore_missing: true - if: "ctx.sophosxg?.firewall?.url != null" + if: "ctx.sophos?.xg?.url != null" - rename: - field: sophosxg.firewall.domain + field: sophos.xg.domain target_field: url.domain ignore_missing: true - if: "ctx.sophosxg?.firewall?.domain != null" + if: "ctx.sophos?.xg?.domain != null" ############################ ## ECS User Agent Mapping ## ############################ - rename: - field: sophosxg.firewall.referer + field: sophos.xg.referer target_field: http.request.referrer ignore_missing: true - if: "ctx.sophosxg?.firewall?.referer != null" + if: "ctx.sophos?.xg?.referer != null" - convert: - field: sophosxg.firewall.httpstatus + field: sophos.xg.httpstatus target_field: destination.bytes type: long ignore_failure: true ignore_missing: true - if: "ctx.sophosxg?.firewall?.httpstatus != null" + if: "ctx.sophos?.xg?.httpstatus != null" - rename: - field: sophosxg.firewall.method + field: sophos.xg.method target_field: http.request.method ignore_missing: true - if: "ctx.sophosxg?.firewall?.method != null" + if: "ctx.sophos?.xg?.method != null" - rename: - field: sophosxg.firewall.ws_protocol + field: sophos.xg.ws_protocol target_field: http.version ignore_missing: true - if: "ctx.sophosxg?.firewall?.ws_protocol != null" + if: "ctx.sophos?.xg?.ws_protocol != null" - rename: - field: sophosxg.firewall.useragent + field: sophos.xg.useragent target_field: user_agent.original ignore_missing: true - if: "ctx.sophosxg?.firewall?.useragent != null" + if: "ctx.sophos?.xg?.useragent != null" ############################# ## ECS Network/Geo Mapping ## @@ -265,11 +265,11 @@ processors: ############# - remove: field: - - sophosxg.firewall.bytesrcv - - sophosxg.firewall.bytessent - - sophosxg.firewall.httpstatus + - sophos.xg.bytesrcv + - sophos.xg.bytessent + - sophos.xg.httpstatus ignore_missing: true on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/wifi.yml b/x-pack/filebeat/module/sophos/xg/ingest/wifi.yml similarity index 84% rename from x-pack/filebeat/module/sophosxg/firewall/ingest/wifi.yml rename to x-pack/filebeat/module/sophos/xg/ingest/wifi.yml index 009c7171849..a5ddc7859bb 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/wifi.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/wifi.yml @@ -9,7 +9,7 @@ processors: - set: field: event.outcome value: success - if: 'ctx?.sophosxg?.firewall?.log_type == "Wireless Protection"' + if: 'ctx?.sophos?.xg?.log_type == "Wireless Protection"' ############# ## Cleanup ## diff --git a/x-pack/filebeat/module/sophosxg/firewall/manifest.yml b/x-pack/filebeat/module/sophos/xg/manifest.yml similarity index 88% rename from x-pack/filebeat/module/sophosxg/firewall/manifest.yml rename to x-pack/filebeat/module/sophos/xg/manifest.yml index dd05b5c4982..5bf74158cee 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/manifest.yml +++ b/x-pack/filebeat/module/sophos/xg/manifest.yml @@ -4,7 +4,7 @@ var: - name: syslog_host default: localhost - name: tags - default: [sophosxg-firewall, forwarded] + default: [sophos-xg, forwarded] - name: syslog_port default: 9005 - name: input @@ -29,10 +29,9 @@ ingest_pipeline: - ingest/event.yml - ingest/waf.yml - ingest/antispam.yml - - ingest/systemhealth.yml - ingest/wifi.yml -input: config/firewall.yml +input: config/config.yml requires.processors: - name: geoip diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/anti-spam.log b/x-pack/filebeat/module/sophos/xg/test/anti-spam.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/anti-spam.log rename to x-pack/filebeat/module/sophos/xg/test/anti-spam.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/anti-spam.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-spam.log-expected.json similarity index 68% rename from x-pack/filebeat/module/sophosxg/firewall/test/anti-spam.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/anti-spam.log-expected.json index 6c5a0d087fb..90a40d0b095 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/anti-spam.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/anti-spam.log-expected.json @@ -11,9 +11,9 @@ "network" ], "event.code": "041101618035", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:48 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=041101618035 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"firewall@firewallgate.com\" to_email_address=\"Sysadmin@elasticuser.com\" email_subject=\"*ALERT* Sophos XG Firewall\" mailid=\"qkW2Y6-LxBk6U-vH-1590055245\" mailsize=19728 spamaction=\"QUEUED\" reason=\"Email has been accepted by Device and queued for scanning.\" src_domainname=\"elasticuser.com\" dst_domainname=\"\" src_ip=\"\" src_country_code=\"\" dst_ip=\"\" dst_country_code=\"\" protocol=\"TCP\" src_port=0 dst_port=0 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"", "event.outcome": "success", "event.severity": "6", @@ -22,7 +22,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -34,28 +34,28 @@ "observer.vendor": "Sophos", "server.bytes": 0, "server.port": 0, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "None", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.email_subject": "*ALERT* Sophos XG Firewall", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "qkW2Y6-LxBk6U-vH-1590055245", - "sophosxg.firewall.mailsize": "19728", - "sophosxg.firewall.message_id": "18035", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.quarantine_reason": "Other", - "sophosxg.firewall.reason": "Email has been accepted by Device and queued for scanning.", - "sophosxg.firewall.spamaction": "QUEUED", + "service.type": "sophos", + "sophos.xg.av_policy_name": "None", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.email_subject": "*ALERT* Sophos XG Firewall", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "qkW2Y6-LxBk6U-vH-1590055245", + "sophos.xg.mailsize": "19728", + "sophos.xg.message_id": "18035", + "sophos.xg.priority": "Information", + "sophos.xg.quarantine_reason": "Other", + "sophos.xg.reason": "Email has been accepted by Device and queued for scanning.", + "sophos.xg.spamaction": "QUEUED", "source.bytes": 0, "source.domain": "elasticuser.com", "source.port": 0, "source.user.email": "firewall@firewallgate.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -82,9 +82,9 @@ "network" ], "event.code": "041105613003", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:49 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=041105613003 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Clean\" status=\"\" priority=Information fw_rule_id=22 user_name=\"\" av_policy_name=\"Default\" from_email_address=\"telekommunikation@constant-big.email\" to_email_address=\"info@pelasticuser.com\" email_subject=\"Telefonservice statt Anrufbeantworter\" mailid=\"\" mailsize=1032152 spamaction=\"Prefix Subject\" reason=\"Sender IP address is blacklisted.\" src_domainname=\"ELTOBGI.COM\" dst_domainname=\"\" src_ip=77.72.3.56 src_country_code=GBR dst_ip=185.8.209.194 dst_country_code=DEU protocol=\"TCP\" src_port=55002 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"RBL\"", "event.outcome": "success", "event.severity": "4", @@ -265,7 +265,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "warning", @@ -278,24 +278,24 @@ "server.bytes": 0, "server.ip": "185.8.209.194", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "rule3", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "DEU", - "sophosxg.firewall.email_subject": "09F1A19017 - 65T BP LNG Hybrid - TS-V-061-01 - HVAC Package - RFQ - BCD - 27-May-20", - "sophosxg.firewall.fw_rule_id": "22", - "sophosxg.firewall.log_component": "SMTPS", - "sophosxg.firewall.log_subtype": "Probable Spam", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "<20200518070235.C1623996C64F9957@ELTOBGI.COM>", - "sophosxg.firewall.mailsize": "1032152", - "sophosxg.firewall.message_id": "13004", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.quarantine_reason": "RBL", - "sophosxg.firewall.reason": "Sender IP address is blacklisted.", - "sophosxg.firewall.spamaction": "Prefix Subject", - "sophosxg.firewall.src_country_code": "GBR", + "service.type": "sophos", + "sophos.xg.av_policy_name": "rule3", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "DEU", + "sophos.xg.email_subject": "09F1A19017 - 65T BP LNG Hybrid - TS-V-061-01 - HVAC Package - RFQ - BCD - 27-May-20", + "sophos.xg.fw_rule_id": "22", + "sophos.xg.log_component": "SMTPS", + "sophos.xg.log_subtype": "Probable Spam", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "<20200518070235.C1623996C64F9957@ELTOBGI.COM>", + "sophos.xg.mailsize": "1032152", + "sophos.xg.message_id": "13004", + "sophos.xg.priority": "Warning", + "sophos.xg.quarantine_reason": "RBL", + "sophos.xg.reason": "Sender IP address is blacklisted.", + "sophos.xg.spamaction": "Prefix Subject", + "sophos.xg.src_country_code": "GBR", "source.as.number": 12488, "source.as.organization.name": "Krystal Hosting Ltd", "source.bytes": 0, @@ -308,7 +308,7 @@ "source.port": 55002, "source.user.email": "SHERIF.TOBGI@ELTOBGI.COM", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -327,9 +327,9 @@ "network" ], "event.code": "041113413005", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=18:34:41 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=041113413005 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Outbound Spam\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"gaurav\" av_policy_name=\"Gaurav123\" from_email_address=\"gaurav1@iview.com\" to_email_address=\" gaurav2@iview.com\" email_subject=\"RPD Spam Test: Spam\" mailid=\"\" mailsize=405 spamaction=\"Accept\" reason=\"\" src_domainname=\" iview.com\" dst_domainname=\"\" src_ip=10.198.47.71 src_country_code=R1 dst_ip=10.198.233.61 dst_country_code=R1 protocol=\"TCP\" src_port=22420 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Spam\"", "event.outcome": "success", "event.severity": "4", @@ -339,7 +339,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -352,31 +352,31 @@ "server.bytes": 0, "server.ip": "10.198.233.61", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "Gaurav123", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.email_subject": "RPD Spam Test: Spam", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Outbound Spam", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "", - "sophosxg.firewall.mailsize": "405", - "sophosxg.firewall.message_id": "13005", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.quarantine_reason": "Spam", - "sophosxg.firewall.spamaction": "Accept", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.user_name": "gaurav", + "service.type": "sophos", + "sophos.xg.av_policy_name": "Gaurav123", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.email_subject": "RPD Spam Test: Spam", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Outbound Spam", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "", + "sophos.xg.mailsize": "405", + "sophos.xg.message_id": "13005", + "sophos.xg.priority": "Warning", + "sophos.xg.quarantine_reason": "Spam", + "sophos.xg.spamaction": "Accept", + "sophos.xg.src_country_code": "R1", + "sophos.xg.user_name": "gaurav", "source.bytes": 0, "source.domain": " iview.com", "source.ip": "10.198.47.71", "source.port": 22420, "source.user.email": "gaurav1@iview.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -395,9 +395,9 @@ "network" ], "event.code": "041114413006", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=11:10:11 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=041114413006 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Outbound Probable Spam\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"\" av_policy_name=\"rule 8\" from_email_address=\"pankhil@postman.local\" to_email_address=\"pankhil1@Postman.local\" email_subject=\"RPD Spam test: Bulk\" mailid=\"\" mailsize=439 spamaction=\"Drop\" reason=\"Mail detected as OUTBOUND PROBABLE SPAM.\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol=\"TCP\" src_port=58043 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Spam\"", "event.outcome": "success", "event.severity": "4", @@ -407,7 +407,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -420,31 +420,31 @@ "server.bytes": 0, "server.ip": "10.198.234.240", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "rule 8", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.email_subject": "RPD Spam test: Bulk", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Outbound Probable Spam", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "", - "sophosxg.firewall.mailsize": "439", - "sophosxg.firewall.message_id": "13006", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.quarantine_reason": "Spam", - "sophosxg.firewall.reason": "Mail detected as OUTBOUND PROBABLE SPAM.", - "sophosxg.firewall.spamaction": "Drop", - "sophosxg.firewall.src_country_code": "R1", + "service.type": "sophos", + "sophos.xg.av_policy_name": "rule 8", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.email_subject": "RPD Spam test: Bulk", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Outbound Probable Spam", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "", + "sophos.xg.mailsize": "439", + "sophos.xg.message_id": "13006", + "sophos.xg.priority": "Warning", + "sophos.xg.quarantine_reason": "Spam", + "sophos.xg.reason": "Mail detected as OUTBOUND PROBABLE SPAM.", + "sophos.xg.spamaction": "Drop", + "sophos.xg.src_country_code": "R1", "source.bytes": 0, "source.domain": "postman.local", "source.ip": "10.198.16.121", "source.port": 58043, "source.user.email": "pankhil@postman.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -463,9 +463,9 @@ "network" ], "event.code": "041121613009", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=12:50:07 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=041121613009 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"DLP\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" av_policy_name=\"postman\" from_email_address=\"pankhil@postman.local\" to_email_address=\"pankhil1@Postman. local\" email_subject=\"Fwd: TESt\" mailid=\"c0000002-1528269606\" mailsize=5041 spamaction=\"DROP\" reason=\"Email containing confidential data detected. Relevant Data Protection Policy applied.\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.17.121 dst_country_code=R1 protocol=\"TCP\" src_port=60134 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"DLP\"", "event.outcome": "success", "event.severity": "6", @@ -475,7 +475,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -488,31 +488,31 @@ "server.bytes": 0, "server.ip": "10.198.17.121", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "postman", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.email_subject": "Fwd: TESt", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "DLP", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "c0000002-1528269606", - "sophosxg.firewall.mailsize": "5041", - "sophosxg.firewall.message_id": "13009", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.quarantine_reason": "DLP", - "sophosxg.firewall.reason": "Email containing confidential data detected. Relevant Data Protection Policy applied.", - "sophosxg.firewall.spamaction": "DROP", - "sophosxg.firewall.src_country_code": "R1", + "service.type": "sophos", + "sophos.xg.av_policy_name": "postman", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.email_subject": "Fwd: TESt", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "DLP", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "c0000002-1528269606", + "sophos.xg.mailsize": "5041", + "sophos.xg.message_id": "13009", + "sophos.xg.priority": "Information", + "sophos.xg.quarantine_reason": "DLP", + "sophos.xg.reason": "Email containing confidential data detected. Relevant Data Protection Policy applied.", + "sophos.xg.spamaction": "DROP", + "sophos.xg.src_country_code": "R1", "source.bytes": 0, "source.domain": "postman.local", "source.ip": "10.198.16.121", "source.port": 60134, "source.user.email": "pankhil@postman.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -530,9 +530,9 @@ "network" ], "event.code": "041122613010", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=12:51:34 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=041122613010 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"SPX\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"pankhil@postman.local\" to_email_address=\"pankhil1@Postman.local\" email_subject=\"[secure:pankhil]\" mailid=\"c0000003-1528269693\" mailsize=442 spamaction=\"Accept\" reason=\"SPX Template of type Specified by Sender successfully applied on Email.\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.16.204 dst_country_code=R1 protocol=\"TCP\" src_port=60298 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"", "event.outcome": "success", "event.severity": "6", @@ -541,7 +541,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -554,31 +554,31 @@ "server.bytes": 0, "server.ip": "10.198.16.204", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "None", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.email_subject": "[secure:pankhil]", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "SPX", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "c0000003-1528269693", - "sophosxg.firewall.mailsize": "442", - "sophosxg.firewall.message_id": "13010", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.quarantine_reason": "Other", - "sophosxg.firewall.reason": "SPX Template of type Specified by Sender successfully applied on Email.", - "sophosxg.firewall.spamaction": "Accept", - "sophosxg.firewall.src_country_code": "R1", + "service.type": "sophos", + "sophos.xg.av_policy_name": "None", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.email_subject": "[secure:pankhil]", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "SPX", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "c0000003-1528269693", + "sophos.xg.mailsize": "442", + "sophos.xg.message_id": "13010", + "sophos.xg.priority": "Information", + "sophos.xg.quarantine_reason": "Other", + "sophos.xg.reason": "SPX Template of type Specified by Sender successfully applied on Email.", + "sophos.xg.spamaction": "Accept", + "sophos.xg.src_country_code": "R1", "source.bytes": 0, "source.domain": "postman.local", "source.ip": "10.198.16.121", "source.port": 60298, "source.user.email": "pankhil@postman.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -596,9 +596,9 @@ "network" ], "event.code": "041123413012", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=12:53:39 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=041123413012 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Dos\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"\" to_email_address=\"\" email_subject=\"\" mailid=\"\" mailsize=0 spamaction=\"TMPREJECT\" reason=\"SMTP DoS\" src_domainname=\"\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.17.121 dst_country_code=R1 protocol=\"TCP\" src_port=60392 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"", "event.outcome": "success", "event.severity": "4", @@ -608,7 +608,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -621,27 +621,27 @@ "server.bytes": 0, "server.ip": "10.198.17.121", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "None", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Dos", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailsize": "0", - "sophosxg.firewall.message_id": "13012", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.quarantine_reason": "Other", - "sophosxg.firewall.reason": "SMTP DoS", - "sophosxg.firewall.spamaction": "TMPREJECT", - "sophosxg.firewall.src_country_code": "R1", + "service.type": "sophos", + "sophos.xg.av_policy_name": "None", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Dos", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailsize": "0", + "sophos.xg.message_id": "13012", + "sophos.xg.priority": "Warning", + "sophos.xg.quarantine_reason": "Other", + "sophos.xg.reason": "SMTP DoS", + "sophos.xg.spamaction": "TMPREJECT", + "sophos.xg.src_country_code": "R1", "source.bytes": 0, "source.ip": "10.198.16.121", "source.port": 60392, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -660,9 +660,9 @@ "network" ], "event.code": "041102413014", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=12:56:53 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=041102413014 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Denied\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"\" av_policy_name=\"postman\" from_email_address=\"pankhil1@postman.local\" to_email_address=\"pankhil@postman. local\" email_subject=\"Fwd: test sand\" mailid=\"c0000008-1528270010\" mailsize=419835 spamaction=\"DROP\" reason=\"Email is marked Malicious by Sophos Sandstorm.\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.17.121 dst_country_code=R1 protocol=\"TCP\" src_port=60608 dst_port=25 sent_bytes=0 recv_bytes=0", "event.outcome": "success", "event.severity": "4", @@ -672,7 +672,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -685,30 +685,30 @@ "server.bytes": 0, "server.ip": "10.198.17.121", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "postman", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.email_subject": "Fwd: test sand", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "c0000008-1528270010", - "sophosxg.firewall.mailsize": "419835", - "sophosxg.firewall.message_id": "13014", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.reason": "Email is marked Malicious by Sophos Sandstorm.", - "sophosxg.firewall.spamaction": "DROP", - "sophosxg.firewall.src_country_code": "R1", + "service.type": "sophos", + "sophos.xg.av_policy_name": "postman", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.email_subject": "Fwd: test sand", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "c0000008-1528270010", + "sophos.xg.mailsize": "419835", + "sophos.xg.message_id": "13014", + "sophos.xg.priority": "Warning", + "sophos.xg.reason": "Email is marked Malicious by Sophos Sandstorm.", + "sophos.xg.spamaction": "DROP", + "sophos.xg.src_country_code": "R1", "source.bytes": 0, "source.domain": "postman.local", "source.ip": "10.198.16.121", "source.port": 60608, "source.user.email": "pankhil1@postman.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -727,9 +727,9 @@ "network" ], "event.code": "041207414001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=18:31:11 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=041207414001 log_type=\"Anti-Spam\" log_component=\"POP3\" log_subtype=\"Spam\" status=\"\" priority=Warning fw_rule_id=0 user_name=\"gaurav\" av_policy_name=\"GauravPatel\" from_email_address=\"gaurav1@iview.com\" to_email_address=\"gaurav2@iview. com\" email_subject=\"RPD Spam Test: Spam\" mailid=\"<2a2dd5d4-1a30-617b-27b1-7961ad07cf07@iview.com>\" mailsize=574 spamaction=\"Accept\" reason=\"\" src_domainname=\" iview.com\" dst_domainname=\"iview.com\" src_ip=10.198.47.71 src_country_code=R1 dst_ip=10.198.233.61 dst_country_code=R1 protocol=\"TCP\" src_port=22333 dst_port=110 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"", "event.outcome": "success", "event.severity": "4", @@ -739,7 +739,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -752,32 +752,32 @@ "server.bytes": 0, "server.ip": "10.198.233.61", "server.port": 110, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "GauravPatel", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.dst_domainname": "iview.com", - "sophosxg.firewall.email_subject": "RPD Spam Test: Spam", - "sophosxg.firewall.fw_rule_id": "0", - "sophosxg.firewall.log_component": "POP3", - "sophosxg.firewall.log_subtype": "Spam", - "sophosxg.firewall.log_type": "Anti-Spam", - "sophosxg.firewall.mailid": "<2a2dd5d4-1a30-617b-27b1-7961ad07cf07@iview.com>", - "sophosxg.firewall.mailsize": "574", - "sophosxg.firewall.message_id": "14001", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.quarantine_reason": "Other", - "sophosxg.firewall.spamaction": "Accept", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.user_name": "gaurav", + "service.type": "sophos", + "sophos.xg.av_policy_name": "GauravPatel", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.dst_domainname": "iview.com", + "sophos.xg.email_subject": "RPD Spam Test: Spam", + "sophos.xg.fw_rule_id": "0", + "sophos.xg.log_component": "POP3", + "sophos.xg.log_subtype": "Spam", + "sophos.xg.log_type": "Anti-Spam", + "sophos.xg.mailid": "<2a2dd5d4-1a30-617b-27b1-7961ad07cf07@iview.com>", + "sophos.xg.mailsize": "574", + "sophos.xg.message_id": "14001", + "sophos.xg.priority": "Warning", + "sophos.xg.quarantine_reason": "Other", + "sophos.xg.spamaction": "Accept", + "sophos.xg.src_country_code": "R1", + "sophos.xg.user_name": "gaurav", "source.bytes": 0, "source.domain": " iview.com", "source.ip": "10.198.47.71", "source.port": 22333, "source.user.email": "gaurav1@iview.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/anti-virus.log b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/anti-virus.log rename to x-pack/filebeat/module/sophos/xg/test/anti-virus.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/anti-virus.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json similarity index 74% rename from x-pack/filebeat/module/sophosxg/firewall/test/anti-virus.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json index 5bf1d7401dc..a78e27fa46e 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/anti-virus.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json @@ -22,9 +22,9 @@ "network" ], "event.code": "030906208001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:33 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=030906208001 log_type=\"Anti-Virus\" log_component=\"HTTP\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=2 user_name=\"\" iap=13 av_policy_name=\"\" virus=\"Sandstorm\" url=\"http://sophostest.com/Sandstorm/SBTestFile1.pdf\" domainname=\"sophostest.com\" src_ip=172.16.34.24 src_country_code=R1 dst_ip=13.226.155.93 dst_country_code=USA protocol=\"TCP\" src_port=57695 dst_port=80 sent_bytes=550 recv_bytes=1616 user_agent=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36\" status_code=403", "event.outcome": "success", "event.severity": "2", @@ -34,7 +34,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "http.response.status_code": 403, "input.type": "log", @@ -53,23 +53,23 @@ "server.bytes": 1616, "server.ip": "13.226.155.93", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "USA", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.message_id": "08001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.virus": "Sandstorm", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "USA", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.message_id": "08001", + "sophos.xg.priority": "Critical", + "sophos.xg.src_country_code": "R1", + "sophos.xg.virus": "Sandstorm", "source.bytes": 550, "source.ip": "172.16.34.24", "source.port": 57695, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "sophostest.com", @@ -99,9 +99,9 @@ "network" ], "event.code": "030906208001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:34 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=030906208001 log_type=\"Anti-Virus\" log_component=\"HTTP\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=2 user_name=\"\" iap=13 av_policy_name=\"\" virus=\"EICAR-AV-Test\" url=\"http://sophostest.com/eicar/index.html\" domainname=\"sophostest.com\" src_ip=172.16.34.24 src_country_code=R1 dst_ip=13.226.155.18 dst_country_code=USA protocol=\"TCP\" src_port=57835 dst_port=80 sent_bytes=541 recv_bytes=553 user_agent=\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36\" status_code=403", "event.outcome": "success", "event.severity": "2", @@ -111,7 +111,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "http.response.status_code": 403, "input.type": "log", @@ -130,23 +130,23 @@ "server.bytes": 553, "server.ip": "13.226.155.18", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "USA", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.message_id": "08001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.virus": "EICAR-AV-Test", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "USA", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.message_id": "08001", + "sophos.xg.priority": "Critical", + "sophos.xg.src_country_code": "R1", + "sophos.xg.virus": "EICAR-AV-Test", "source.bytes": 541, "source.ip": "172.16.34.24", "source.port": 57835, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "sophostest.com", @@ -174,9 +174,9 @@ "network" ], "event.code": "031106210001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:35 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=031106210001 log_type=\"Anti-Virus\" log_component=\"SMTP\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=22 user_name=\"\" av_policy_name=\"default-smtp-av\" from_email_address=\"info@farasamed.com\" to_email_address=\"info@elastic-user.local\" subject=\"ZAHLUNG (PROFORMA INVOICE)\" mailid=\"<20200520004312.Horde.lEUeVf2I6PwO5K5TtMndnC7@webmail.sevengayr\" mailsize=2254721 virus=\"TR/AD.AgentTesla.eaz\" filename=\"\" quarantine=\"\" src_domainname=\"farasamed.com\" dst_domainname=\"\" src_ip=82.165.194.211 src_country_code=DEU dst_ip=186.8.209.194 dst_country_code=DEU protocol=\"TCP\" src_port=56336 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Infected\"", "event.outcome": "success", "event.severity": "2", @@ -186,7 +186,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "critical", @@ -204,22 +204,22 @@ "server.bytes": 0, "server.ip": "186.8.209.194", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "default-smtp-av", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "DEU", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.mailid": "<20200520004312.Horde.lEUeVf2I6PwO5K5TtMndnC7@webmail.sevengayr", - "sophosxg.firewall.mailsize": "2254721", - "sophosxg.firewall.message_id": "10001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.quarantine_reason": "Infected", - "sophosxg.firewall.src_country_code": "DEU", - "sophosxg.firewall.subject": "ZAHLUNG (PROFORMA INVOICE)", - "sophosxg.firewall.virus": "TR/AD.AgentTesla.eaz", + "service.type": "sophos", + "sophos.xg.av_policy_name": "default-smtp-av", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "DEU", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.mailid": "<20200520004312.Horde.lEUeVf2I6PwO5K5TtMndnC7@webmail.sevengayr", + "sophos.xg.mailsize": "2254721", + "sophos.xg.message_id": "10001", + "sophos.xg.priority": "Critical", + "sophos.xg.quarantine_reason": "Infected", + "sophos.xg.src_country_code": "DEU", + "sophos.xg.subject": "ZAHLUNG (PROFORMA INVOICE)", + "sophos.xg.virus": "TR/AD.AgentTesla.eaz", "source.as.number": 8560, "source.as.organization.name": "1&1 Ionos Se", "source.bytes": 0, @@ -231,7 +231,7 @@ "source.port": 56336, "source.user.email": "info@farasamed.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "farasamed.com" @@ -257,9 +257,9 @@ "network" ], "event.code": "031106210001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:36 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=031106210001 log_type=\"Anti-Virus\" log_component=\"SMTP\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=22 user_name=\"\" av_policy_name=\"default-smtp-av\" from_email_address=\"spedizioni@divella.it\" to_email_address=\"info@elastic-user.local\" subject=\"Re: NEW PRO-FORMA INVOICE\" mailid=\"<20200519072944.AFCA295AF2A037A6@divella.it>\" mailsize=537457 virus=\"Mal/BredoZp-B\" filename=\"\" quarantine=\"\" src_domainname=\"divella.it\" dst_domainname=\"\" src_ip=23.254.247.78 src_country_code=USA dst_ip=185.7.209.194 dst_country_code=DEU protocol=\"TCP\" src_port=54693 dst_port=25 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Infected\"", "event.outcome": "success", "event.severity": "2", @@ -269,7 +269,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "critical", @@ -287,22 +287,22 @@ "server.bytes": 0, "server.ip": "185.7.209.194", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "default-smtp-av", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "DEU", - "sophosxg.firewall.log_component": "SMTP", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.mailid": "<20200519072944.AFCA295AF2A037A6@divella.it>", - "sophosxg.firewall.mailsize": "537457", - "sophosxg.firewall.message_id": "10001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.quarantine_reason": "Infected", - "sophosxg.firewall.src_country_code": "USA", - "sophosxg.firewall.subject": "Re: NEW PRO-FORMA INVOICE", - "sophosxg.firewall.virus": "Mal/BredoZp-B", + "service.type": "sophos", + "sophos.xg.av_policy_name": "default-smtp-av", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "DEU", + "sophos.xg.log_component": "SMTP", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.mailid": "<20200519072944.AFCA295AF2A037A6@divella.it>", + "sophos.xg.mailsize": "537457", + "sophos.xg.message_id": "10001", + "sophos.xg.priority": "Critical", + "sophos.xg.quarantine_reason": "Infected", + "sophos.xg.src_country_code": "USA", + "sophos.xg.subject": "Re: NEW PRO-FORMA INVOICE", + "sophos.xg.virus": "Mal/BredoZp-B", "source.as.number": 54290, "source.as.organization.name": "Hostwinds LLC.", "source.bytes": 0, @@ -317,7 +317,7 @@ "source.port": 54693, "source.user.email": "spedizioni@divella.it", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "divella.it" @@ -337,9 +337,9 @@ "network" ], "event.code": "036106211001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=10:51:29 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=036106211001 log_type=\"Anti-Virus\" log_component=\"POPS\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"pankhil@postman.local\" to_email_address=\"pankhil@postman.local\" subject=\"EICAR\" mailid=\"\" mailsize=0 virus=\"EICAR-AV-Test\" filename=\"\" quarantine=\"\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol=\"TCP\" src_port=56653 dst_port=995 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"", "event.outcome": "success", "event.severity": "2", @@ -349,7 +349,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "critical", @@ -367,28 +367,28 @@ "server.bytes": 0, "server.ip": "10.198.234.240", "server.port": 995, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "None", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.log_component": "POPS", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.mailid": "", - "sophosxg.firewall.mailsize": "0", - "sophosxg.firewall.message_id": "11001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.quarantine_reason": "Other", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.subject": "EICAR", - "sophosxg.firewall.virus": "EICAR-AV-Test", + "service.type": "sophos", + "sophos.xg.av_policy_name": "None", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.log_component": "POPS", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.mailid": "", + "sophos.xg.mailsize": "0", + "sophos.xg.message_id": "11001", + "sophos.xg.priority": "Critical", + "sophos.xg.quarantine_reason": "Other", + "sophos.xg.src_country_code": "R1", + "sophos.xg.subject": "EICAR", + "sophos.xg.virus": "EICAR-AV-Test", "source.bytes": 0, "source.ip": "10.198.16.121", "source.port": 56653, "source.user.email": "pankhil@postman.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "postman.local" @@ -408,9 +408,9 @@ "network" ], "event.code": "036206212001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=10:58:29 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=036206212001 log_type=\"Anti-Virus\" log_component=\"IMAPS\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"pankhil@postman.local\" to_email_address=\"ganga@postman.local\" subject=\"EICAR test email\" mailid=\"<2ca37b7c-e93a-743a-99c4-a0796f0bbb79@postman.local>\" mailsize=0 virus=\"EICAR-AV-Test\" filename=\"\" quarantine=\"\" src_domainname=\"postman.local\" dst_domainname=\"\" src_ip=10.198.16.121 src_country_code=R1 dst_ip=10.198.234.240 dst_country_code=R1 protocol=\"TCP\" src_port=56632 dst_port=993 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"", "event.outcome": "success", "event.severity": "2", @@ -420,7 +420,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "critical", @@ -438,28 +438,28 @@ "server.bytes": 0, "server.ip": "10.198.234.240", "server.port": 993, - "service.type": "sophosxg", - "sophosxg.firewall.av_policy_name": "None", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.log_component": "IMAPS", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.mailid": "<2ca37b7c-e93a-743a-99c4-a0796f0bbb79@postman.local>", - "sophosxg.firewall.mailsize": "0", - "sophosxg.firewall.message_id": "12001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.quarantine_reason": "Other", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.subject": "EICAR test email", - "sophosxg.firewall.virus": "EICAR-AV-Test", + "service.type": "sophos", + "sophos.xg.av_policy_name": "None", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.log_component": "IMAPS", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.mailid": "<2ca37b7c-e93a-743a-99c4-a0796f0bbb79@postman.local>", + "sophos.xg.mailsize": "0", + "sophos.xg.message_id": "12001", + "sophos.xg.priority": "Critical", + "sophos.xg.quarantine_reason": "Other", + "sophos.xg.src_country_code": "R1", + "sophos.xg.subject": "EICAR test email", + "sophos.xg.virus": "EICAR-AV-Test", "source.bytes": 0, "source.ip": "10.198.16.121", "source.port": 56632, "source.user.email": "pankhil@postman.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "postman.local" @@ -478,9 +478,9 @@ "network" ], "event.code": "031006209001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-21 time=19:50:23 timezone=\"CEST\" device_name=\"SF01V\" device_id=SFDemo-2df0960 log_id=031006209001 log_type=\"Anti-Virus\" log_component=\"FTP\" log_subtype=\"Virus\" status=\"\" priority=Critical fw_rule_id=0 user_name=\"\" virus=\"EICAR-AV-Test\" FTP_url=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" FTP_direction=\"Upload\" filename=\" /home/ftp-user/ta_test_file_1ta-cl1-46\" file_size=0 file_path=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" ftpcommand=\"STOR\" src_ip=10.146.13.49 src_country_code=R1 dst_ip=10.8.142.181 dst_country_code=R1 protocol=\"TCP\" src_port=39910 dst_port=21 dstdomain=\"\" sent_bytes=0 recv_bytes=0", "event.outcome": "success", "event.severity": "2", @@ -493,7 +493,7 @@ "file.directory": "/var/www//home/ftp-user/ta_test_file_1ta-cl1-46", "file.name": " /home/ftp-user/ta_test_file_1ta-cl1-46", "file.size": 0, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "critical", @@ -511,23 +511,23 @@ "server.bytes": 0, "server.ip": "10.8.142.181", "server.port": 21, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SF01V", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.ftpcommand": "STOR", - "sophosxg.firewall.log_component": "FTP", - "sophosxg.firewall.log_subtype": "Virus", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.message_id": "09001", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.virus": "EICAR-AV-Test\" FTP_url=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" FTP_direction=\"Upload", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SF01V", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.ftpcommand": "STOR", + "sophos.xg.log_component": "FTP", + "sophos.xg.log_subtype": "Virus", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.message_id": "09001", + "sophos.xg.priority": "Critical", + "sophos.xg.src_country_code": "R1", + "sophos.xg.virus": "EICAR-AV-Test\" FTP_url=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" FTP_direction=\"Upload", "source.bytes": 0, "source.ip": "10.146.13.49", "source.port": 39910, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -544,9 +544,9 @@ "network" ], "event.code": "031001609002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-21 time=19:50:48 timezone=\"CEST\" device_name=\"SF01V\" device_id=SFDemo-2df0960 log_id=031001609002 log_type=\"Anti-Virus\" log_component=\"FTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" virus=\"\" FTP_url=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" FTP_direction=\"Download\" filename=\"/home/ftp-user /ta_test_file_1ta-cl1-46\" file_size=19926248 file_path=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" ftpcommand=\"RETR\" src_ip=10.146.13.49 src_country_code= dst_ip=10.8.142.181 dst_country_code= protocol=\"TCP\" src_port=39936 dst_port=21 dstdomain=\"\" sent_bytes=0 recv_bytes=19926248", "event.outcome": "success", "event.severity": "6", @@ -558,7 +558,7 @@ "file.directory": "/var/www//home/ftp-user/ta_test_file_1ta-cl1-46", "file.name": "/home/ftp-user /ta_test_file_1ta-cl1-46", "file.size": 19926248, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -576,21 +576,21 @@ "server.bytes": 19926248, "server.ip": "10.8.142.181", "server.port": 21, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SF01V", - "sophosxg.firewall.ftpcommand": "RETR", - "sophosxg.firewall.log_component": "FTP", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Anti-Virus", - "sophosxg.firewall.message_id": "09002", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.virus": " FTP_url=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" FTP_direction=\"Download", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SF01V", + "sophos.xg.ftpcommand": "RETR", + "sophos.xg.log_component": "FTP", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Anti-Virus", + "sophos.xg.message_id": "09002", + "sophos.xg.priority": "Information", + "sophos.xg.virus": " FTP_url=\"/var/www//home/ftp-user/ta_test_file_1ta-cl1-46\" FTP_direction=\"Download", "source.bytes": 0, "source.ip": "10.146.13.49", "source.port": 39936, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/atp.log b/x-pack/filebeat/module/sophos/xg/test/atp.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/atp.log rename to x-pack/filebeat/module/sophos/xg/test/atp.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/atp.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json similarity index 78% rename from x-pack/filebeat/module/sophosxg/firewall/test/atp.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json index c2eeb697b8d..7dbb6289456 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/atp.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json @@ -17,10 +17,10 @@ "network" ], "event.code": "086304418010", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.id": "C366ACFB-7A6F-4870-B359-A6CFDA8C85F7", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=18:44:31 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=086304418010 log_type=\"ATP\" log_component=\"Firewall\" log_subtype=\"Drop\" priority=Warning user_name=\"jsmith\" protocol=\"TCP\" src_port=22623 dst_port=80 sourceip=10.198.47.71 destinationip=46.161.30.47 url=46.161.30.47 threatname=C2/Generic-A eventid=C366ACFB-7A6F-4870-B359-A6CFDA8C85F7 eventtype=\"Standard\" login_user=\"\" process_user=\"\" ep_uuid= execution_path=\"\"", "event.outcome": "success", "event.severity": "4", @@ -29,7 +29,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -48,21 +48,21 @@ ], "server.ip": "46.161.30.47", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.eventtype": "Standard", - "sophosxg.firewall.log_component": "Firewall", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "ATP", - "sophosxg.firewall.message_id": "18010", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.threatname": "C2/Generic-A", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.eventtype": "Standard", + "sophos.xg.log_component": "Firewall", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "ATP", + "sophos.xg.message_id": "18010", + "sophos.xg.priority": "Warning", + "sophos.xg.threatname": "C2/Generic-A", "source.ip": "10.198.47.71", "source.port": 22623, "source.user.name": "jsmith", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.original": "46.161.30.47" @@ -88,10 +88,10 @@ "network" ], "event.code": "086504418010", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.id": "E91DAD80-BDE4-4682-B7E8-FE394B70A36C", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:34 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=086504418010 log_type=\"ATP\" log_component=\"Web\" log_subtype=\"Drop\" priority=Warning user_name=\"\" protocol=\"TCP\" src_port=57579 dst_port=80 sourceip=172.16.34.24 destinationip=13.226.155.22 url=http://sophostest.com/callhome/index.html threatname=C2/Generic-A eventid=E91DAD80-BDE4-4682-B7E8-FE394B70A36C eventtype=\"Standard\" login_user=\"\" process_user=\"\" ep_uuid=\"\" execution_path=\"\"", "event.outcome": "success", "event.severity": "4", @@ -100,7 +100,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "warning", @@ -116,20 +116,20 @@ ], "server.ip": "13.226.155.22", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.eventtype": "Standard", - "sophosxg.firewall.log_component": "Web", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "ATP", - "sophosxg.firewall.message_id": "18010", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.threatname": "C2/Generic-A", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.eventtype": "Standard", + "sophos.xg.log_component": "Web", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "ATP", + "sophos.xg.message_id": "18010", + "sophos.xg.priority": "Warning", + "sophos.xg.threatname": "C2/Generic-A", "source.ip": "172.16.34.24", "source.port": 57579, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.original": "http://sophostest.com/callhome/index.html" @@ -155,10 +155,10 @@ "network" ], "event.code": "086504418010", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.id": "34AC8531-E7C0-4368-9978-5740952EE9AB", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:35 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=086504418010 log_type=\"ATP\" log_component=\"Web\" log_subtype=\"Drop\" priority=Warning user_name=\"\" protocol=\"TCP\" src_port=57540 dst_port=80 sourceip=172.16.34.24 destinationip=13.226.155.22 url=http://sophostest.com/callhome/index.html threatname=C2/Generic-A eventid=34AC8531-E7C0-4368-9978-5740952EE9AB eventtype=\"Standard\" login_user=\"\" process_user=\"\" ep_uuid=\"\" execution_path=\"\"", "event.outcome": "success", "event.severity": "4", @@ -167,7 +167,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "warning", @@ -183,20 +183,20 @@ ], "server.ip": "13.226.155.22", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.eventtype": "Standard", - "sophosxg.firewall.log_component": "Web", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "ATP", - "sophosxg.firewall.message_id": "18010", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.threatname": "C2/Generic-A", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.eventtype": "Standard", + "sophos.xg.log_component": "Web", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "ATP", + "sophos.xg.message_id": "18010", + "sophos.xg.priority": "Warning", + "sophos.xg.threatname": "C2/Generic-A", "source.ip": "172.16.34.24", "source.port": 57540, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.original": "http://sophostest.com/callhome/index.html" @@ -219,10 +219,10 @@ "network" ], "event.code": "086320518009", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.id": "C7E26E6F-0097-4EA2-89DE-C31C40636CB2", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-05 time=08:49:00 timezone=\"BST\" device_name=\"XG310\" device_id=C30006T22TGR89B log_id=086320518009 log_type=\"ATP\" log_component=\"Firewall\" log_subtype=\"Alert\" priority=Notice user_name=\"\" protocol=\"ICMP\" src_port=0 dst_port=0 sourceip=10.198.32.89 destinationip=82.211.30.202 url=82.211.30.202 threatname=C2/Generic-A eventid=C7E26E6F-0097-4EA2-89DE-C31C40636CB2 eventtype=\"Standard\" login_user=\"\" process_user=\"\" ep_uuid= execution_path=\"\"", "event.outcome": "success", "event.severity": "5", @@ -231,7 +231,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "notification", @@ -247,20 +247,20 @@ ], "server.ip": "82.211.30.202", "server.port": 0, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG310", - "sophosxg.firewall.eventtype": "Standard", - "sophosxg.firewall.log_component": "Firewall", - "sophosxg.firewall.log_subtype": "Alert", - "sophosxg.firewall.log_type": "ATP", - "sophosxg.firewall.message_id": "18009", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.threatname": "C2/Generic-A", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG310", + "sophos.xg.eventtype": "Standard", + "sophos.xg.log_component": "Firewall", + "sophos.xg.log_subtype": "Alert", + "sophos.xg.log_type": "ATP", + "sophos.xg.message_id": "18009", + "sophos.xg.priority": "Notice", + "sophos.xg.threatname": "C2/Generic-A", "source.ip": "10.198.32.89", "source.port": 0, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.original": "82.211.30.202" diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/cfilter.log b/x-pack/filebeat/module/sophos/xg/test/cfilter.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/cfilter.log rename to x-pack/filebeat/module/sophos/xg/test/cfilter.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/cfilter.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json similarity index 73% rename from x-pack/filebeat/module/sophosxg/firewall/test/cfilter.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json index 17a26c9f3cd..a82d4550f57 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/cfilter.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json @@ -16,9 +16,9 @@ "network" ], "event.code": "050901616001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=14:03:33 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=050901616001 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=2 user_name=\"jsmith\" user_gp=\"Open Group\" iap=1 category=\"Entertainment\" category_type=\"Unproductive\" url=\"https://r8---sn-ci5gup-qxas.googlevideo.com/\" contenttype=\"\" override_token=\"\" httpresponsecode=\"\" src_ip=10.198.47.71 dst_ip=182.79.221.19 protocol=\"TCP\" src_port=9444 dst_port=443 sent_bytes=0 recv_bytes=319007 domain=r8---sn-ci5gup-qxas.googlevideo.com exceptions= activityname=\"\" reason=\"\"", "event.outcome": "success", "event.severity": "6", @@ -27,7 +27,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -46,24 +46,24 @@ ], "server.ip": "182.79.221.19", "server.port": 443, - "service.type": "sophosxg", - "sophosxg.firewall.category": "Entertainment", - "sophosxg.firewall.category_type": "Unproductive", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.iap": "1", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16001", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.category": "Entertainment", + "sophos.xg.category_type": "Unproductive", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.iap": "1", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16001", + "sophos.xg.priority": "Information", "source.ip": "10.198.47.71", "source.port": 9444, "source.user.group.name": "Open Group", "source.user.name": "jsmith", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "r8---sn-ci5gup-qxas.googlevideo.com", @@ -90,9 +90,9 @@ "network" ], "event.code": "050902616002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-02-01 time=18:20:21 timezone=\"IST\" device_name=\"SG115\" device_id=S110000E28BA631 log_id=050902616002 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Denied\" status=\"\" priority=Information fw_rule_id=1 user_name=\"\" user_gp=\"\" iap=13 category=\"Religion & Spirituality\" category_type=\"Unproductive\" url=\"http://hanuman.com/\" contenttype=\"\" override_token=\"\" httpresponsecode=\"\" src_ip=5.5.5.15 dst_ip=216.58.197.44 protocol=\"TCP\" src_port=46719 dst_port=80 sent_bytes=0 recv_bytes=0 domain=hanuman.com exceptions= activityname=\"\"", "event.outcome": "success", "event.severity": "6", @@ -102,7 +102,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -118,18 +118,18 @@ ], "server.ip": "216.58.197.44", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.category": "Religion & Spirituality", - "sophosxg.firewall.category_type": "Unproductive", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG115", - "sophosxg.firewall.fw_rule_id": "1", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16002", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.category": "Religion & Spirituality", + "sophos.xg.category_type": "Unproductive", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG115", + "sophos.xg.fw_rule_id": "1", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16002", + "sophos.xg.priority": "Information", "source.as.number": 6805, "source.as.organization.name": "Telefonica Germany", "source.geo.continent_name": "Europe", @@ -139,7 +139,7 @@ "source.ip": "5.5.5.15", "source.port": 46719, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "hanuman.com", @@ -163,9 +163,9 @@ "network" ], "event.code": "054402617051", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-02-01 time=18:13:29 timezone=\"IST\" device_name=\"SG115\" device_id=S110016E28BA631 log_id=054402617051 log_type=\"Content Filtering\" log_component=\"Application\" log_subtype=\"Denied\" priority=Information fw_rule_id=1 user_name=\"\" user_gp=\"\" application_filter_policy=8 category=\"Mobile Applications\" application_name=\"Gtalk Android\" application_risk=4 application_technology=\"Client Server\" application_category=\"Mobile Applications\" src_ip=5.5.5.15 src_country_code=DEU dst_ip=74.125.130.188 dst_country_code=USA protocol=\"TCP\" src_port=49128 dst_port=5228 sent_bytes=0 recv_bytes=0 status=\"Deny\" message=\"\"", "event.outcome": "success", "event.severity": "6", @@ -175,7 +175,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -191,24 +191,24 @@ ], "server.ip": "74.125.130.188", "server.port": 5228, - "service.type": "sophosxg", - "sophosxg.firewall.application_category": "Mobile Applications", - "sophosxg.firewall.application_filter_policy": "8", - "sophosxg.firewall.application_name": "Gtalk Android", - "sophosxg.firewall.application_risk": "4", - "sophosxg.firewall.application_technology": "Client Server", - "sophosxg.firewall.category": "Mobile Applications", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG115", - "sophosxg.firewall.dst_country_code": "USA", - "sophosxg.firewall.fw_rule_id": "1", - "sophosxg.firewall.log_component": "Application", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "17051", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.src_country_code": "DEU", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.application_category": "Mobile Applications", + "sophos.xg.application_filter_policy": "8", + "sophos.xg.application_name": "Gtalk Android", + "sophos.xg.application_risk": "4", + "sophos.xg.application_technology": "Client Server", + "sophos.xg.category": "Mobile Applications", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG115", + "sophos.xg.dst_country_code": "USA", + "sophos.xg.fw_rule_id": "1", + "sophos.xg.log_component": "Application", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "17051", + "sophos.xg.priority": "Information", + "sophos.xg.src_country_code": "DEU", + "sophos.xg.status": "Deny", "source.as.number": 6805, "source.as.organization.name": "Telefonica Germany", "source.geo.continent_name": "Europe", @@ -218,7 +218,7 @@ "source.ip": "5.5.5.15", "source.port": 49128, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -242,9 +242,9 @@ "network" ], "event.code": "050901616001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:51 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=050901616001 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=2 user_name=\"\" user_gp=\"\" iap=13 category=\"Information Technology\" category_type=\"Acceptable\" url=\"https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket\" contenttype=\"\" override_token=\"\" httpresponsecode=\"\" src_ip=172.17.34.10 dst_ip=13.79.168.201 protocol=\"TCP\" src_port=62851 dst_port=443 sent_bytes=259 recv_bytes=168 domain=his-eur1-neur1.servicebus.windows.net exceptions=\"\" activityname=\"\" reason=\"\" user_agent=\"\" status_code=\"400\" transactionid=\"\" referer=\"\" download_file_name=\"\" download_file_type=\"\" upload_file_name=\"\" upload_file_type=\"\" con_id=80042000 application=\"\" app_is_cloud=0 override_name=\"\" override_authorizer=\"\"", "event.outcome": "success", "event.severity": "6", @@ -253,7 +253,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "http.response.status_code": "400", "input.type": "log", @@ -270,24 +270,24 @@ ], "server.ip": "13.79.168.201", "server.port": 443, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.category": "Information Technology", - "sophosxg.firewall.category_type": "Acceptable", - "sophosxg.firewall.con_id": "80042000", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16001", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.category": "Information Technology", + "sophos.xg.category_type": "Acceptable", + "sophos.xg.con_id": "80042000", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16001", + "sophos.xg.priority": "Information", "source.ip": "172.17.34.10", "source.port": 62851, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "his-eur1-neur1.servicebus.windows.net", @@ -314,9 +314,9 @@ "network" ], "event.code": "050902616002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:52 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=050902616002 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Denied\" status=\"\" priority=Information fw_rule_id=51 user_name=\"\" user_gp=\"\" iap=2 category=\"IPAddress\" category_type=\"Acceptable\" url=\"https://40.90.137.127/\" contenttype=\"\" override_token=\"\" httpresponsecode=\"\" src_ip=172.16.34.15 dst_ip=40.90.137.127 protocol=\"TCP\" src_port=60471 dst_port=443 sent_bytes=0 recv_bytes=0 domain=40.90.137.127 exceptions=\"\" activityname=\"\" reason=\"\" user_agent=\"\" status_code=\"200\" transactionid=\"\" referer=\"\" download_file_name=\"\" download_file_type=\"\" upload_file_name=\"\" upload_file_type=\"\" con_id=642960832 application=\"\" app_is_cloud=0 override_name=\"\" override_authorizer=\"\"", "event.outcome": "success", "event.severity": "6", @@ -326,7 +326,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "http.response.status_code": "200", "input.type": "log", @@ -343,24 +343,24 @@ ], "server.ip": "40.90.137.127", "server.port": 443, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.category": "IPAddress", - "sophosxg.firewall.category_type": "Acceptable", - "sophosxg.firewall.con_id": "642960832", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.fw_rule_id": "51", - "sophosxg.firewall.iap": "2", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16002", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.category": "IPAddress", + "sophos.xg.category_type": "Acceptable", + "sophos.xg.con_id": "642960832", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.fw_rule_id": "51", + "sophos.xg.iap": "2", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16002", + "sophos.xg.priority": "Information", "source.ip": "172.16.34.15", "source.port": 60471, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "40.90.137.127", @@ -386,9 +386,9 @@ "network" ], "event.code": "050901616001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:53 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=050901616001 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=2 user_name=\"\" user_gp=\"\" iap=13 category=\"Information Technology\" category_type=\"Acceptable\" url=\"http://update.eset.com/eset_upd/ep7/dll/update.ver.signed\" contenttype=\"\" override_token=\"\" httpresponsecode=\"\" src_ip=172.17.34.15 dst_ip=91.228.167.133 protocol=\"TCP\" src_port=65391 dst_port=80 sent_bytes=980 recv_bytes=295 domain=update.eset.com exceptions=av,https,sandstorm activityname=\"\" reason=\"\" user_agent=\"EFSW Update (Windows; U; 64bit; BPC 7.1.12010.0; OS: 10.0.17763 SP 0.0 NT; TDB 45511; CL 1.1.1; x64s; APP efsw; PX 1; PUA 1; CD 1; RA 1; PEV 0; UNS 1; UBR 1158; HVCI 0; SHA256 1; WU 3; HWF: 01009DAA-757A-D666-EFD2-92DD0D501284; PLOC de_de; PCODE 211.0.0; \" status_code=\"304\" transactionid=\"\" referer=\"\" download_file_name=\"\" download_file_type=\"\" upload_file_name=\"\" upload_file_type=\"\" con_id=248426360 application=\"\" app_is_cloud=0 override_name=\"\" override_authorizer=\"\"", "event.outcome": "success", "event.severity": "6", @@ -397,7 +397,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "http.response.status_code": "304", "input.type": "log", @@ -414,25 +414,25 @@ ], "server.ip": "91.228.167.133", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.category": "Information Technology", - "sophosxg.firewall.category_type": "Acceptable", - "sophosxg.firewall.con_id": "248426360", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.exceptions": "av,https,sandstorm", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16001", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.category": "Information Technology", + "sophos.xg.category_type": "Acceptable", + "sophos.xg.con_id": "248426360", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.exceptions": "av,https,sandstorm", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16001", + "sophos.xg.priority": "Information", "source.ip": "172.17.34.15", "source.port": 65391, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "update.eset.com", @@ -447,14 +447,14 @@ "network" ], "event.code": "058420116010", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2016-12-02 time=18:50:20 timezone=\"GMT\" device_name=\"SF01V\" device_id=1234567890123456 log_id=058420116010 log_type=\"Content Filtering\" log_component=\"Web Content Policy\" log_subtype=\"Alert\" user=\"gi123456\" src_ip=10.108.108.49 transaction_id=\"e4a127f7-a850-477c-920e-a471b38727c1\" dictionary_name=\"complicated_Custom\" site_category=Information Technology website=\"ta-web-static-testing.qa. astaro.de\" direction=\"in\" action=\"Deny\" file_name=\"cgi_echo.pl\" context_match=\"Not\" context_prefix=\"blah blah hello \" context_suffix=\" hello blah \"", "event.outcome": "success", "event.severity": "1", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "alert", @@ -466,27 +466,27 @@ "related.ip": [ "10.108.108.49" ], - "service.type": "sophosxg", - "sophosxg.firewall.action": "Deny", - "sophosxg.firewall.context_match": "Not", - "sophosxg.firewall.context_prefix": "blah blah hello ", - "sophosxg.firewall.context_suffix": " hello blah ", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SF01V", - "sophosxg.firewall.dictionary_name": "complicated_Custom", - "sophosxg.firewall.direction": "in", - "sophosxg.firewall.file_name": "cgi_echo.pl", - "sophosxg.firewall.log_component": "Web Content Policy", - "sophosxg.firewall.log_subtype": "Alert", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16010", - "sophosxg.firewall.site_category": "Information Technology", - "sophosxg.firewall.transaction_id": "e4a127f7-a850-477c-920e-a471b38727c1", - "sophosxg.firewall.user": "gi123456", - "sophosxg.firewall.website": "ta-web-static-testing.qa. astaro.de", + "service.type": "sophos", + "sophos.xg.action": "Deny", + "sophos.xg.context_match": "Not", + "sophos.xg.context_prefix": "blah blah hello ", + "sophos.xg.context_suffix": " hello blah ", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SF01V", + "sophos.xg.dictionary_name": "complicated_Custom", + "sophos.xg.direction": "in", + "sophos.xg.file_name": "cgi_echo.pl", + "sophos.xg.log_component": "Web Content Policy", + "sophos.xg.log_subtype": "Alert", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16010", + "sophos.xg.site_category": "Information Technology", + "sophos.xg.transaction_id": "e4a127f7-a850-477c-920e-a471b38727c1", + "sophos.xg.user": "gi123456", + "sophos.xg.website": "ta-web-static-testing.qa. astaro.de", "source.ip": "10.108.108.49", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -507,9 +507,9 @@ "network" ], "event.code": "050927616005", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2016-12-02 time=18:50:20 timezone=\"GMT\" device_name=\"SFVUNL\" device_id=C01001K234RXPA1 log_id=050927616005 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Warned\" status=\"\" priority=Information fw_rule_id=2 user_name=\"rich\" user_gp=\"Clientless Open Group\" iap=13 category=\"Search Engines\" category_type=\"Acceptable\" url=\"http://www.google.com/\" contenttype=\"\" override_token=\"\" httpresponsecode=\"\" src_ip=192.168.73.220 dst_ip=64.233.189.147 protocol=\"TCP\" src_port=37832 dst_port=80 sent_bytes=0 recv_bytes=0 domain=www.google.com exceptions= activityname=\" Search\" reason=\"\"", "event.outcome": "success", "event.severity": "6", @@ -518,7 +518,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -537,25 +537,25 @@ ], "server.ip": "64.233.189.147", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.activityname": " Search", - "sophosxg.firewall.category": "Search Engines", - "sophosxg.firewall.category_type": "Acceptable", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SFVUNL", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Warned", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16005", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.activityname": " Search", + "sophos.xg.category": "Search Engines", + "sophos.xg.category_type": "Acceptable", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SFVUNL", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Warned", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16005", + "sophos.xg.priority": "Information", "source.ip": "192.168.73.220", "source.port": 37832, "source.user.group.name": "Clientless Open Group", "source.user.name": "rich", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "www.google.com", @@ -578,9 +578,9 @@ "network" ], "event.code": "050901616006", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2016-12-02 time=18:50:22 timezone=\"GMT\" device_name=\"SFVUNL\" device_id=C01001K234RXPA1 log_id=050901616006 log_type=\"Content Filtering\" log_component=\"HTTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=2 user_name=\"rich\" user_gp=\"Clientless Open Group\" iap=13 category=\"Search Engines\" category_type=\"Acceptable\" url=\"http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw\" contenttype=\"text/html\" override_token=\"\" httpresponsecode=\"\" src_ip=192.168.73.220 dst_ip=64.233.188.94 protocol=\"TCP\" src_port=46322 dst_port=80 sent_bytes=0 recv_bytes=619 domain=www.google.ca exceptions= activityname=\"Search\" reason=\"not eligible\"", "event.outcome": "success", "event.severity": "6", @@ -589,7 +589,7 @@ "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -608,27 +608,27 @@ ], "server.ip": "64.233.188.94", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.activityname": "Search", - "sophosxg.firewall.category": "Search Engines", - "sophosxg.firewall.category_type": "Acceptable", - "sophosxg.firewall.contenttype": "text/html", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SFVUNL", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.iap": "13", - "sophosxg.firewall.log_component": "HTTP", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Content Filtering", - "sophosxg.firewall.message_id": "16006", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "not eligible", + "service.type": "sophos", + "sophos.xg.activityname": "Search", + "sophos.xg.category": "Search Engines", + "sophos.xg.category_type": "Acceptable", + "sophos.xg.contenttype": "text/html", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SFVUNL", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.iap": "13", + "sophos.xg.log_component": "HTTP", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Content Filtering", + "sophos.xg.message_id": "16006", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "not eligible", "source.ip": "192.168.73.220", "source.port": 46322, "source.user.group.name": "Clientless Open Group", "source.user.name": "rich", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.domain": "www.google.ca", diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/event.log b/x-pack/filebeat/module/sophos/xg/test/event.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/event.log rename to x-pack/filebeat/module/sophos/xg/test/event.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/event.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/event.log-expected.json similarity index 68% rename from x-pack/filebeat/module/sophosxg/firewall/test/event.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/event.log-expected.json index 85d4233908d..d14c2bb9924 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/event.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/event.log-expected.json @@ -6,9 +6,9 @@ "authentication" ], "event.code": "062910617701", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:57 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=062910617701 log_type=\"Event\" log_component=\"Firewall Authentication\" log_subtype=\"Authentication\" status=\"Successful\" priority=Information user_name=\"elastic.user@elastic.test.com\" usergroupname=\"Open Group\" auth_client=\"CTA\" auth_mechanism=\"AD\" reason=\"\" src_ip=172.17.35.116 message=\"User elastic.user@elastic.test.com of group Open Group logged in successfully to Firewall through AD authentication mechanism from 172.17.35.116\" name=\"elastic.user@elastic.test.com\" src_mac=", "event.outcome": "success", "event.severity": "6", @@ -17,7 +17,7 @@ "user", "start" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -33,22 +33,22 @@ "related.user": [ "elastic.user@elastic.test.com" ], - "service.type": "sophosxg", - "sophosxg.firewall.auth_client": "CTA", - "sophosxg.firewall.auth_mechanism": "AD", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "Firewall Authentication", - "sophosxg.firewall.log_subtype": "Authentication", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17701", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Successful", + "service.type": "sophos", + "sophos.xg.auth_client": "CTA", + "sophos.xg.auth_mechanism": "AD", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "Firewall Authentication", + "sophos.xg.log_subtype": "Authentication", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17701", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Successful", "source.ip": "172.17.35.116", "source.user.group.name": "Open Group", "source.user.name": "elastic.user@elastic.test.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -63,13 +63,13 @@ "destination.geo.location.lon": -97.822, "destination.ip": "214.167.51.66", "event.code": "062511418055", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:58 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=062511418055 log_type=\"Event\" log_component=\"IPSec\" log_subtype=\"System\" status=\"Failed\" priority=Warning user_name=\"elastic.user@elastic.test.com\" connectionname=\"Location-1\" connectiontype=\"0\" localinterfaceip=214.167.51.66 localgateway=\"\" localnetwork=\"172.17.32.0/19\" remoteinterfaceip=83.20.132.250 remotenetwork=\"10.84.234.5/32\" message=\"location-1 - IKE message retransmission timed out (Remote: 83.20.132.250)\"", "event.severity": "4", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "warning", @@ -87,19 +87,19 @@ "elastic.user@elastic.test.com" ], "server.ip": "214.167.51.66", - "service.type": "sophosxg", - "sophosxg.firewall.connectionname": "Location-1", - "sophosxg.firewall.connectiontype": "0", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.localnetwork": "172.17.32.0/19", - "sophosxg.firewall.log_component": "IPSec", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "18055", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.remotenetwork": "10.84.234.5/32", - "sophosxg.firewall.status": "Failed", + "service.type": "sophos", + "sophos.xg.connectionname": "Location-1", + "sophos.xg.connectiontype": "0", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.localnetwork": "172.17.32.0/19", + "sophos.xg.log_component": "IPSec", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "18055", + "sophos.xg.priority": "Warning", + "sophos.xg.remotenetwork": "10.84.234.5/32", + "sophos.xg.status": "Failed", "source.as.number": 5617, "source.as.organization.name": "Orange Polska Spolka Akcyjna", "source.geo.city_name": "Elblag", @@ -112,20 +112,20 @@ "source.ip": "83.20.132.250", "source.user.name": "elastic.user@elastic.test.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2020-05-18T14:38:59.000-02:00", "event.code": "062511318057", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:59 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=062511318057 log_type=\"Event\" log_component=\"IPSec\" log_subtype=\"System\" status=\"Expire\" priority=Error user_name=\"\" connectionname=\"\" connectiontype=\"0\" localinterfaceip=\"\" localgateway=\"\" localnetwork=\"\" remoteinterfaceip=\"\" remotenetwork=\"\" message=\"IKE_SA timed out before it could be established\"", "event.severity": "3", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "error", @@ -135,18 +135,18 @@ "observer.serial_number": "1234567890123456", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.connectiontype": "0", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "IPSec", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "18057", - "sophosxg.firewall.priority": "Error", - "sophosxg.firewall.status": "Expire", + "service.type": "sophos", + "sophos.xg.connectiontype": "0", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "IPSec", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "18057", + "sophos.xg.priority": "Error", + "sophos.xg.status": "Expire", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -157,9 +157,9 @@ "authentication" ], "event.code": "063210617704", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:00 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=063210617704 log_type=\"Event\" log_component=\"My Account Authentication\" log_subtype=\"Authentication\" status=\"Successful\" priority=Information user_name=\"elastic.user@elastic.test.com\" usergroupname=\"\" auth_client=\"N/A\" auth_mechanism=\"Local\" reason=\"\" src_ip=83.9.140.96 message=\"User elastic.user@elastic.test.com logged in successfully to MyAccount through Local authentication mechanism\" name=\"\" src_mac=", "event.outcome": "success", "event.severity": "6", @@ -168,7 +168,7 @@ "user", "start" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -184,16 +184,16 @@ "related.user": [ "elastic.user@elastic.test.com" ], - "service.type": "sophosxg", - "sophosxg.firewall.auth_mechanism": "Local", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "My Account Authentication", - "sophosxg.firewall.log_subtype": "Authentication", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17704", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Successful", + "service.type": "sophos", + "sophos.xg.auth_mechanism": "Local", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "My Account Authentication", + "sophos.xg.log_subtype": "Authentication", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17704", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Successful", "source.as.number": 5617, "source.as.organization.name": "Orange Polska Spolka Akcyjna", "source.geo.city_name": "August\u00f3w", @@ -206,7 +206,7 @@ "source.ip": "83.9.140.96", "source.user.name": "elastic.user@elastic.test.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -217,16 +217,16 @@ "malware" ], "event.code": "064011517819", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:01 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=064011517819 log_type=\"Event\" log_component=\"Anti-Virus\" log_subtype=\"System\" priority=Notice status=\"Successful\" oldversion=1.0.407794 newversion=1.0.407795 message=\"Avira AV definitions upgraded from 1.0.407794 to 1.0.407795.\"", "event.severity": "5", "event.timezone": "-02:00", "event.type": [ "info" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "notification", @@ -236,32 +236,32 @@ "observer.serial_number": "1234567890123456", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "Anti-Virus", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17819", - "sophosxg.firewall.newversion": "1.0.407795 ", - "sophosxg.firewall.oldversion": "1.0.407794", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.status": "Successful", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "Anti-Virus", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17819", + "sophos.xg.newversion": "1.0.407795 ", + "sophos.xg.oldversion": "1.0.407794", + "sophos.xg.priority": "Notice", + "sophos.xg.status": "Successful", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2020-05-18T14:39:02.000-02:00", "event.code": "063411660022", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:02 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=063411660022 log_type=\"Event\" log_component=\"DHCP Server\" log_subtype=\"System\" status=\"Expire\" priority=Information ipaddress=\"192.168.110.10\" client_physical_address=\"-\" client_host_name=\"\" message=\"Lease 192.168.110.10 expired\" raw_data=\"192.168.110.10\"", "event.severity": "6", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "informational", @@ -271,19 +271,19 @@ "observer.serial_number": "1234567890123457", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.ipaddress": "192.168.110.10", - "sophosxg.firewall.log_component": "DHCP Server", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "60022", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.raw_data": "192.168.110.10", - "sophosxg.firewall.status": "Expire", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.ipaddress": "192.168.110.10", + "sophos.xg.log_component": "DHCP Server", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "60022", + "sophos.xg.priority": "Information", + "sophos.xg.raw_data": "192.168.110.10", + "sophos.xg.status": "Expire", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -294,9 +294,9 @@ "authentication" ], "event.code": "063110617710", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:03 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=063110617710 log_type=\"Event\" log_component=\"SSL VPN Authentication\" log_subtype=\"Authentication\" status=\"Successful\" priority=Information user_name=\"elastic.user@elastic.test.com\" usergroupname=\"\" auth_client=\"N/A\" auth_mechanism=\"AD\" reason=\"\" src_ip=217.250.157.135 message=\"User elastic.user@elastic.test.com authenticated successfully to login to SSLVPN through AD authentication mechanism\" name=\"\" src_mac=", "event.outcome": "success", "event.severity": "6", @@ -305,7 +305,7 @@ "user", "start" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -321,16 +321,16 @@ "related.user": [ "elastic.user@elastic.test.com" ], - "service.type": "sophosxg", - "sophosxg.firewall.auth_mechanism": "AD", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "SSL VPN Authentication", - "sophosxg.firewall.log_subtype": "Authentication", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17710", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Successful", + "service.type": "sophos", + "sophos.xg.auth_mechanism": "AD", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "SSL VPN Authentication", + "sophos.xg.log_subtype": "Authentication", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17710", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Successful", "source.as.number": 3320, "source.as.organization.name": "Deutsche Telekom AG", "source.geo.city_name": "Schleidweiler", @@ -343,7 +343,7 @@ "source.ip": "217.250.157.135", "source.user.name": "elastic.user@elastic.test.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -352,13 +352,13 @@ "client.bytes": 0, "destination.bytes": 0, "event.code": "062811617824", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:04 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=062811617824 log_type=\"Event\" log_component=\"SSL VPN\" log_subtype=\"System\" priority=Information Mode=\"Remote Access\" sessionid=\"\" starttime=0 user_name=\"elastic.user@elastic.test.com\" ipaddress=10.82.234.5 sent_bytes=0 recv_bytes=0 status=\"Established\" message=\"SSL VPN User 'elastic.user@elastic.test.com' connected \" timestamp=1589960866 connectionname=\"\" remote_ip=10.82.234.12", "event.severity": "6", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -372,23 +372,23 @@ "elastic.user@elastic.test.com" ], "server.bytes": 0, - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.ipaddress": "10.82.234.5", - "sophosxg.firewall.log_component": "SSL VPN", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17824", - "sophosxg.firewall.priority": "Information Mode=\"Remote Access", - "sophosxg.firewall.remote_ip": "10.82.234.12", - "sophosxg.firewall.starttime": "0", - "sophosxg.firewall.status": "Established", - "sophosxg.firewall.timestamp": "1589960866", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.ipaddress": "10.82.234.5", + "sophos.xg.log_component": "SSL VPN", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17824", + "sophos.xg.priority": "Information Mode=\"Remote Access", + "sophos.xg.remote_ip": "10.82.234.12", + "sophos.xg.starttime": "0", + "sophos.xg.status": "Established", + "sophos.xg.timestamp": "1589960866", "source.bytes": 0, "source.user.name": "elastic.user@elastic.test.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -399,14 +399,14 @@ "authentication" ], "event.code": "063010517708", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:05 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=063010517708 log_type=\"Event\" log_component=\"VPN Authentication\" log_subtype=\"Authentication\" status=\"Failed\" priority=Notice user_name=\"hendrikl\" usergroupname=\"\" auth_client=\"N/A\" auth_mechanism=\"AD,AD,Local\" reason=\"wrong credentials\" src_ip=91.67.201.4 message=\"User elastic01 failed to login to VPN through AD,AD,Local authentication mechanism because of wrong credentials\" name=\"\" src_mac=", "event.outcome": "failure", "event.severity": "5", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "notification", @@ -422,17 +422,17 @@ "related.user": [ "hendrikl" ], - "service.type": "sophosxg", - "sophosxg.firewall.auth_mechanism": "AD,AD,Local", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "VPN Authentication", - "sophosxg.firewall.log_subtype": "Authentication", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17708", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.reason": "wrong credentials", - "sophosxg.firewall.status": "Failed", + "service.type": "sophos", + "sophos.xg.auth_mechanism": "AD,AD,Local", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "VPN Authentication", + "sophos.xg.log_subtype": "Authentication", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17708", + "sophos.xg.priority": "Notice", + "sophos.xg.reason": "wrong credentials", + "sophos.xg.status": "Failed", "source.as.number": 31334, "source.as.organization.name": "Vodafone Kabel Deutschland GmbH", "source.geo.city_name": "Fell", @@ -445,20 +445,20 @@ "source.ip": "91.67.201.4", "source.user.name": "hendrikl", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2020-05-18T14:39:06.000-02:00", "event.code": "066911518017", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:06 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=066911518017 log_type=\"Event\" log_component=\"ATP\" log_subtype=\"System\" priority=Notice status=\"Successful\" oldversion=1.0.0297 newversion=1.0.0298 message=\"ATP definitions upgraded from 1.0.0297 to 1.0.0298.\"", "event.severity": "5", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "notification", @@ -468,19 +468,19 @@ "observer.serial_number": "1234567890123456", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "ATP", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "18017", - "sophosxg.firewall.newversion": "1.0.0298 ", - "sophosxg.firewall.oldversion": "1.0.0297", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.status": "Successful", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "ATP", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "18017", + "sophos.xg.newversion": "1.0.0298 ", + "sophos.xg.oldversion": "1.0.0297", + "sophos.xg.priority": "Notice", + "sophos.xg.status": "Successful", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -488,14 +488,14 @@ "@timestamp": "2020-05-18T14:39:08.000-02:00", "client.ip": "172.66.35.15", "event.code": "062109517507", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:08 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=062109517507 log_type=\"Event\" log_component=\"CLI\" log_subtype=\"Admin\" status=\"Failed\" priority=Notice user_name=\"root\" src_ip=172.66.35.15 message=\"User 'root' failed to login from '172.66.35.15' using ssh because of wrong credentials\"", "event.outcome": "failure", "event.severity": "5", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "notification", @@ -511,15 +511,15 @@ "related.user": [ "root" ], - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "CLI", - "sophosxg.firewall.log_subtype": "Admin", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17507", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.status": "Failed", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "CLI", + "sophos.xg.log_subtype": "Admin", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17507", + "sophos.xg.priority": "Notice", + "sophos.xg.status": "Failed", "source.geo.continent_name": "North America", "source.geo.country_iso_code": "US", "source.geo.location.lat": 37.751, @@ -527,20 +527,20 @@ "source.ip": "172.66.35.15", "source.user.name": "root", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2020-05-18T14:39:09.000-02:00", "event.code": "063911517818", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:09 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=063911517818 log_type=\"Event\" log_component=\"IPS\" log_subtype=\"System\" priority=Notice status=\"Successful\" oldversion=9.17.09 newversion=9.17.10 message=\"IPS definitions upgraded from 9.17.09 to 9.17.10.\"", "event.severity": "5", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "notification", @@ -550,32 +550,32 @@ "observer.serial_number": "1234567890123456", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "IPS", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17818", - "sophosxg.firewall.newversion": "9.17.10 ", - "sophosxg.firewall.oldversion": "9.17.09", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.status": "Successful", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "IPS", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17818", + "sophos.xg.newversion": "9.17.10 ", + "sophos.xg.oldversion": "9.17.09", + "sophos.xg.priority": "Notice", + "sophos.xg.status": "Successful", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2020-05-18T14:39:10.000-02:00", "event.code": "063311617923", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:10 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=063311617923 log_type=\"Event\" log_component=\"Appliance\" log_subtype=\"System\" priority=Information backup_mode='appliance' message=\"Scheduled backup to appliance is successful.\"", "event.severity": "6", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -585,17 +585,17 @@ "observer.serial_number": "1234567890123456", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.backup_mode": "'appliance' ", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "Appliance", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17923", - "sophosxg.firewall.priority": "Information", + "service.type": "sophos", + "sophos.xg.backup_mode": "'appliance' ", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "Appliance", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17923", + "sophos.xg.priority": "Information", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -609,9 +609,9 @@ "authentication" ], "event.code": "062910617703", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:39:20 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=062910617703 log_type=\"Event\" log_component=\"Firewall Authentication\" log_subtype=\"Authentication\" status=\"Successful\" priority=Information user_name=\"elastic.user@elastic.test.com\" usergroupname=\"VPN.SSL.Users.elastic\" auth_client=\"IPSec\" auth_mechanism=\"N/A\" reason=\"\" src_ip=10.84.234.38 src_mac=\"\" start_time=1591086575 sent_bytes=0 recv_bytes=0 message=\"User elastic.user@elastic.test.com was logged out of firewall\" name=\"elastic.user@elastic.test.com\" timestamp=1591086576", "event.outcome": "success", "event.severity": "6", @@ -621,7 +621,7 @@ "end", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "informational", @@ -638,37 +638,37 @@ "elastic.user@elastic.test.com" ], "server.bytes": 0, - "service.type": "sophosxg", - "sophosxg.firewall.auth_client": "IPSec", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.log_component": "Firewall Authentication", - "sophosxg.firewall.log_subtype": "Authentication", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17703", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.start_time": "1591086575", - "sophosxg.firewall.status": "Successful", - "sophosxg.firewall.timestamp": "1591086576", + "service.type": "sophos", + "sophos.xg.auth_client": "IPSec", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.log_component": "Firewall Authentication", + "sophos.xg.log_subtype": "Authentication", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17703", + "sophos.xg.priority": "Information", + "sophos.xg.start_time": "1591086575", + "sophos.xg.status": "Successful", + "sophos.xg.timestamp": "1591086576", "source.bytes": 0, "source.ip": "10.84.234.38", "source.user.group.name": "VPN.SSL.Users.elastic", "source.user.name": "elastic.user@elastic.test.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2018-06-06T11:12:10.000-02:00", "event.code": "063711517815", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-06 time=11:12:10 timezone=\"IST\" device_name=\"SG430\" device_id=S4000806149EE49 log_id=063711517815 log_type=\"Event\" log_component=\"DDNS\" log_subtype=\"System\" status=\"Success\" priority=Notice host=test1. customtest.dyndns.org updatedip=10.198.232.86 reason=\"\" message=\"DDNS update for host test1.customtest.dyndns.org was Successful. Updated with IP 10.198.232.86.\"", "event.severity": "5", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "notification", @@ -678,19 +678,19 @@ "observer.serial_number": "S4000806149EE49", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG430", - "sophosxg.firewall.host": "test1. customtest.dyndns.org", - "sophosxg.firewall.log_component": "DDNS", - "sophosxg.firewall.log_subtype": "System", - "sophosxg.firewall.log_type": "Event", - "sophosxg.firewall.message_id": "17815", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.status": "Success", - "sophosxg.firewall.updatedip": "10.198.232.86", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG430", + "sophos.xg.host": "test1. customtest.dyndns.org", + "sophos.xg.log_component": "DDNS", + "sophos.xg.log_subtype": "System", + "sophos.xg.log_type": "Event", + "sophos.xg.message_id": "17815", + "sophos.xg.priority": "Notice", + "sophos.xg.status": "Success", + "sophos.xg.updatedip": "10.198.232.86", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/firewall.log b/x-pack/filebeat/module/sophos/xg/test/firewall.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/firewall.log rename to x-pack/filebeat/module/sophos/xg/test/firewall.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/firewall.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/firewall.log-expected.json similarity index 74% rename from x-pack/filebeat/module/sophosxg/firewall/test/firewall.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/firewall.log-expected.json index b4de3f90788..d392790d795 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/firewall.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/firewall.log-expected.json @@ -26,19 +26,22 @@ "network" ], "event.code": "010101600001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 11000000000, + "event.end": "2020-05-18T14:38:48.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:37 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=010101600001 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" status=\"Allow\" priority=Information duration=11 fw_rule_id=21 policy_type=1 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"HTTP\" application_risk=1 application_technology=\"Browser Based\" application_category=\"General Internet\" in_interface=\"Port1\" out_interface=\"Port2\" src_mac=00:00:00:00:00:00 src_ip=172.17.34.15 src_country_code=R1 dst_ip=91.228.167.86 dst_country_code=SVK protocol=\"TCP\" src_port=62841 dst_port=80 sent_pkts=6 recv_pkts=5 sent_bytes=459 recv_bytes=606 tran_src_ip=213.167.51.66 tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"LAN\" srczone=\"LAN\" dstzonetype=\"WAN\" dstzone=\"WAN\" dir_disp=\"\" connevent=\"Stop\" connid=\"1617925280\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:37.000-02:00", "event.timezone": "-02:00", "event.type": [ "end", "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -68,28 +71,28 @@ "server.nat.port": 0, "server.packets": 5, "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_category": "General Internet", - "sophosxg.firewall.application_risk": "1", - "sophosxg.firewall.application_technology": "Browser Based", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.connevent": "Stop", - "sophosxg.firewall.connid": "1617925280", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "SVK", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.status": "Allow", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_category": "General Internet", + "sophos.xg.application_risk": "1", + "sophos.xg.application_technology": "Browser Based", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.connevent": "Stop", + "sophos.xg.connid": "1617925280", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "SVK", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00001", + "sophos.xg.priority": "Information", + "sophos.xg.src_country_code": "R1", + "sophos.xg.status": "Allow", "source.as.number": 8905, "source.as.organization.name": "Digit One LLC", "source.bytes": 459, @@ -104,7 +107,7 @@ "source.packets": 6, "source.port": 62841, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -135,19 +138,22 @@ "network" ], "event.code": "010101600001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:38.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:38 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=010101600001 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" status=\"Allow\" priority=Information duration=0 fw_rule_id=67 policy_type=1 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=15 appfilter_policy_id=0 application=\"DNS\" application_risk=1 application_technology=\"Network Protocol\" application_category=\"Infrastructure\" in_interface=\"Port3.400\" out_interface=\"Port2\" src_mac=00:00:00:00:00:00 src_ip=172.16.66.155 src_country_code=R1 dst_ip=91.228.165.117 dst_country_code=SVK protocol=\"UDP\" src_port=49144 dst_port=53 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=185.8.209.194 tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"DMZ\" srczone=\"DMZ\" dstzonetype=\"WAN\" dstzone=\"WAN\" dir_disp=\"\" connevent=\"Start\" connid=\"3360392048\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:38.000-02:00", "event.timezone": "-02:00", "event.type": [ "start", "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "informational", @@ -177,28 +183,28 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 53, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_category": "Infrastructure", - "sophosxg.firewall.application_risk": "1", - "sophosxg.firewall.application_technology": "Network Protocol", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.connevent": "Start", - "sophosxg.firewall.connid": "3360392048", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "SVK", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "15", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.status": "Allow", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_category": "Infrastructure", + "sophos.xg.application_risk": "1", + "sophos.xg.application_technology": "Network Protocol", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.connevent": "Start", + "sophos.xg.connid": "3360392048", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "SVK", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "15", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00001", + "sophos.xg.priority": "Information", + "sophos.xg.src_country_code": "R1", + "sophos.xg.status": "Allow", "source.as.number": 199567, "source.as.organization.name": "Fr. Sauter AG", "source.bytes": 0, @@ -216,7 +222,7 @@ "source.packets": 0, "source.port": 49144, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -238,18 +244,21 @@ "network" ], "event.code": "010102600002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:39.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:39 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=010102600002 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=29 policy_type=1 user_name=\"\" user_gp=\"\" iap=2 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port1\" out_interface=\"Port2\" src_mac=24:01:c7:07:2b:a2 src_ip=172.17.35.113 src_country_code=\"\" dst_ip=172.20.4.52 dst_country_code=\"\" protocol=\"TCP\" src_port=53287 dst_port=4980 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:39.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -274,22 +283,22 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 4980, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "2", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00002", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "2", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00002", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "172.17.35.113", "source.mac": "24:01:c7:07:2b:a2", @@ -297,7 +306,7 @@ "source.packets": 0, "source.port": 53287, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -318,18 +327,21 @@ "network" ], "event.code": "010102600002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:40.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:40 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=010102600002 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=29 policy_type=1 user_name=\"elastic@user.local\" user_gp=\"elastic.group.local\" iap=2 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"tun0\" out_interface=\"Port1\" src_mac=\"\" src_ip=10.82.234.6 src_country_code=\"\" dst_ip=192.168.0.1 dst_country_code=\"\" protocol=\"TCP\" src_port=60102 dst_port=53 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:40.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -357,22 +369,22 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 53, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "2", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00002", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "2", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00002", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.82.234.6", "source.nat.port": 0, @@ -381,7 +393,7 @@ "source.user.group.name": "elastic.group.local", "source.user.name": "elastic@user.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -409,18 +421,21 @@ "network" ], "event.code": "010302602002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:41.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:41 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=010302602002 log_type=\"Firewall\" log_component=\"Appliance Access\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port2\" out_interface=\"\" src_mac=c4:f7:d5:b5:47:f4 src_ip=51.77.56.9 src_country_code=\"\" dst_ip=185.7.209.207 dst_country_code=\"\" protocol=\"TCP\" src_port=55039 dst_port=18 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:41.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "informational", @@ -444,22 +459,22 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 18, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Appliance Access", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "02002", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Appliance Access", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "02002", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.as.number": 16276, "source.as.organization.name": "OVH SAS", "source.bytes": 0, @@ -476,7 +491,7 @@ "source.packets": 0, "source.port": 55039, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -498,18 +513,21 @@ "network" ], "event.code": "010102600002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:42.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:42 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=010102600002 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=29 policy_type=1 user_name=\"elastic@user.local\" user_gp=\"elastic.group.local\" iap=2 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port1\" out_interface=\"Port2\" src_mac=24:01:c7:07:2b:a2 src_ip=172.17.35.101 src_country_code=\"\" dst_ip=192.168.5.11 dst_country_code=\"\" protocol=\"TCP\" src_port=51826 dst_port=1109 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:42.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -537,22 +555,22 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 1109, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "2", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00002", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "2", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00002", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "172.17.35.101", "source.mac": "24:01:c7:07:2b:a2", @@ -562,7 +580,7 @@ "source.user.group.name": "elastic.group.local", "source.user.name": "elastic@user.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -585,18 +603,21 @@ "network" ], "event.code": "010402403001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:43.000-02:00", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:43 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=010402403001 log_type=\"Firewall\" log_component=\"DoS Attack\" log_subtype=\"Denied\" status=\"Deny\" priority=Warning duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port1\" out_interface=\"\" src_mac=34:db:fd:83:d8:09 src_ip=172.16.36.105 src_country_code=\"\" dst_ip=10.84.234.14 dst_country_code=\"\" protocol=\"UDP\" src_port=3389 dst_port=64465 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "4", + "event.start": "2020-05-18T14:38:43.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "warning", @@ -620,22 +641,22 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 64465, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "DoS Attack", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "03001", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "DoS Attack", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "03001", + "sophos.xg.priority": "Warning", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "172.16.36.105", "source.mac": "34:db:fd:83:d8:09", @@ -643,7 +664,7 @@ "source.packets": 0, "source.port": 3389, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -664,18 +685,21 @@ "network" ], "event.code": "012802605201", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:44.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:44 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=012802605201 log_type=\"Firewall\" log_component=\"SSL VPN\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"tun0\" out_interface=\"\" src_mac=\"\" src_ip=10.82.234.9 src_country_code=\"\" dst_ip=10.82.234.11 dst_country_code=\"\" protocol=\"TCP\" src_port=58331 dst_port=56267 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:44.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -699,29 +723,29 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 56267, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "SSL VPN", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "05201", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "SSL VPN", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "05201", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.82.234.9", "source.nat.port": 0, "source.packets": 0, "source.port": 58331, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -743,19 +767,22 @@ "network" ], "event.code": "010101600001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:45.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:45 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=010101600001 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" status=\"Allow\" priority=Information duration=0 fw_rule_id=61 policy_type=2 user_name=\"elastic@user.local\" user_gp=\"elastic.group.local\" iap=0 ips_policy_id=11 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"ipsec0\" out_interface=\"Port2\" src_mac=00:00:00:00:00:00 src_ip=10.84.234.7 src_country_code=R1 dst_ip=172.16.34.50 dst_country_code=R1 protocol=\"TCP\" src_port=58543 dst_port=443 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"VPN\" srczone=\"VPN\" dstzonetype=\"VPN\" dstzone=\"VPN\" dir_disp=\"\" connevent=\"Start\" connid=\"1615935064\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-05-18T14:38:45.000-02:00", "event.timezone": "-02:00", "event.type": [ "start", "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "informational", @@ -786,26 +813,26 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 443, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.connevent": "Start", - "sophosxg.firewall.connid": "1615935064", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "11", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.status": "Allow", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.connevent": "Start", + "sophos.xg.connid": "1615935064", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "11", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00001", + "sophos.xg.priority": "Information", + "sophos.xg.src_country_code": "R1", + "sophos.xg.status": "Allow", "source.bytes": 0, "source.ip": "10.84.234.7", "source.mac": "00:00:00:00:00:00", @@ -815,7 +842,7 @@ "source.user.group.name": "elastic.group.local", "source.user.name": "elastic@user.local", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -835,19 +862,22 @@ "network" ], "event.code": "018201500005", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2020-05-18T14:38:45.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:45 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=018201500005 log_type=\"Firewall\" log_component=\"ICMP ERROR MESSAGE\" log_subtype=\"Allowed\" status=\"Allow\" priority=Notice duration=0 fw_rule_id=60 policy_type=1 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=17 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port1\" out_interface=\"\" src_mac=34:db:fd:83:d8:09 src_ip=192.168.1.254 src_country_code=\"\" dst_ip=172.17.32.19 dst_country_code=\"\" protocol=\"ICMP\" icmp_type=3 icmp_code=1 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip=\"\" tran_src_port=0 tran_dst_ip=\"\" tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connevent=\"Interim\" connid=\"2685668438\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "5", + "event.start": "2020-05-18T14:38:45.000-02:00", "event.timezone": "-02:00", "event.type": [ "start", "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "notification", @@ -870,33 +900,33 @@ "server.ip": "172.17.32.19", "server.nat.port": 0, "server.packets": 0, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.connevent": "Interim", - "sophosxg.firewall.connid": "2685668438", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.icmp_code": "1", - "sophosxg.firewall.icmp_type": "3", - "sophosxg.firewall.ips_policy_id": "17", - "sophosxg.firewall.log_component": "ICMP ERROR MESSAGE", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00005", - "sophosxg.firewall.priority": "Notice", - "sophosxg.firewall.status": "Allow", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.connevent": "Interim", + "sophos.xg.connid": "2685668438", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.icmp_code": "1", + "sophos.xg.icmp_type": "3", + "sophos.xg.ips_policy_id": "17", + "sophos.xg.log_component": "ICMP ERROR MESSAGE", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00005", + "sophos.xg.priority": "Notice", + "sophos.xg.status": "Allow", "source.bytes": 0, "source.ip": "192.168.1.254", "source.mac": "34:db:fd:83:d8:09", "source.nat.port": 0, "source.packets": 0, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -917,19 +947,22 @@ "network" ], "event.code": "010101600001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 10000000000, + "event.end": "2020-06-05T12:39:03.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-06-05 time=12:38:53 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=010101600001 log_type=\"Firewall\" log_component=\"Firewall Rule\" log_subtype=\"Allowed\" status=\"Allow\" priority=Information duration=10 fw_rule_id=60 policy_type=1 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=17 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"ipsec0\" out_interface=\"Port1\" src_mac=00:00:00:00:00:00 src_ip=172.17.35.119 src_country_code=R1 dst_ip=172.16.34.10 dst_country_code=R1 protocol=\"TCP\" src_port=61925 dst_port=88 sent_pkts=6 recv_pkts=6 sent_bytes=1802 recv_bytes=1732 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0srczonetype=\"VPN\" srczone=\"VPN\" dstzonetype=\"LAN\" dstzone=\"LAN\" dir_disp=\"\" connevent=\"Stop\" connid=\"1617126256\" vconnid=\"\" hb_health=\"NoHeartbeat\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2020-06-05T12:38:53.000-02:00", "event.timezone": "-02:00", "event.type": [ "end", "allowed", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "informational", @@ -954,26 +987,26 @@ "server.ip": "172.16.34.10", "server.packets": 6, "server.port": 88, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.connevent": "Stop", - "sophosxg.firewall.connid": "1617126256", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.hb_health": "NoHeartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "17", - "sophosxg.firewall.log_component": "Firewall Rule", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.status": "Allow", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.connevent": "Stop", + "sophos.xg.connid": "1617126256", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.hb_health": "NoHeartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "17", + "sophos.xg.log_component": "Firewall Rule", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00001", + "sophos.xg.priority": "Information", + "sophos.xg.src_country_code": "R1", + "sophos.xg.status": "Allow", "source.bytes": 1802, "source.ip": "172.17.35.119", "source.mac": "00:00:00:00:00:00", @@ -981,7 +1014,7 @@ "source.packets": 6, "source.port": 61925, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1008,18 +1041,21 @@ "network" ], "event.code": "010202601001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-05-30T13:26:37.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-30 time=13:26:37 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=010202601001 log_type=\"Firewall\" log_component=\"Invalid Traffic\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"\" out_interface=\"\" src_mac= src_ip=10.198.32.19 src_country_code= dst_ip=8.8.8.8 dst_country_code= protocol=\"UDP\" src_port=1353 dst_port=0 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"Invalid UDP destination.\" appresolvedby=\" Signature\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-05-30T13:26:37.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1042,29 +1078,29 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 0, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": " Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Invalid Traffic", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message": "Invalid UDP destination.", - "sophosxg.firewall.message_id": "01001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": " Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Invalid Traffic", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message": "Invalid UDP destination.", + "sophos.xg.message_id": "01001", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.32.19", "source.nat.port": 0, "source.packets": 0, "source.port": 1353, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1085,18 +1121,21 @@ "network" ], "event.code": "011402601301", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-06-04T17:20:24.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-04 time=17:20:24 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=011402601301 log_type=\"Firewall\" log_component=\"Fragmented Traffic\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"\" out_interface=\"\" src_mac= src_ip=0.0.0.0 src_country_code= dst_ip=0.0.0.0 dst_country_code= protocol=\"0\" src_port=0 dst_port=0 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-06-04T17:20:24.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1119,28 +1158,28 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 0, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Fragmented Traffic", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "01301", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Fragmented Traffic", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "01301", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "0.0.0.0", "source.nat.port": 0, "source.packets": 0, "source.port": 0, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1162,18 +1201,21 @@ "network" ], "event.code": "010302602002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-05-30T14:01:32.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-30 time=14:01:32 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=010302602002 log_type=\"Firewall\" log_component=\"Appliance Access\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=2 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port2.611\" out_interface=\"\" src_mac=c8:5b:76:ab:72:d3 src_ip=10.198.38.184 src_country_code= dst_ip=10.198.39.255 dst_country_code= protocol=\"UDP\" src_port=137 dst_port=137 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-05-30T14:01:32.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1197,21 +1239,21 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 137, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Appliance Access", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "02002", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Appliance Access", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "02002", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.38.184", "source.mac": "c8:5b:76:ab:72:d3", @@ -1219,7 +1261,7 @@ "source.packets": 0, "source.port": 137, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1242,18 +1284,21 @@ "network" ], "event.code": "010402403001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-05-30T14:17:17.000-02:00", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-30 time=14:17:17 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=010402403001 log_type=\"Firewall\" log_component=\"DoS Attack\" log_subtype=\"Denied\" status=\"Deny\" priority=Warning duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port1\" out_interface=\"\" src_mac=b8:97:5a:5b:0f:fd src_ip=10.198.32.19 src_country_code= dst_ip=10.198.32.48 dst_country_code= protocol=\"TCP\" src_port=41960 dst_port=22 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\" Signature\"", "event.outcome": "success", "event.severity": "4", + "event.start": "2018-05-30T14:17:17.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -1277,21 +1322,21 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 22, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": " Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "DoS Attack", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "03001", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": " Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "DoS Attack", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "03001", + "sophos.xg.priority": "Warning", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.32.19", "source.mac": "b8:97:5a:5b:0f:fd", @@ -1299,7 +1344,7 @@ "source.packets": 0, "source.port": 41960, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1318,18 +1363,21 @@ "network" ], "event.code": "010502604001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-06-05T14:30:31.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-05 time=14:30:31 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=010502604001 log_type=\"Firewall\" log_component=\"ICMP Redirection\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"\" out_interface=\"\" src_mac= src_ip=10.198.37.23 src_country_code= dst_ip=10.198.36.48 dst_country_code= protocol=\"ICMP\" icmp_type=5 icmp_code=1 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\" Signature\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-06-05T14:30:31.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1351,29 +1399,29 @@ "server.ip": "10.198.36.48", "server.nat.port": 0, "server.packets": 0, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": " Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.icmp_code": "1", - "sophosxg.firewall.icmp_type": "5", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "ICMP Redirection", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "04001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": " Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.icmp_code": "1", + "sophos.xg.icmp_type": "5", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "ICMP Redirection", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "04001", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.37.23", "source.nat.port": 0, "source.packets": 0, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1401,18 +1449,21 @@ "network" ], "event.code": "010602605001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-05-31T17:05:14.000-02:00", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-31 time=17:05:14 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=010602605001 log_type=\"Firewall\" log_component=\"Source Routed\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=1 policy_type=1 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"\" out_interface=\"\" src_mac= src_ip=10.198.12.19 src_country_code= dst_ip=8.8.8.8 dst_country_code= protocol=\"TCP\" src_port=1571 dst_port=80 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-05-31T17:05:14.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1435,28 +1486,28 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Source Routed", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "05001", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Source Routed", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "05001", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.12.19", "source.nat.port": 0, "source.packets": 0, "source.port": 1571, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1478,18 +1529,21 @@ "network" ], "event.code": "011702605051", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-05-30T15:09:51.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-30 time=15:09:51 timezone=\"IST\" device_name=\"XG125w\" device_id=SFDemo-763180a log_id=011702605051 log_type=\"Firewall\" log_component=\"MAC Filter\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=0 policy_type=0 user_name=\"\" user_gp=\"\" iap=0 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port2.531\" out_interface=\"\" src_mac=1e:3a:5a:5b:23:ab src_ip=fe80::59f5:3ce8:c98e:5062 src_country_code= dst_ip=ff02::1:2 dst_country_code= protocol=\"UDP\" src_port=546 dst_port=547 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"No Heartbeat\" message=\"\" appresolvedby=\"Signature\"", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-05-30T15:09:51.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1513,21 +1567,21 @@ "server.nat.port": 0, "server.packets": 0, "server.port": 547, - "service.type": "sophosxg", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG125w", - "sophosxg.firewall.hb_health": "No Heartbeat", - "sophosxg.firewall.iap": "0", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "MAC Filter", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "05051", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG125w", + "sophos.xg.hb_health": "No Heartbeat", + "sophos.xg.iap": "0", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "MAC Filter", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "05051", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "fe80::59f5:3ce8:c98e:5062", "source.mac": "1e:3a:5a:5b:23:ab", @@ -1535,7 +1589,7 @@ "source.packets": 0, "source.port": 546, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1555,18 +1609,21 @@ "network" ], "event.code": "016602600006", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-06-01T10:57:55.000-02:00", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-01 time=10:57:55 timezone=\"BST\" device_name=\"XG310\" device_id=SFDemo-9a04c43 log_id=016602600006 log_type=\"Firewall\" log_component=\"Heartbeat\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=16 policy_type=1 user_name=\"\" user_gp=\"\" iap=2 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port3.611\" out_interface=\"\" src_mac=08:00:27:4c:49:e3 src_ip=10.198.37.57 src_country_code= dst_ip=10.198.32.19 dst_country_code= protocol=\"ICMP\" icmp_type=8 icmp_code=0 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"Red\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-06-01T10:57:55.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1589,31 +1646,31 @@ "server.ip": "10.198.32.19", "server.nat.port": 0, "server.packets": 0, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG310", - "sophosxg.firewall.hb_health": "Red", - "sophosxg.firewall.iap": "2", - "sophosxg.firewall.icmp_code": "0", - "sophosxg.firewall.icmp_type": "8", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Heartbeat", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00006", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG310", + "sophos.xg.hb_health": "Red", + "sophos.xg.iap": "2", + "sophos.xg.icmp_code": "0", + "sophos.xg.icmp_type": "8", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Heartbeat", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00006", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.37.57", "source.mac": "08:00:27:4c:49:e3", "source.nat.port": 0, "source.packets": 0, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -1643,18 +1700,21 @@ "network" ], "event.code": "016602600003", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", + "event.duration": 0, + "event.end": "2018-06-01T10:55:41.000-02:00", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-06-01 time=10:55:41 timezone=\"BST\" device_name=\"XG310\" device_id=SFDemo-9a04c43 log_id=016602600003 log_type=\"Firewall\" log_component=\"Heartbeat\" log_subtype=\"Denied\" status=\"Deny\" priority=Information duration=0 fw_rule_id=16 policy_type=1 user_name=\"\" user_gp=\"\" iap=2 ips_policy_id=0 appfilter_policy_id=0 application=\"\" application_risk=0 application_technology=\"\" application_category=\"\" in_interface=\"Port3.611\" out_interface=\"\" src_mac=08:00:27:4c:49:e3 src_ip=10.198.37.57 src_country_code= dst_ip=72.163.4.185 dst_country_code= protocol=\"ICMP\" icmp_type=8 icmp_code=0 sent_pkts=0 recv_pkts=0 sent_bytes=0 recv_bytes=0 tran_src_ip= tran_src_port=0 tran_dst_ip= tran_dst_port=0 srczonetype=\"\" srczone=\"\" dstzonetype=\"\" dstzone=\"\" dir_disp=\"\" connid=\"\" vconnid=\"\" hb_health=\"Red\" message=\"\" appresolvedby=\"Signature\" app_is_cloud=0", "event.outcome": "success", "event.severity": "6", + "event.start": "2018-06-01T10:55:41.000-02:00", "event.timezone": "-02:00", "event.type": [ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -1677,31 +1737,31 @@ "server.ip": "72.163.4.185", "server.nat.port": 0, "server.packets": 0, - "service.type": "sophosxg", - "sophosxg.firewall.app_is_cloud": "0", - "sophosxg.firewall.appfilter_policy_id": "0", - "sophosxg.firewall.application_risk": "0", - "sophosxg.firewall.appresolvedby": "Signature", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG310", - "sophosxg.firewall.hb_health": "Red", - "sophosxg.firewall.iap": "2", - "sophosxg.firewall.icmp_code": "0", - "sophosxg.firewall.icmp_type": "8", - "sophosxg.firewall.ips_policy_id": "0", - "sophosxg.firewall.log_component": "Heartbeat", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Firewall", - "sophosxg.firewall.message_id": "00003", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.status": "Deny", + "service.type": "sophos", + "sophos.xg.app_is_cloud": "0", + "sophos.xg.appfilter_policy_id": "0", + "sophos.xg.application_risk": "0", + "sophos.xg.appresolvedby": "Signature", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG310", + "sophos.xg.hb_health": "Red", + "sophos.xg.iap": "2", + "sophos.xg.icmp_code": "0", + "sophos.xg.icmp_type": "8", + "sophos.xg.ips_policy_id": "0", + "sophos.xg.log_component": "Heartbeat", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Firewall", + "sophos.xg.message_id": "00003", + "sophos.xg.priority": "Information", + "sophos.xg.status": "Deny", "source.bytes": 0, "source.ip": "10.198.37.57", "source.mac": "08:00:27:4c:49:e3", "source.nat.port": 0, "source.packets": 0, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/idp.log b/x-pack/filebeat/module/sophos/xg/test/idp.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/idp.log rename to x-pack/filebeat/module/sophos/xg/test/idp.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/idp.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/idp.log-expected.json similarity index 69% rename from x-pack/filebeat/module/sophosxg/firewall/test/idp.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/idp.log-expected.json index ef1fdf7973f..7caee4d72eb 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/idp.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/idp.log-expected.json @@ -11,9 +11,9 @@ "network" ], "event.code": "020804407002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:54 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=020804407002 log_type=\"IDP\" log_component=\"Signatures\" log_subtype=\"Drop\" priority=Warning idp_policy_id=7 fw_rule_id=25 user_name=\"\" signature_id=1881 signature_msg=\"SERVER-WEBAPP bad HTTP 1.1 request - potential worm attack\" classification=\"access to a potentially vulnerable web application\" rule_priority=2 src_ip=89.40.182.58 src_country_code=ROU dst_ip=172.16.68.20 dst_country_code=R1 protocol=\"TCP\" src_port=41528 dst_port=80 platform=\"BSD,Linux,Mac,Other,Solaris,Unix,Windows\" category=\"server-webapp\" target=\"Server\"", "event.outcome": "success", "event.severity": "4", @@ -22,7 +22,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "warning", @@ -41,22 +41,22 @@ "rule.name": "SERVER-WEBAPP bad HTTP 1.1 request - potential worm attack", "server.ip": "172.16.68.20", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.category": "server-webapp", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.fw_rule_id": "25", - "sophosxg.firewall.idp_policy_id": "7", - "sophosxg.firewall.log_component": "Signatures", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "IDP", - "sophosxg.firewall.message_id": "07002", - "sophosxg.firewall.platform": "BSD,Linux,Mac,Other,Solaris,Unix,Windows", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.rule_priority": "2", - "sophosxg.firewall.src_country_code": "ROU", - "sophosxg.firewall.target": "Server", + "service.type": "sophos", + "sophos.xg.category": "server-webapp", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.fw_rule_id": "25", + "sophos.xg.idp_policy_id": "7", + "sophos.xg.log_component": "Signatures", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "IDP", + "sophos.xg.message_id": "07002", + "sophos.xg.platform": "BSD,Linux,Mac,Other,Solaris,Unix,Windows", + "sophos.xg.priority": "Warning", + "sophos.xg.rule_priority": "2", + "sophos.xg.src_country_code": "ROU", + "sophos.xg.target": "Server", "source.as.number": 28684, "source.as.organization.name": "Bestnet Service SRL", "source.geo.continent_name": "Europe", @@ -66,7 +66,7 @@ "source.ip": "89.40.182.58", "source.port": 41528, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -82,9 +82,9 @@ "network" ], "event.code": "020804407002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:55 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=020804407002 log_type=\"IDP\" log_component=\"Signatures\" log_subtype=\"Drop\" priority=Warning idp_policy_id=7 fw_rule_id=23 user_name=\"\" signature_id=1616 signature_msg=\"PROTOCOL-DNS named version attempt\" classification=\"Attempted Information Leak\" rule_priority=1 src_ip=117.50.11.192 src_country_code=CHN dst_ip=172.16.66.155 dst_country_code=R1 protocol=\"UDP\" src_port=58914 dst_port=53 platform=\"BSD,Linux,Mac,Other,Solaris,Unix,Windows\" category=\"protocol-dns\" target=\"Server\"", "event.outcome": "success", "event.severity": "4", @@ -93,7 +93,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "input.type": "log", "log.level": "warning", @@ -112,22 +112,22 @@ "rule.name": "PROTOCOL-DNS named version attempt", "server.ip": "172.16.66.155", "server.port": 53, - "service.type": "sophosxg", - "sophosxg.firewall.category": "protocol-dns", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.fw_rule_id": "23", - "sophosxg.firewall.idp_policy_id": "7", - "sophosxg.firewall.log_component": "Signatures", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "IDP", - "sophosxg.firewall.message_id": "07002", - "sophosxg.firewall.platform": "BSD,Linux,Mac,Other,Solaris,Unix,Windows", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.rule_priority": "1", - "sophosxg.firewall.src_country_code": "CHN", - "sophosxg.firewall.target": "Server", + "service.type": "sophos", + "sophos.xg.category": "protocol-dns", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.fw_rule_id": "23", + "sophos.xg.idp_policy_id": "7", + "sophos.xg.log_component": "Signatures", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "IDP", + "sophos.xg.message_id": "07002", + "sophos.xg.platform": "BSD,Linux,Mac,Other,Solaris,Unix,Windows", + "sophos.xg.priority": "Warning", + "sophos.xg.rule_priority": "1", + "sophos.xg.src_country_code": "CHN", + "sophos.xg.target": "Server", "source.as.number": 4808, "source.as.organization.name": "China Unicom Beijing Province Network", "source.geo.continent_name": "Asia", @@ -139,7 +139,7 @@ "source.ip": "117.50.11.192", "source.port": 58914, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -155,9 +155,9 @@ "network" ], "event.code": "020804407002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:56 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=020804407002 log_type=\"IDP\" log_component=\"Signatures\" log_subtype=\"Drop\" priority=Warning idp_policy_id=7 fw_rule_id=25 user_name=\"\" signature_id=53589 signature_msg=\"SERVER-WEBAPP DrayTek multiple products command injection attempt\" classification=\"Web Application Attack\" rule_priority=2 src_ip=77.61.185.101 src_country_code=NLD dst_ip=172.16.68.20 dst_country_code=R1 protocol=\"TCP\" src_port=59476 dst_port=80 platform=\"Linux,Mac,Other,Unix,Windows\" category=\"server-webapp\" target=\"Server\"", "event.outcome": "success", "event.severity": "4", @@ -166,7 +166,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "input.type": "log", "log.level": "warning", @@ -185,22 +185,22 @@ "rule.name": "SERVER-WEBAPP DrayTek multiple products command injection attempt", "server.ip": "172.16.68.20", "server.port": 80, - "service.type": "sophosxg", - "sophosxg.firewall.category": "server-webapp", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.fw_rule_id": "25", - "sophosxg.firewall.idp_policy_id": "7", - "sophosxg.firewall.log_component": "Signatures", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "IDP", - "sophosxg.firewall.message_id": "07002", - "sophosxg.firewall.platform": "Linux,Mac,Other,Unix,Windows", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.rule_priority": "2", - "sophosxg.firewall.src_country_code": "NLD", - "sophosxg.firewall.target": "Server", + "service.type": "sophos", + "sophos.xg.category": "server-webapp", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.fw_rule_id": "25", + "sophos.xg.idp_policy_id": "7", + "sophos.xg.log_component": "Signatures", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "IDP", + "sophos.xg.message_id": "07002", + "sophos.xg.platform": "Linux,Mac,Other,Unix,Windows", + "sophos.xg.priority": "Warning", + "sophos.xg.rule_priority": "2", + "sophos.xg.src_country_code": "NLD", + "sophos.xg.target": "Server", "source.as.number": 1136, "source.as.organization.name": "KPN B.V.", "source.geo.continent_name": "Europe", @@ -210,7 +210,7 @@ "source.ip": "77.61.185.101", "source.port": 59476, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -226,9 +226,9 @@ "network" ], "event.code": "020703406001", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-23 time=16:20:34 timezone=\"BST\" device_name=\"XG750\" device_id=SFDemo-f64dd6be log_id=020703406001 log_type=\"IDP\" log_component=\"Anomaly\" log_subtype=\"Detect\" priority=Warning idp_policy_id=1 fw_rule_id=2 user_name=\"\" signature_id=26022 signature_msg=\"FILE-PDF EmbeddedFile contained within a PDF\" classification=\"A Network Trojan was detected\" rule_priority=1 src_ip=10.0.0.168 src_country_code=R1 dst_ip=10.1.1.234 dst_country_code=R1 protocol=\"TCP\" src_port=28938 dst_port=25 platform=\"Windows\" category=\"Malware Communication\" target=\"Server\"", "event.outcome": "success", "event.severity": "4", @@ -237,7 +237,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -256,26 +256,26 @@ "rule.name": "FILE-PDF EmbeddedFile contained within a PDF", "server.ip": "10.1.1.234", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.category": "Malware Communication", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG750", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.idp_policy_id": "1", - "sophosxg.firewall.log_component": "Anomaly", - "sophosxg.firewall.log_subtype": "Detect", - "sophosxg.firewall.log_type": "IDP", - "sophosxg.firewall.message_id": "06001", - "sophosxg.firewall.platform": "Windows", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.rule_priority": "1", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.target": "Server", + "service.type": "sophos", + "sophos.xg.category": "Malware Communication", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG750", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.idp_policy_id": "1", + "sophos.xg.log_component": "Anomaly", + "sophos.xg.log_subtype": "Detect", + "sophos.xg.log_type": "IDP", + "sophos.xg.message_id": "06001", + "sophos.xg.platform": "Windows", + "sophos.xg.priority": "Warning", + "sophos.xg.rule_priority": "1", + "sophos.xg.src_country_code": "R1", + "sophos.xg.target": "Server", "source.ip": "10.0.0.168", "source.port": 28938, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -291,9 +291,9 @@ "network" ], "event.code": "020704406002", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2018-05-23 time=16:16:43 timezone=\"BST\" device_name=\"XG750\" device_id=SFDemo-f64dd6be log_id=020704406002 log_type=\"IDP\" log_component=\"Anomaly\" log_subtype=\"Drop\" priority=Warning idp_policy_id=1 fw_rule_id=2 user_name=\"\" signature_id=26022 signature_msg=\"FILE-PDF EmbeddedFile contained within a PDF\" classification=\"A Network Trojan was detected\" rule_priority=1 src_ip=10.0.1.31 src_country_code=R1 dst_ip=10.1.0.115 dst_country_code=R1 protocol=\"TCP\" src_port=40140 dst_port=25 platform=\"Windows\" category=\"Malware Communication\" target=\"Server\"", "event.outcome": "success", "event.severity": "4", @@ -302,7 +302,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "warning", @@ -321,26 +321,26 @@ "rule.name": "FILE-PDF EmbeddedFile contained within a PDF", "server.ip": "10.1.0.115", "server.port": 25, - "service.type": "sophosxg", - "sophosxg.firewall.category": "Malware Communication", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG750", - "sophosxg.firewall.dst_country_code": "R1", - "sophosxg.firewall.fw_rule_id": "2", - "sophosxg.firewall.idp_policy_id": "1", - "sophosxg.firewall.log_component": "Anomaly", - "sophosxg.firewall.log_subtype": "Drop", - "sophosxg.firewall.log_type": "IDP", - "sophosxg.firewall.message_id": "06002", - "sophosxg.firewall.platform": "Windows", - "sophosxg.firewall.priority": "Warning", - "sophosxg.firewall.rule_priority": "1", - "sophosxg.firewall.src_country_code": "R1", - "sophosxg.firewall.target": "Server", + "service.type": "sophos", + "sophos.xg.category": "Malware Communication", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG750", + "sophos.xg.dst_country_code": "R1", + "sophos.xg.fw_rule_id": "2", + "sophos.xg.idp_policy_id": "1", + "sophos.xg.log_component": "Anomaly", + "sophos.xg.log_subtype": "Drop", + "sophos.xg.log_type": "IDP", + "sophos.xg.message_id": "06002", + "sophos.xg.platform": "Windows", + "sophos.xg.priority": "Warning", + "sophos.xg.rule_priority": "1", + "sophos.xg.src_country_code": "R1", + "sophos.xg.target": "Server", "source.ip": "10.0.1.31", "source.port": 40140, "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/sandbox.log b/x-pack/filebeat/module/sophos/xg/test/sandbox.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/sandbox.log rename to x-pack/filebeat/module/sophos/xg/test/sandbox.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/sandbox.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/sandbox.log-expected.json similarity index 70% rename from x-pack/filebeat/module/sophosxg/firewall/test/sandbox.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/sandbox.log-expected.json index 19e1cf7ddae..ed32ee3f213 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/sandbox.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/sandbox.log-expected.json @@ -6,9 +6,9 @@ "network" ], "event.code": "138301618041", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=14:52:11 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=138301618041 log_type=\"Sandbox\" log_component=\"Mail\" log_subtype=\"Allowed\" priority=Information user_name=\"\" src_ip= filename=\"\" filetype=\"\" filesize=0 sha1sum=\"\" source=\"\" reason=\"eligible\" destination=\"\" subject=\"\"", "event.outcome": "success", "event.severity": "6", @@ -19,7 +19,7 @@ "connection" ], "file.size": 0, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -28,17 +28,17 @@ "observer.serial_number": "C44310050024-P29PUA", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.log_component": "Mail", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Sandbox", - "sophosxg.firewall.message_id": "18041", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "eligible", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.log_component": "Mail", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Sandbox", + "sophos.xg.message_id": "18041", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "eligible", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -51,9 +51,9 @@ "network" ], "event.code": "138302218042", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=14:52:11 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=138302218042 log_type=\"Sandbox\" log_component=\"Mail\" log_subtype=\"Denied\" priority=Critical user_name=\"jsmith@iview.com\" src_ip=10.198.47.112 filename=\"1.exe\" filetype=\"application/octet-stream\" filesize=153006 sha1sum=\"83cd339302bf5e8ed5240ca6383418089c337a81\" source=\"jsmith@iview.com\" reason=\"cached malicious\" destination=\"\" subject=\"\"", "event.outcome": "success", "event.severity": "2", @@ -65,7 +65,7 @@ "file.hash.sha1": "83cd339302bf5e8ed5240ca6383418089c337a81", "file.mime_type": "application/octet-stream", "file.size": 153006, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "critical", @@ -83,21 +83,21 @@ "related.user": [ "jsmith@iview.com" ], - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.filename": "1.exe", - "sophosxg.firewall.log_component": "Mail", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Sandbox", - "sophosxg.firewall.message_id": "18042", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.reason": "cached malicious", - "sophosxg.firewall.source": "jsmith@iview.com", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.filename": "1.exe", + "sophos.xg.log_component": "Mail", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Sandbox", + "sophos.xg.message_id": "18042", + "sophos.xg.priority": "Critical", + "sophos.xg.reason": "cached malicious", + "sophos.xg.source": "jsmith@iview.com", "source.ip": "10.198.47.112", "source.user.name": "jsmith@iview.com", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -108,9 +108,9 @@ "network" ], "event.code": "136501618041", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=15:28:25 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44313350024-P29PUA log_id=136501618041 log_type=\"Sandbox\" log_component=\"Web\" log_subtype=\"Allowed\" priority=Information user_name=\"\" src_ip= filename=\"\" filetype=\"\" filesize=0 sha1sum=\"\" source=\"\" reason=\"eligible\" destination=\"\" subject=\"\"", "event.outcome": "success", "event.severity": "6", @@ -121,7 +121,7 @@ "connection" ], "file.size": 0, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -130,17 +130,17 @@ "observer.serial_number": "C44313350024-P29PUA", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.log_component": "Web", - "sophosxg.firewall.log_subtype": "Allowed", - "sophosxg.firewall.log_type": "Sandbox", - "sophosxg.firewall.message_id": "18041", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "eligible", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.log_component": "Web", + "sophos.xg.log_subtype": "Allowed", + "sophos.xg.log_type": "Sandbox", + "sophos.xg.message_id": "18041", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "eligible", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -152,9 +152,9 @@ "network" ], "event.code": "136528618043", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=15:28:25 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=136528618043 log_type=\"Sandbox\" log_component=\"Web\" log_subtype=\"Pending\" priority=Information user_name=\"jsmith\" src_ip=10.198.47.112 filename=\"19.exe\" filetype=\"application/octet-stream\" filesize=153010 sha1sum=\"3ce799580908df9ca0dc649aa8c2d06ab267e8c8\" source=\"10.198.241.50\" reason=\"pending\" destination=\"\" subject=\"\"", "event.outcome": "success", "event.severity": "6", @@ -166,7 +166,7 @@ "file.hash.sha1": "3ce799580908df9ca0dc649aa8c2d06ab267e8c8", "file.mime_type": "application/octet-stream", "file.size": 153010, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -184,21 +184,21 @@ "related.user": [ "jsmith" ], - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.filename": "19.exe", - "sophosxg.firewall.log_component": "Web", - "sophosxg.firewall.log_subtype": "Pending", - "sophosxg.firewall.log_type": "Sandbox", - "sophosxg.firewall.message_id": "18043", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "pending", - "sophosxg.firewall.source": "10.198.241.50", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.filename": "19.exe", + "sophos.xg.log_component": "Web", + "sophos.xg.log_subtype": "Pending", + "sophos.xg.log_type": "Sandbox", + "sophos.xg.message_id": "18043", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "pending", + "sophos.xg.source": "10.198.241.50", "source.ip": "10.198.47.112", "source.user.name": "jsmith", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -211,9 +211,9 @@ "network" ], "event.code": "136502218042", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-01-31 time=15:28:25 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=136502218042 log_type=\"Sandbox\" log_component=\"Web\" log_subtype=\"Denied\" priority=Critical user_name=\"jsmith\" src_ip=10.198.47.112 filename=\"19.exe\" filetype=\"application/octet-stream\" filesize=153010 sha1sum=\"3ce799580908df9ca0dc649aa8c2d06ab267e8c8\" source=\"10.198.241.50\" reason=\"cloud malicious\" destination=\"\" subject=\"", "event.outcome": "success", "event.severity": "2", @@ -225,7 +225,7 @@ "file.hash.sha1": "3ce799580908df9ca0dc649aa8c2d06ab267e8c8", "file.mime_type": "application/octet-stream", "file.size": 153010, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "critical", @@ -243,21 +243,21 @@ "related.user": [ "jsmith" ], - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.filename": "19.exe", - "sophosxg.firewall.log_component": "Web", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Sandbox", - "sophosxg.firewall.message_id": "18042", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.reason": "cloud malicious", - "sophosxg.firewall.source": "10.198.241.50", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.filename": "19.exe", + "sophos.xg.log_component": "Web", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Sandbox", + "sophos.xg.message_id": "18042", + "sophos.xg.priority": "Critical", + "sophos.xg.reason": "cloud malicious", + "sophos.xg.source": "10.198.241.50", "source.ip": "10.198.47.112", "source.user.name": "jsmith", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, @@ -270,9 +270,9 @@ "network" ], "event.code": "136502218042", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:36 timezone=\"IST\" device_name=\"CR750iNG-XP\" device_id=C44310050024-P29PUA log_id=136502218042 log_type=\"Sandbox\" log_component=\"Web\" log_subtype=\"Denied\" priority=Critical user_name=\"\" src_ip=172.16.34.24 filename=\"SBTestFile1.pdf\" filetype=\"application/pdf\" filesize=1124 sha1sum=\"d910c4a81122c360fe57f67a04999425a65249db\" source=\"sophostest.com\" reason=\"cached malicious\" destination=\"\" subject=\"\"", "event.outcome": "success", "event.severity": "2", @@ -284,7 +284,7 @@ "file.hash.sha1": "d910c4a81122c360fe57f67a04999425a65249db", "file.mime_type": "application/pdf", "file.size": 1124, - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "critical", @@ -299,20 +299,20 @@ "related.ip": [ "172.16.34.24" ], - "service.type": "sophosxg", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "CR750iNG-XP", - "sophosxg.firewall.filename": "SBTestFile1.pdf", - "sophosxg.firewall.log_component": "Web", - "sophosxg.firewall.log_subtype": "Denied", - "sophosxg.firewall.log_type": "Sandbox", - "sophosxg.firewall.message_id": "18042", - "sophosxg.firewall.priority": "Critical", - "sophosxg.firewall.reason": "cached malicious", - "sophosxg.firewall.source": "sophostest.com", + "service.type": "sophos", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "CR750iNG-XP", + "sophos.xg.filename": "SBTestFile1.pdf", + "sophos.xg.log_component": "Web", + "sophos.xg.log_subtype": "Denied", + "sophos.xg.log_type": "Sandbox", + "sophos.xg.message_id": "18042", + "sophos.xg.priority": "Critical", + "sophos.xg.reason": "cached malicious", + "sophos.xg.source": "sophostest.com", "source.ip": "172.16.34.24", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/waf.log b/x-pack/filebeat/module/sophos/xg/test/waf.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/waf.log rename to x-pack/filebeat/module/sophos/xg/test/waf.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/waf.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json similarity index 72% rename from x-pack/filebeat/module/sophosxg/firewall/test/waf.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json index b49dfde3ca4..fe6af644611 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/waf.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json @@ -20,9 +20,9 @@ "network" ], "event.code": "075000617071", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:46 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=075000617071 log_type=\"WAF\" log_component=\"Web Application Firewall\" priority=Information user_name=\"-\" server=webmail.elasticuser.com sourceip=89.68.140.204 localip=185.8.209.207 ws_protocol=\"HTTP/1.1\" url=/mapi/nspi/ querystring=?MailboxId=642ea57c-90ab-4571-8e05-c6997b2256f8@elastic.user.com cookie=\"MapiContext=MAPIAAAAAPaw98Xx0uDQ4tL/z/rX59b2x/LI+8z2x/+lhrKGtIO7jbmBsxYNAAAAAAAA;MapiRouting=UlVNOjdmNWY0OGE3LTM5OWItNDc4Yi04ZDgwLWFmZTRmMzAyZTViMDoAitM4bv3XCA==;MapiSequence=10-GtgsIA==;X-BackEndCookie=642ea57c-90ab-4571-8e05-c6997b2256f8=u56Lnp2ejJqByZrHyZ7Mys7Sm8zJnNLLnM3J0sbJxsvSnZzIxs2ans+ezMrLgYHNz83P0s/J0s3Pq87Pxc7PxcrL\" referer=- method=POST httpstatus=401 reason=\"-\" extra=\"-\" contenttype=\"-\" useragent=\"Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.4954; Pro)\" host=89.68.140.204 responsetime=11199 bytessent=5669 bytesrcv=1419 fw_rule_id=79", "event.severity": "6", "event.timezone": "-02:00", @@ -30,7 +30,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "http.request.method": "POST", "http.version": "HTTP/1.1", @@ -47,19 +47,19 @@ ], "server.bytes": 5669, "server.ip": "185.8.209.207", - "service.type": "sophosxg", - "sophosxg.firewall.cookie": "MapiContext=MAPIAAAAAPaw98Xx0uDQ4tL/z/rX59b2x/LI+8z2x/+lhrKGtIO7jbmBsxYNAAAAAAAA;MapiRouting=UlVNOjdmNWY0OGE3LTM5OWItNDc4Yi04ZDgwLWFmZTRmMzAyZTViMDoAitM4bv3XCA==;MapiSequence=10-GtgsIA==;X-BackEndCookie=642ea57c-90ab-4571-8e05-c6997b2256f8=u56Lnp2ejJqByZrHyZ7Mys7Sm8zJnNLLnM3J0sbJxsvSnZzIxs2ans+ezMrLgYHNz83P0s/J0s3Pq87Pxc7PxcrL", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.fw_rule_id": "79", - "sophosxg.firewall.host": "89.68.140.204", - "sophosxg.firewall.log_component": "Web Application Firewall", - "sophosxg.firewall.log_type": "WAF", - "sophosxg.firewall.message_id": "17071", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.querystring": "?MailboxId=642ea57c-90ab-4571-8e05-c6997b2256f8@elastic.user.com", - "sophosxg.firewall.responsetime": "11199", - "sophosxg.firewall.server": "webmail.elasticuser.com", + "service.type": "sophos", + "sophos.xg.cookie": "MapiContext=MAPIAAAAAPaw98Xx0uDQ4tL/z/rX59b2x/LI+8z2x/+lhrKGtIO7jbmBsxYNAAAAAAAA;MapiRouting=UlVNOjdmNWY0OGE3LTM5OWItNDc4Yi04ZDgwLWFmZTRmMzAyZTViMDoAitM4bv3XCA==;MapiSequence=10-GtgsIA==;X-BackEndCookie=642ea57c-90ab-4571-8e05-c6997b2256f8=u56Lnp2ejJqByZrHyZ7Mys7Sm8zJnNLLnM3J0sbJxsvSnZzIxs2ans+ezMrLgYHNz83P0s/J0s3Pq87Pxc7PxcrL", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.fw_rule_id": "79", + "sophos.xg.host": "89.68.140.204", + "sophos.xg.log_component": "Web Application Firewall", + "sophos.xg.log_type": "WAF", + "sophos.xg.message_id": "17071", + "sophos.xg.priority": "Information", + "sophos.xg.querystring": "?MailboxId=642ea57c-90ab-4571-8e05-c6997b2256f8@elastic.user.com", + "sophos.xg.responsetime": "11199", + "sophos.xg.server": "webmail.elasticuser.com", "source.as.number": 6830, "source.as.organization.name": "Liberty Global B.V.", "source.bytes": 1419, @@ -72,7 +72,7 @@ "source.geo.region_name": "Pomerania", "source.ip": "89.68.140.204", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.full": "/mapi/nspi/", @@ -99,9 +99,9 @@ "network" ], "event.code": "075000617071", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-18 time=14:38:47 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123457 log_id=075000617071 log_type=\"WAF\" log_component=\"Web Application Firewall\" priority=Information user_name=\"-\" server=webmail.elasticuser.com sourceip=89.68.140.204 localip=185.8.209.207 ws_protocol=\"HTTP/1.1\" url=/mapi/nspi/ querystring=?MailboxId=642ea57c-90ab-4571-8e05-c6997b2256f8@elastic.user.com cookie=\"MapiContext=MAPIAAAAAPaw98Xx0uDQ4tL/z/rX59b2x/LI+8z2x/+lhrKGtIO7jbmBsw0NAAAAAAAA;MapiRouting=UlVNOjdmNWY0OGE3LTM5OWItNDc4Yi04ZDgwLWFmZTRmMzAyZTViMDpeyft5bv3XCA==;MapiSequence=9-Km2JMg==;X-BackEndCookie=642ea57c-90ab-4571-8e05-c6997b2256f8=u56Lnp2ejJqByZrHyZ7Mys7Sm8zJnNLLnM3J0sbJxsvSnZzIxs2ans+ezMrLgYHNz83P0s/J0s3Pq87Pxc7Oxc3M\" referer=- method=POST httpstatus=200 reason=\"-\" extra=\"-\" contenttype=\"application/mapi-http\" useragent=\"Microsoft Office/16.0 (Windows NT 6.2; Microsoft Outlook 16.0.4954; Pro)\" host=89.68.140.204 responsetime=14086 bytessent=1357 bytesrcv=1774 fw_rule_id=79", "event.severity": "6", "event.timezone": "-02:00", @@ -109,7 +109,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "http.request.method": "POST", "http.version": "HTTP/1.1", @@ -126,20 +126,20 @@ ], "server.bytes": 1357, "server.ip": "185.8.209.207", - "service.type": "sophosxg", - "sophosxg.firewall.contenttype": "application/mapi-http", - "sophosxg.firewall.cookie": "MapiContext=MAPIAAAAAPaw98Xx0uDQ4tL/z/rX59b2x/LI+8z2x/+lhrKGtIO7jbmBsw0NAAAAAAAA;MapiRouting=UlVNOjdmNWY0OGE3LTM5OWItNDc4Yi04ZDgwLWFmZTRmMzAyZTViMDpeyft5bv3XCA==;MapiSequence=9-Km2JMg==;X-BackEndCookie=642ea57c-90ab-4571-8e05-c6997b2256f8=u56Lnp2ejJqByZrHyZ7Mys7Sm8zJnNLLnM3J0sbJxsvSnZzIxs2ans+ezMrLgYHNz83P0s/J0s3Pq87Pxc7Oxc3M", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.fw_rule_id": "79", - "sophosxg.firewall.host": "89.68.140.204", - "sophosxg.firewall.log_component": "Web Application Firewall", - "sophosxg.firewall.log_type": "WAF", - "sophosxg.firewall.message_id": "17071", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.querystring": "?MailboxId=642ea57c-90ab-4571-8e05-c6997b2256f8@elastic.user.com", - "sophosxg.firewall.responsetime": "14086", - "sophosxg.firewall.server": "webmail.elasticuser.com", + "service.type": "sophos", + "sophos.xg.contenttype": "application/mapi-http", + "sophos.xg.cookie": "MapiContext=MAPIAAAAAPaw98Xx0uDQ4tL/z/rX59b2x/LI+8z2x/+lhrKGtIO7jbmBsw0NAAAAAAAA;MapiRouting=UlVNOjdmNWY0OGE3LTM5OWItNDc4Yi04ZDgwLWFmZTRmMzAyZTViMDpeyft5bv3XCA==;MapiSequence=9-Km2JMg==;X-BackEndCookie=642ea57c-90ab-4571-8e05-c6997b2256f8=u56Lnp2ejJqByZrHyZ7Mys7Sm8zJnNLLnM3J0sbJxsvSnZzIxs2ans+ezMrLgYHNz83P0s/J0s3Pq87Pxc7Oxc3M", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.fw_rule_id": "79", + "sophos.xg.host": "89.68.140.204", + "sophos.xg.log_component": "Web Application Firewall", + "sophos.xg.log_type": "WAF", + "sophos.xg.message_id": "17071", + "sophos.xg.priority": "Information", + "sophos.xg.querystring": "?MailboxId=642ea57c-90ab-4571-8e05-c6997b2256f8@elastic.user.com", + "sophos.xg.responsetime": "14086", + "sophos.xg.server": "webmail.elasticuser.com", "source.as.number": 6830, "source.as.organization.name": "Liberty Global B.V.", "source.bytes": 1774, @@ -152,7 +152,7 @@ "source.geo.region_name": "Pomerania", "source.ip": "89.68.140.204", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.full": "/mapi/nspi/", @@ -170,9 +170,9 @@ "network" ], "event.code": "075000617071", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-19 time=17:20:29 timezone=\"IST\" device_name=\"XG230\" device_id=1234567890123457 log_id=075000617071 log_type=\"WAF\" log_component=\"Web Application Firewall\" priority=Information user_name=\"jsmith\" server=www.iviewtest.com:8989 sourceip=10.198.235.254 localip=10.198.233.48 ws_protocol=\"HTTP/1.1\" url=/ querystring= cookie=\"-\" referer=- method=GET httpstatus=403 reason=\"Static URL Hardening\" extra=\"No signature found\" contenttype=\"text/html\" useragent=\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0\" host=10.198.235.254 responsetime=19310 bytessent=726 bytesrcv=510 fw_rule_id=3", "event.outcome": "success", "event.severity": "6", @@ -181,7 +181,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "http.request.method": "GET", "http.version": "HTTP/1.1", @@ -201,25 +201,25 @@ ], "server.bytes": 726, "server.ip": "10.198.233.48", - "service.type": "sophosxg", - "sophosxg.firewall.contenttype": "text/html", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.extra": "No signature found", - "sophosxg.firewall.fw_rule_id": "3", - "sophosxg.firewall.host": "10.198.235.254", - "sophosxg.firewall.log_component": "Web Application Firewall", - "sophosxg.firewall.log_type": "WAF", - "sophosxg.firewall.message_id": "17071", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "Static URL Hardening", - "sophosxg.firewall.responsetime": "19310", - "sophosxg.firewall.server": "www.iviewtest.com:8989", + "service.type": "sophos", + "sophos.xg.contenttype": "text/html", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.extra": "No signature found", + "sophos.xg.fw_rule_id": "3", + "sophos.xg.host": "10.198.235.254", + "sophos.xg.log_component": "Web Application Firewall", + "sophos.xg.log_type": "WAF", + "sophos.xg.message_id": "17071", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "Static URL Hardening", + "sophos.xg.responsetime": "19310", + "sophos.xg.server": "www.iviewtest.com:8989", "source.bytes": 510, "source.ip": "10.198.235.254", "source.user.name": "jsmith", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.full": "/", @@ -237,9 +237,9 @@ "network" ], "event.code": "075000617071", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-19 time=18:03:30 timezone=\"IST\" device_name=\"XG230\" device_id=1234567890123456 log_id=075000617071 log_type=\"WAF\" log_component=\"Web Application Firewall\" priority=Information user_name=\"jsmith\" server=www.iviewtest.com:8990 sourceip=10.198.235.254 localip=10.198.233.48 ws_protocol=\"HTTP/1.1\" url=/download/eicarcom2.zip querystring= cookie=\"; PHPSESSID=jetkd9iadd969hsr77jpj4q974; _pk_id.1.fc3a=3a6250e215194a92.1485866024.1.1485866069.1485866024.; _pk_ses.1.fc3a=*\" referer=http://www.iviewtest.com:8990/85-0-Download.html method=GET httpstatus=403 reason=\"Antivirus\" extra=\"EICAR-AV-Test\" contenttype=\"text/html\" useragent=\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0\" host=10.198.235.254 responsetime=403214 bytessent=739 bytesrcv=715 fw_rule_id=6", "event.outcome": "success", "event.severity": "6", @@ -248,7 +248,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "my_fancy_host", "http.request.method": "GET", "http.request.referrer": "http://www.iviewtest.com:8990/85-0-Download.html", @@ -269,26 +269,26 @@ ], "server.bytes": 739, "server.ip": "10.198.233.48", - "service.type": "sophosxg", - "sophosxg.firewall.contenttype": "text/html", - "sophosxg.firewall.cookie": "; PHPSESSID=jetkd9iadd969hsr77jpj4q974; _pk_id.1.fc3a=3a6250e215194a92.1485866024.1.1485866069.1485866024.; _pk_ses.1.fc3a=*", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.extra": "EICAR-AV-Test", - "sophosxg.firewall.fw_rule_id": "6", - "sophosxg.firewall.host": "10.198.235.254", - "sophosxg.firewall.log_component": "Web Application Firewall", - "sophosxg.firewall.log_type": "WAF", - "sophosxg.firewall.message_id": "17071", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "Antivirus", - "sophosxg.firewall.responsetime": "403214", - "sophosxg.firewall.server": "www.iviewtest.com:8990", + "service.type": "sophos", + "sophos.xg.contenttype": "text/html", + "sophos.xg.cookie": "; PHPSESSID=jetkd9iadd969hsr77jpj4q974; _pk_id.1.fc3a=3a6250e215194a92.1485866024.1.1485866069.1485866024.; _pk_ses.1.fc3a=*", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.extra": "EICAR-AV-Test", + "sophos.xg.fw_rule_id": "6", + "sophos.xg.host": "10.198.235.254", + "sophos.xg.log_component": "Web Application Firewall", + "sophos.xg.log_type": "WAF", + "sophos.xg.message_id": "17071", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "Antivirus", + "sophos.xg.responsetime": "403214", + "sophos.xg.server": "www.iviewtest.com:8990", "source.bytes": 715, "source.ip": "10.198.235.254", "source.user.name": "jsmith", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.full": "/download/eicarcom2.zip", @@ -312,9 +312,9 @@ "network" ], "event.code": "075000617071", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "alert", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2020-05-20 time=18:03:31 timezone=\"IST\" device_name=\"XG230\" device_id=1234567890123457 log_id=075000617071 log_type=\"WAF\" log_component=\"Web Application Firewall\" priority=Information user_name=\"-\" server=- sourceip=83.97.20.30 localip=216.167.51.72 ws_protocol=\"HTTP/1.0\" url=/ querystring=\"\" cookie=\"-\" referer=\"-\" method=GET httpstatus=403 reason=\"WAF Anomaly\" extra=\"Inbound Anomaly Score Exceeded (Total Score: 7, SQLi=, XSS=): Last Matched Message: Request Missing a User Agent Header\" contenttype=\"text/html\" useragent=\"-\" host=83.97.20.30 responsetime=608 bytessent=5353 bytesrcv=295 fw_rule_id=3", "event.outcome": "success", "event.severity": "6", @@ -323,7 +323,7 @@ "denied", "connection" ], - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "some_other_host.local", "http.request.method": "GET", "http.version": "HTTP/1.0", @@ -340,19 +340,19 @@ ], "server.bytes": 5353, "server.ip": "216.167.51.72", - "service.type": "sophosxg", - "sophosxg.firewall.contenttype": "text/html", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "XG230", - "sophosxg.firewall.extra": "Inbound Anomaly Score Exceeded (Total Score: 7, SQLi=, XSS=): Last Matched Message: Request Missing a User Agent Header", - "sophosxg.firewall.fw_rule_id": "3", - "sophosxg.firewall.host": "83.97.20.30", - "sophosxg.firewall.log_component": "Web Application Firewall", - "sophosxg.firewall.log_type": "WAF", - "sophosxg.firewall.message_id": "17071", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.reason": "WAF Anomaly", - "sophosxg.firewall.responsetime": "608", + "service.type": "sophos", + "sophos.xg.contenttype": "text/html", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "XG230", + "sophos.xg.extra": "Inbound Anomaly Score Exceeded (Total Score: 7, SQLi=, XSS=): Last Matched Message: Request Missing a User Agent Header", + "sophos.xg.fw_rule_id": "3", + "sophos.xg.host": "83.97.20.30", + "sophos.xg.log_component": "Web Application Firewall", + "sophos.xg.log_type": "WAF", + "sophos.xg.message_id": "17071", + "sophos.xg.priority": "Information", + "sophos.xg.reason": "WAF Anomaly", + "sophos.xg.responsetime": "608", "source.as.number": 9009, "source.as.organization.name": "M247 Ltd", "source.bytes": 295, @@ -365,7 +365,7 @@ "source.geo.region_name": "Bucuresti", "source.ip": "83.97.20.30", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ], "url.full": "/" diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/wifi.log b/x-pack/filebeat/module/sophos/xg/test/wifi.log similarity index 100% rename from x-pack/filebeat/module/sophosxg/firewall/test/wifi.log rename to x-pack/filebeat/module/sophos/xg/test/wifi.log diff --git a/x-pack/filebeat/module/sophosxg/firewall/test/wifi.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/wifi.log-expected.json similarity index 56% rename from x-pack/filebeat/module/sophosxg/firewall/test/wifi.log-expected.json rename to x-pack/filebeat/module/sophos/xg/test/wifi.log-expected.json index 53bd653f02e..64aa8a24494 100644 --- a/x-pack/filebeat/module/sophosxg/firewall/test/wifi.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/wifi.log-expected.json @@ -2,14 +2,14 @@ { "@timestamp": "2017-02-01T14:17:35.000-02:00", "event.code": "106025618011", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-02-01 time=14:17:35 timezone=\"IST\" device_name=\"SG115\" device_id=S110016E28BA631 log_id=106025618011 log_type=\"Wireless Protection\" log_component=\"Wireless Protection\" log_subtype=\"Information\" priority=Information ap=A40024A636F7862 ssid=SPIDIGO2015 clients_conn_ssid=2", "event.outcome": "success", "event.severity": "6", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -18,33 +18,33 @@ "observer.serial_number": "S110016E28BA631", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.ap": "A40024A636F7862", - "sophosxg.firewall.clients_conn_ssid": "2", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG115", - "sophosxg.firewall.log_component": "Wireless Protection", - "sophosxg.firewall.log_subtype": "Information", - "sophosxg.firewall.log_type": "Wireless Protection", - "sophosxg.firewall.message_id": "18011", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.ssid": "SPIDIGO2015", + "service.type": "sophos", + "sophos.xg.ap": "A40024A636F7862", + "sophos.xg.clients_conn_ssid": "2", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG115", + "sophos.xg.log_component": "Wireless Protection", + "sophos.xg.log_subtype": "Information", + "sophos.xg.log_type": "Wireless Protection", + "sophos.xg.message_id": "18011", + "sophos.xg.priority": "Information", + "sophos.xg.ssid": "SPIDIGO2015", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] }, { "@timestamp": "2017-02-01T14:19:47.000-02:00", "event.code": "106025618011", - "event.dataset": "sophosxg.firewall", + "event.dataset": "sophos.xg", "event.kind": "event", - "event.module": "sophosxg", + "event.module": "sophos", "event.original": "device=\"SFW\" date=2017-02-01 time=14:19:47 timezone=\"IST\" device_name=\"SG115\" device_id=S110016E28BA631 log_id=106025618011 log_type=\"Wireless Protection\" log_component=\"Wireless Protection\" log_subtype=\"Information\" priority=Information ap=A40024A636F7862 ssid=SPIDIGO2015 clients_conn_ssid=3", "event.outcome": "success", "event.severity": "6", "event.timezone": "-02:00", - "fileset.name": "firewall", + "fileset.name": "xg", "host.name": "firewall.localgroup.local", "input.type": "log", "log.level": "informational", @@ -53,19 +53,19 @@ "observer.serial_number": "S110016E28BA631", "observer.type": "firewall", "observer.vendor": "Sophos", - "service.type": "sophosxg", - "sophosxg.firewall.ap": "A40024A636F7862", - "sophosxg.firewall.clients_conn_ssid": "3", - "sophosxg.firewall.device": "SFW", - "sophosxg.firewall.device_name": "SG115", - "sophosxg.firewall.log_component": "Wireless Protection", - "sophosxg.firewall.log_subtype": "Information", - "sophosxg.firewall.log_type": "Wireless Protection", - "sophosxg.firewall.message_id": "18011", - "sophosxg.firewall.priority": "Information", - "sophosxg.firewall.ssid": "SPIDIGO2015", + "service.type": "sophos", + "sophos.xg.ap": "A40024A636F7862", + "sophos.xg.clients_conn_ssid": "3", + "sophos.xg.device": "SFW", + "sophos.xg.device_name": "SG115", + "sophos.xg.log_component": "Wireless Protection", + "sophos.xg.log_subtype": "Information", + "sophos.xg.log_type": "Wireless Protection", + "sophos.xg.message_id": "18011", + "sophos.xg.priority": "Information", + "sophos.xg.ssid": "SPIDIGO2015", "tags": [ - "sophosxg-firewall", + "sophos-xg", "forwarded" ] } diff --git a/x-pack/filebeat/module/sophosxg/fields.go b/x-pack/filebeat/module/sophosxg/fields.go deleted file mode 100644 index d564c5e5a31..00000000000 --- a/x-pack/filebeat/module/sophosxg/fields.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. - -package sophosxg - -import ( - "github.com/elastic/beats/v7/libbeat/asset" -) - -func init() { - if err := asset.SetFields("filebeat", "sophosxg", asset.ModuleFieldsPri, AssetSophosxg); err != nil { - panic(err) - } -} - -// AssetSophosxg returns asset data. -// This is the base64 encoded gzipped contents of module/sophosxg. -func AssetSophosxg() string { - return "eJzUnM1y4zYSx+/zFKjkMqnKeHf26MNWObZnx1XjiVaSkyMLAlsk1iDAAKBk+TTvsNfk5eZJtgB+mBRBkZZayezkZMv594+NRqPRAPWOPMLukhiVp8o8JW8IsdwKuCTf1b/67g0hMRimeW65kpfkn28IIc3/Qe5VXAh4Q8iag4jNpf/0HZE0g46u+2d3OVySRKsir34TUO4qtdXWXMOWCtF8EFJ0/zQIoAYuyQosbf0+hjUthI28gUuypsJA5+MATPmvfEqyVprkVBsukxcPmJ0RKrlo/f3+A7QfIoYNZ9D5qH6MR9htlY73PjtAVX7c0wvbpTZsNfDBiMkbaoG83e12u3dZ9i6OfyDbFCSxKRDYgLREMVZoDS9PEiSyPINnJRF9seQZkLdpeplll8aMYw07yzs1cj/g0d2rGASRRbYCTdTag00evRKI75s8AWcBmtMRniCLUAkqyIPkvxVA3v+DsJRqyixoQ5iKgbz9+3v33/sfxh3koBwDYjDtcnBuKWMHLpKLJgGVv5sGxVSWK7n/5yeRXdeSRIPJlTR8VeUmoZLE5SZP+6Gm1WWCngBrihWuExfFyv/vjSMPh1a6ilKgwqZ4BB+BarsCaomx1BbmMECuudLc7jCn2QacIhGwAeHnmabrNWfjI7IHfDLKg7A8c7m7FG6xkK9f/kuuhFBbiInS5AYkH0vfcaGpsxTkE0omr1xUCk1XXDhPtbjeGmBKxmZCBlhvIxfoQ6mJSwsJ6NdBNVNo7lb/uxuyTTlLCTeE5rngzlnlAjN5TAsDGnldCUsOW9+vlk40/y8n5/WJVZWDnEucLbICFwkjk45TRJw7aUFLsOSKMTCG5EpwtnNjNzJkYbTcRKUCalzdzRYnglVQuNl6ViL5hF1DWTUdiub5mgsL+hw+u3JAzKcc8sFbqV14jP/oi1rUYf4/AMYb8DZiJ4OM+gw3hR3Pobl5xOOYc/NYLdTUGJ7Ilwkw5P9RQgsslUqoBLGoWDaadeEewhtFY9RCojQiWHsguXH1qRIbiMlqR5w7qS00uALD7CRLtZL8GeJXsdeKq2/GmxE3EROqQNwR7XnxuqMeXqdk5HKUXlPM3sJdLek3F1wylbndRZWvfvTbjB/JTGlLrg7zqcKeH1AVNlFDgD9NqLc1i3i4FOn9egTsZ80TLqkgRhWaAbmbERrH2hUkgR3AIE1GGZ6z9pnur64PQR0EY6qQVu8it03H3NjGUE/BykK3pnzx5n5hOUgbG4s+qDEYy2U5Pw+P7EGsv8iJe/STSvRcK6uYEoi1ZqXYbkBNnRa50uF2ylF12v68cOoOZ3k9I1TG5OFmNg3NjemZ0NpjNoVvkJGzLEfeNdxd38+aJk/5w6StlSPBDX1vnFXxHyQZpDEgbZQ/2nC75fXtjKWyneZqTtkjWOPtjNNoYMA3EP8ZRLWtiT5a7SycC8lrd1005J7NnwDS80wQxmoqTeTSUn9BOW6dWTpFQS3EgfIhVOhMBcRNTn1Mn5kOAo5AIi7WLbqB5fpYT+Kn+QHWE9xpNHtWEs5zIlKNtjNQl9ifrj5PhsID+kyzfaBJldf5XNMevrZ/fm355xAVvm/2iUYwuI5ibhD7scuZX2ecMjD3Rxdkpkx5ZLWhogBz+fXL70onX7/88SP5+uV3DbnY1T98/fLH+IgyJWX/RO4k6tvywFg2ZTM3RKjE7cYTkKDdXJ0GhtoDrY5KeQzS8jUvFyv/+Gy8V7FBx7luLJO7m3rPkVFjQU+m4nF+/g77ycc2LUrczmeL0x+h8Au46P1yKn4469Z9N8xD+0XTzLu7mWo+M/tV2SnHqg1ABsYYmkzI/ExQY/gavYv+wnLAQriQLQRE+GfOs0rRb4OaWDo8Trmgdq10hkhRKdaJoYrWi8Mc+D1p54GX5nOtP4JhqU4AcUE5zhl+VcO8a3O1nPm1DWx74g7bz6OiwLR/K+NccWnJw8MU+zbVQC3yWdNyFtINJy9fWH47PeFWMffX9TSDZEIlXEaF6a3TJ46U1yUd3eGZglvTO/Pl5a+Obtj8E7DCn6nlFPP2kmeotUlHO8hBN+cpU+6p2FINxDAqpduJHlOeDFKvtcoiyCgXURV3iIszyBg08eokpB5OPOpcPHNwCQVcDgwzhRNRsfoPMMwNjre9Lxu07f7S8OdwNB1VnPcUw/sVrjFvv3m5CbcLPixnUaERTzo+LGfkYf7JR3k1V0qWLTUkVlspFI3HuoyOqtlC47Hd1JIu0TtQ3+Jag74kD7nj8rcBK8bDhGsuADdMFvy5Ocly6sSm1LqNpaVcQhwIkEEw3Kw8ozbtgFVMLjF2oQaBcNPzBwfhczE1RjHuW4lbbtOX2+8T/GRzprKMSsR6z4VUJeoW8bi8kv8S/mtVyJG4Qp2LJ8/DWGWUS9zRu/Gap3H9VlBNpeWYLUQf5m7k6oDtxPyLxRE0o1l0DrdVK3spPSGxx8aehWNeHhO9gkQDNZhZfO71yDbd+eHRwJSOy/ABC8wlA2qIyWn2t4wKzrgaS1Ea1qAxa/n5nmA4UnKaUeQVbpHTjFxNaES6ogRzq7unNzJrI+yY+HcjvR9uYdf7VwKQz+MX5XsGPdEggdqA1jyGyKpHQHTEz5Uu6esGOZj6hnr2jVNoYVOl+TPmnGw8ExAP74pccVhOUMSZsnxRbR0jXP3iN5sjDbLCF6mRW5CQb7VU5W9Zd47u/9scuKtLm2N8ZUmtzauXwmBwLr/+fkR9p+bjcjmrXzob84h/1eRPfs9kYm/5TG8K1vKk8IVJibiFleEWyJoKMbLo1rXeOcK53sNNDWh4YuDF8Hbgn7ixzcXEFNijcVZEEZf3w7ewalkdG0klLXrDrx7H8uakNzAxO+NO+SYvT3gVglm+4XaHC/ArrJpXVSoD5d47o5alrpqUMWHUb+ics6q/1WAKYUdGrhvk2BupdpBPaO+n9L0pEI+8Fh+v3lexXTRHPdxCRlbAZUKopGL3PLZhysAfaKK+ZH1fao4e17rCBNNu54rApMdGf+aRROfbvGfqry6m9Ff9RMBtUPl+0PhpQWxsuWdFnIKtY6Sbrvbww+O2Df3DL0bby61iFnWmdcvZwwxVcYCauTuCwz7Hb0CO51u/qD7ZKNew5k+oScYt1mTW1Q3nfG7hDC++LVyNd70ve9ALpljje+HJkkVXNzzzuQ9QqrEPCm8aYX8Dcbx6wex5lLf2pjR7cA+qH0aPp+sx9wUU/pDfO1kyNuTnOz0aGebCphETHPWG5lVhU3LdFR22ngFLqeQGsdbzAH3dwaqq9BTubLse1g1yaMiUBQl2qzTia9Nh2SCBUIyKhFrYUsTs+ymkOmwf3QGfQqojkYC7d70O6wYZlIg3oA1qKvi5rxm0LWGLbvtzXzNom+fotznuepLhUfd5KsrTneGMnuFWSZkISW1g2qWSiipVxiJXARWOU55yHEa3UUw739R3ov053ZKOYtDuPephR0ctXHqCMbitc7LoSYYtW6qt5QND/PpvAFz09A4sN1hvdM29GrmbHTbqsIylWdjo65+1rxc0u9iZTyqJFrfzX27n0eer+1vEUfbapNQmHe0gy4qyxyKPMtT4/smLkmw8zP31VsSn7+oNTC29wazoF129oE2X3fAsfmyrDcym8txlcB6//lxnHpIcqFvUI0cc1D29oM3fCtA7YzXvPdYJhkOiQevwZDXiktSVC1qcfZwtbheLvYvxp13h6UkOrw8R9gJBxsOqvLaNZ7fsAExZmWLUpt/89ma02bfSVLIUudL6yYtO6LUUufNmjLQW99XCFXcsIGJ5EbS5ForuJ9ARs06PfP/9SDDvjIUM02ypOOJfA/ovsIl4UaWjFrYnOeKK11ELl3LKUhFlkA21ao97295/2cN9VzVof60B8VyioxZ+Xk2lybi1EIPWSiNuEFvSZE97IEGW332BzVHrvgriccXRvgalAeiKjo0HJkJ7LF5HEWuVn4Oiq3twLDARmrGYYJ8pIbgZvJByzJ2mnmB4DcP/2rq+ZNDytZJrnhz6muUj1pWwaNA+mUOu9EDYH2G6pxe02rxNjWW2Lxi0u4SBlsERJjtag+UCYlrtyoXLIoNZ8HbUgvYwv9SZjniz7GSayH/DBe6DVtKtxn3Ik4S8+V8AAAD//9hwygA=" -} diff --git a/x-pack/filebeat/module/sophosxg/firewall/ingest/systemhealth.yml b/x-pack/filebeat/module/sophosxg/firewall/ingest/systemhealth.yml deleted file mode 100644 index 78e83f1e2ee..00000000000 --- a/x-pack/filebeat/module/sophosxg/firewall/ingest/systemhealth.yml +++ /dev/null @@ -1,158 +0,0 @@ -description: Pipeline for parsing sophos firewall logs (systemhealth pipeline) -processors: -####################### -## ECS Event Mapping ## -####################### -#TODO: Need to setup a different field naming convention, maybe "cpu.idle, cpu.system etc" -- set: - field: event.kind - value: event -- rename: - field: sophosxg.firewall.idle - target_field: sophosxg.firewall.idle_cpu - ignore_missing: true - if: "ctx.sophosxg?.firewall?.idle !=null" -- gsub: - field: sophosxg.firewall.idle_cpu - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "CPU"' -- convert: - field: sophosxg.firewall.idle_cpu - target_field: sophosxg.firewall.idle_cpu - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.idle_cpu != null" -- rename: - field: sophosxg.firewall.system - target_field: sophosxg.firewall.system_cpu - ignore_missing: true - if: "ctx.sophosxg?.firewall?.system !=null" -- gsub: - field: sophosxg.firewall.system_cpu - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "CPU"' -- convert: - field: sophosxg.firewall.system_cpu - target_field: sophosxg.firewall.system_cpu - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.system_cpu != null" -- rename: - field: sophosxg.firewall.user - target_field: sophosxg.firewall.user_cpu - ignore_missing: true - if: "ctx.sophosxg?.firewall?.user !=null" -- gsub: - field: sophosxg.firewall.user_cpu - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "CPU"' -- convert: - field: sophosxg.firewall.user_cpu - target_field: sophosxg.firewall.user_cpu - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.user_cpu != null" -- convert: - field: sophosxg.firewall.used - target_field: sophosxg.firewall.used - type: integer - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.used != null" -- convert: - field: sophosxg.firewall.total_memory - target_field: sophosxg.firewall.total_memory - type: integer - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.total_memory != null" -- convert: - field: sophosxg.firewall.free - target_field: sophosxg.firewall.free - type: integer - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.free != null" -- gsub: - field: sophosxg.firewall.Configuration - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "Disk"' -- convert: - field: sophosxg.firewall.Configuration - target_field: sophosxg.firewall.configuration - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.Configuration != null" -- gsub: - field: sophosxg.firewall.Reports - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "Disk"' -- convert: - field: sophosxg.firewall.Reports - target_field: sophosxg.firewall.Reports - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.Reports != null" -- gsub: - field: sophosxg.firewall.Temp - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "Disk"' -- convert: - field: sophosxg.firewall.Temp - target_field: sophosxg.firewall.Temp - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.Temp != null" -- gsub: - field: sophosxg.firewall.Signature - pattern: "(.{1}$)" - replacement: "" - if: 'ctx.sophosxg?.firewall?.log_component == "Disk"' -- convert: - field: sophosxg.firewall.Signature - target_field: sophosxg.firewall.Signature - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.Signature != null" -- convert: - field: sophosxg.firewall.users - target_field: sophosxg.firewall.users - type: integer - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.users != null" -- convert: - field: sophosxg.firewall.transmittedkbits - target_field: sophosxg.firewall.transmittedkbits - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.transmittedkbits != null" -- convert: - field: sophosxg.firewall.receivedkbits - target_field: sophosxg.firewall.receivedkbits - type: float - ignore_failure: true - ignore_missing: true - if: "ctx.sophosxg?.firewall?.receivedkbits != null" - -############# -## Cleanup ## -############# -on_failure: -- set: - field: error.message - value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/modules.d/sophosxg.yml.disabled b/x-pack/filebeat/modules.d/sophos.yml.disabled similarity index 84% rename from x-pack/filebeat/modules.d/sophosxg.yml.disabled rename to x-pack/filebeat/modules.d/sophos.yml.disabled index c10505d1b93..c870ebb4910 100644 --- a/x-pack/filebeat/modules.d/sophosxg.yml.disabled +++ b/x-pack/filebeat/modules.d/sophos.yml.disabled @@ -1,8 +1,8 @@ -# Module: sophosxg -# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-sophosxg.html +# Module: sophos +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-sophos.html -- module: sophosxg - firewall: +- module: sophos + xg: enabled: true # Set which input to use between tcp, udp (default) or file. @@ -15,9 +15,9 @@ # The port to listen for syslog traffic. Defaults to 9004. #var.syslog_port: 9005 - # firewall default hostanme + # firewall default hostname #var.default_host_name: firewall.localgroup.local - + # known firewalls #var.known_devices: # "device1_serialnumber": "a.host.local" From aaeead0f3209638b8dbd82474ce8bd417a7425b9 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 3 Aug 2020 12:25:26 -0700 Subject: [PATCH 05/19] Support "cluster" scope in Metricbeat elasticsearch module (#18547) * Adding configuration for hosts_mode * Only perform master check in HostsModeNode * Only ask the node if it's the master node if we're in HostsModeNode * Unpack host_mode string into enum * Adding some specific TODOs in node_stats code * Updating x-pack/metricbeat reference config * Set correct service URI * Get master node ID * Adding CHANGELOG entry * Rename hosts_mode => scope * Removing stale TODO comment * Adding docs * Refactoring common code into helper method * Do not set service URI up front * Updating documentation per review * Remove comments from doc examples * Adding configuration for hosts_mode * Set correct service URI * Adding CHANGELOG entry * Rename hosts_mode => scope * Do not set service URI up front * Update metricbeat/docs/modules/elasticsearch.asciidoc Co-authored-by: DeDe Morton * Update metricbeat/module/elasticsearch/_meta/docs.asciidoc Co-authored-by: DeDe Morton * Update reference config * Cleaning up CHANGELOG * Updating generated files Co-authored-by: DeDe Morton --- CHANGELOG.next.asciidoc | 1 + .../docs/modules/elasticsearch.asciidoc | 16 ++++-- metricbeat/metricbeat.reference.yml | 5 +- .../elasticsearch/_meta/config.reference.yml | 5 +- .../module/elasticsearch/_meta/docs.asciidoc | 11 +++- metricbeat/module/elasticsearch/ccr/ccr.go | 9 ++-- .../cluster_stats/cluster_stats.go | 11 ++-- .../module/elasticsearch/elasticsearch.go | 22 ++++++++ .../module/elasticsearch/enrich/enrich.go | 9 ++-- .../module/elasticsearch/index/index.go | 10 ++-- .../index_recovery/index_recovery.go | 11 ++-- .../index_summary/index_summary.go | 9 ++-- metricbeat/module/elasticsearch/metricset.go | 54 ++++++++++++++++++- .../module/elasticsearch/ml_job/ml_job.go | 12 ++--- .../elasticsearch/node_stats/data_xpack.go | 17 +++--- .../elasticsearch/node_stats/node_stats.go | 40 +++++++++++++- .../node_stats/node_stats_test.go | 50 +++++++++++++++++ .../pending_tasks/pending_tasks.go | 11 ++-- .../module/elasticsearch/shard/shard.go | 11 ++-- x-pack/metricbeat/metricbeat.reference.yml | 5 +- 20 files changed, 222 insertions(+), 97 deletions(-) create mode 100644 metricbeat/module/elasticsearch/node_stats/node_stats_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c12c2a9574f..81e6a650ee7 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -610,6 +610,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added cache and connection_errors metrics to status metricset of MySQL module {issue}16955[16955] {pull}19844[19844] - Update MySQL dashboard with connection errors and cache metrics {pull}19913[19913] {issue}16955[16955] - Add cloud.instance.name into aws ec2 metricset. {pull}20077[20077] +- Add `scope` setting for elasticsearch module, allowing it to monitor an Elasticsearch cluster behind a load-balancing proxy. {issue}18539[18539] {pull}18547[18547] *Packetbeat* diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index 6dc13a8dd5c..18f9004a99a 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -13,7 +13,16 @@ The `elasticsearch` module collects metrics about {es}. The `elasticsearch` module works with {es} 6.7.0 and later. [float] -=== Usage for Stack Monitoring +=== Module-specific configuration notes + +Like other {beatname_uc} modules, the `elasticsearch` module accepts a `hosts` configuration setting. +This setting can contain a list of entries. The related `scope` setting determines how each entry in +the `hosts` list is interpreted by the module. + +* If `scope` is set to `node` (default), each entry in the `hosts` list indicates a distinct node in an + {es} cluster. +* If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct + {es} cluster (for example, a load-balancing proxy fronting the cluster). The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` @@ -45,12 +54,9 @@ metricbeat.modules: #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node ---- This module supports TLS connections when using `ssl` config field, as described in <>. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 92aed4ead8f..28cf3e5c477 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -273,12 +273,9 @@ metricbeat.modules: #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy diff --git a/metricbeat/module/elasticsearch/_meta/config.reference.yml b/metricbeat/module/elasticsearch/_meta/config.reference.yml index cc446dfc6e7..61599fa2ae2 100644 --- a/metricbeat/module/elasticsearch/_meta/config.reference.yml +++ b/metricbeat/module/elasticsearch/_meta/config.reference.yml @@ -13,9 +13,6 @@ #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node diff --git a/metricbeat/module/elasticsearch/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/_meta/docs.asciidoc index 0a259e61e3c..b2b1a585a69 100644 --- a/metricbeat/module/elasticsearch/_meta/docs.asciidoc +++ b/metricbeat/module/elasticsearch/_meta/docs.asciidoc @@ -6,7 +6,16 @@ The `elasticsearch` module collects metrics about {es}. The `elasticsearch` module works with {es} 6.7.0 and later. [float] -=== Usage for Stack Monitoring +=== Module-specific configuration notes + +Like other {beatname_uc} modules, the `elasticsearch` module accepts a `hosts` configuration setting. +This setting can contain a list of entries. The related `scope` setting determines how each entry in +the `hosts` list is interpreted by the module. + +* If `scope` is set to `node` (default), each entry in the `hosts` list indicates a distinct node in an + {es} cluster. +* If `scope` is set to `cluster`, each entry in the `hosts` list indicates a single endpoint for a distinct + {es} cluster (for example, a load-balancing proxy fronting the cluster). The `elasticsearch` module can be used to collect metrics shown in our {stack} {monitor-features} UI in {kib}. To enable this usage, set `xpack.enabled: true` and remove any `metricsets` diff --git a/metricbeat/module/elasticsearch/ccr/ccr.go b/metricbeat/module/elasticsearch/ccr/ccr.go index 591f3d12e22..74f7a232281 100644 --- a/metricbeat/module/elasticsearch/ccr/ccr.go +++ b/metricbeat/module/elasticsearch/ccr/ccr.go @@ -56,14 +56,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each follower shard from the _ccr/stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch ccr stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go index 425ef0abacf..cd076cac83d 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go +++ b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go @@ -18,8 +18,6 @@ package cluster_stats import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -51,14 +49,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+clusterStatsPath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch cluster stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index 48050b224cc..52ee233d88c 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -436,6 +436,28 @@ func IsMLockAllEnabled(http *helper.HTTP, resetURI, nodeID string) (bool, error) return false, fmt.Errorf("could not determine if mlockall is enabled on node ID = %v", nodeID) } +// GetMasterNodeID returns the ID of the Elasticsearch cluster's master node +func GetMasterNodeID(http *helper.HTTP, resetURI string) (string, error) { + content, err := fetchPath(http, resetURI, "_nodes/_master", "filter_path=nodes.*.name") + if err != nil { + return "", err + } + + var response struct { + Nodes map[string]interface{} `json:"nodes"` + } + + if err := json.Unmarshal(content, &response); err != nil { + return "", err + } + + for nodeID, _ := range response.Nodes { + return nodeID, nil + } + + return "", errors.New("could not determine master node ID") +} + // PassThruField copies the field at the given path from the given source data object into // the same path in the given target data object. func PassThruField(fieldPath string, sourceData, targetData common.MapStr) error { diff --git a/metricbeat/module/elasticsearch/enrich/enrich.go b/metricbeat/module/elasticsearch/enrich/enrich.go index 6b60394a23e..c533657502a 100644 --- a/metricbeat/module/elasticsearch/enrich/enrich.go +++ b/metricbeat/module/elasticsearch/enrich/enrich.go @@ -55,14 +55,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each enrich coordinator node func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch enrich stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/index/index.go b/metricbeat/module/elasticsearch/index/index.go index 372f9a2dc82..221e78ccfea 100644 --- a/metricbeat/module/elasticsearch/index/index.go +++ b/metricbeat/module/elasticsearch/index/index.go @@ -59,15 +59,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statsPath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch index stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go index 68d1ee295d3..e30463e3848 100644 --- a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go +++ b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go @@ -18,8 +18,6 @@ package index_recovery import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -67,14 +65,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch index recovery stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/index_summary/index_summary.go b/metricbeat/module/elasticsearch/index_summary/index_summary.go index 569e23492cb..dc3dbdd7207 100644 --- a/metricbeat/module/elasticsearch/index_summary/index_summary.go +++ b/metricbeat/module/elasticsearch/index_summary/index_summary.go @@ -62,14 +62,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statsPath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch index summary stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/metricset.go b/metricbeat/module/elasticsearch/metricset.go index c1daae4f577..22b4b2c6c49 100644 --- a/metricbeat/module/elasticsearch/metricset.go +++ b/metricbeat/module/elasticsearch/metricset.go @@ -18,6 +18,10 @@ package elasticsearch import ( + "fmt" + + "github.com/pkg/errors" + "github.com/elastic/beats/v7/metricbeat/helper" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -36,6 +40,31 @@ var ( }.Build() ) +type Scope int + +const ( + // Indicates that each item in the hosts list points to a distinct Elasticsearch node in a + // cluster. + ScopeNode Scope = iota + + // Indicates that each item in the hosts lists points to a endpoint for a distinct Elasticsearch + // cluster (e.g. a load-balancing proxy) fronting the cluster. + ScopeCluster +) + +func (h *Scope) Unpack(str string) error { + switch str { + case "node": + *h = ScopeNode + case "cluster": + *h = ScopeCluster + default: + return fmt.Errorf("invalid scope: %v", str) + } + + return nil +} + // MetricSet can be used to build other metric sets that query RabbitMQ // management plugin type MetricSet struct { @@ -43,6 +72,7 @@ type MetricSet struct { servicePath string *helper.HTTP XPack bool + Scope Scope } // NewMetricSet creates an metric set that can be used to build other metric @@ -54,9 +84,11 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) } config := struct { - XPack bool `config:"xpack.enabled"` + XPack bool `config:"xpack.enabled"` + Scope Scope `config:"scope"` }{ XPack: false, + Scope: ScopeNode, } if err := base.Module().UnpackConfig(&config); err != nil { return nil, err @@ -67,6 +99,7 @@ func NewMetricSet(base mb.BaseMetricSet, servicePath string) (*MetricSet, error) servicePath, http, config.XPack, + config.Scope, } ms.SetServiceURI(servicePath) @@ -84,3 +117,22 @@ func (m *MetricSet) SetServiceURI(servicePath string) { m.servicePath = servicePath m.HTTP.SetURI(m.GetServiceURI()) } + +func (m *MetricSet) ShouldSkipFetch() (bool, error) { + // If we're talking to a set of ES nodes directly, only collect stats from the master node so + // we don't collect the same stats from every node and end up duplicating them. + if m.Scope == ScopeNode { + isMaster, err := IsMaster(m.HTTP, m.GetServiceURI()) + if err != nil { + return false, errors.Wrap(err, "error determining if connected Elasticsearch node is master") + } + + // Not master, no event sent + if !isMaster { + m.Logger().Debugf("trying to fetch %v stats from a non-master node", m.Name()) + return true, nil + } + } + + return false, nil +} diff --git a/metricbeat/module/elasticsearch/ml_job/ml_job.go b/metricbeat/module/elasticsearch/ml_job/ml_job.go index 5f4dbe9f002..d5d58b2d2c6 100644 --- a/metricbeat/module/elasticsearch/ml_job/ml_job.go +++ b/metricbeat/module/elasticsearch/ml_job/ml_job.go @@ -18,8 +18,6 @@ package ml_job import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -54,15 +52,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch machine learning job stats from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/node_stats/data_xpack.go b/metricbeat/module/elasticsearch/node_stats/data_xpack.go index 53340103176..e4efeb3b8e8 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/node_stats/data_xpack.go @@ -187,18 +187,14 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, return errors.Wrap(err, "failure parsing Elasticsearch Node Stats API response") } - // Normally the nodeStruct should only contain one node. But if _local is removed - // from the path and Metricbeat is not installed on the same machine as the node - // it will provid the data for multiple nodes. This will mean the detection of the - // master node will not be accurate anymore as often in these cases a proxy is in front - // of ES and it's not know if the request will be routed to the same node as before. + masterNodeID, err := elasticsearch.GetMasterNodeID(m.HTTP, m.HTTP.GetURI()) + if err != nil { + return err + } + var errs multierror.Errors for nodeID, node := range nodesStruct.Nodes { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HTTP.GetURI()) - if err != nil { - errs = append(errs, errors.Wrap(err, "error determining if connected Elasticsearch node is master")) - continue - } + isMaster := nodeID == masterNodeID event := mb.Event{} @@ -207,6 +203,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, errs = append(errs, errors.Wrap(err, "failure to apply node schema")) continue } + nodeData["node_master"] = isMaster nodeData["node_id"] = nodeID diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index 7498948fd76..5f856e2eeac 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -18,6 +18,8 @@ package node_stats import ( + "net/url" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -33,7 +35,8 @@ func init() { } const ( - nodeStatsPath = "/_nodes/_local/stats" + nodeLocalStatsPath = "/_nodes/_local/stats" + nodesAllStatsPath = "/_nodes/_all/stats" ) // MetricSet type defines all fields of the MetricSet @@ -44,7 +47,7 @@ type MetricSet struct { // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get the stats from the local node - ms, err := elasticsearch.NewMetricSet(base, nodeStatsPath) + ms, err := elasticsearch.NewMetricSet(base, "") // servicePath will be set in Fetch() if err != nil { return nil, err } @@ -53,6 +56,14 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { + if err := m.updateServiceURI(); err != nil { + if m.XPack { + m.Logger().Error(err) + return nil + } + return err + } + content, err := m.HTTP.FetchContent() if err != nil { return err @@ -78,3 +89,28 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return nil } + +func (m *MetricSet) updateServiceURI() error { + u, err := getServiceURI(m.GetURI(), m.Scope) + if err != nil { + return err + } + + m.HTTP.SetURI(u) + return nil + +} + +func getServiceURI(currURI string, scope elasticsearch.Scope) (string, error) { + u, err := url.Parse(currURI) + if err != nil { + return "", err + } + + u.Path = nodeLocalStatsPath + if scope == elasticsearch.ScopeCluster { + u.Path = nodesAllStatsPath + } + + return u.String(), nil +} diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats_test.go b/metricbeat/module/elasticsearch/node_stats/node_stats_test.go new file mode 100644 index 00000000000..120493947b5 --- /dev/null +++ b/metricbeat/module/elasticsearch/node_stats/node_stats_test.go @@ -0,0 +1,50 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package node_stats + +import ( + "testing" + + "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" + + "github.com/stretchr/testify/require" +) + +func TestGetServiceURI(t *testing.T) { + tests := map[string]struct { + scope elasticsearch.Scope + expectedURI string + }{ + "scope_node": { + scope: elasticsearch.ScopeNode, + expectedURI: "/_nodes/_local/stats", + }, + "scope_cluster": { + scope: elasticsearch.ScopeCluster, + expectedURI: "/_nodes/_all/stats", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + newURI, err := getServiceURI("/foo/bar", test.scope) + require.NoError(t, err) + require.Equal(t, test.expectedURI, newURI) + }) + } +} diff --git a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go index 01f11e76318..a25ffb2b9a8 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go +++ b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go @@ -18,8 +18,6 @@ package pending_tasks import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -59,14 +57,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.GetServiceURI()) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch pending tasks from a non-master node") + if shouldSkip { return nil } diff --git a/metricbeat/module/elasticsearch/shard/shard.go b/metricbeat/module/elasticsearch/shard/shard.go index fa46777dffd..4367810a8ca 100644 --- a/metricbeat/module/elasticsearch/shard/shard.go +++ b/metricbeat/module/elasticsearch/shard/shard.go @@ -18,8 +18,6 @@ package shard import ( - "github.com/pkg/errors" - "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/elasticsearch" ) @@ -53,14 +51,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) error { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+statePath) + shouldSkip, err := m.ShouldSkipFetch() if err != nil { - return errors.Wrap(err, "error determining if connected Elasticsearch node is master") + return err } - - // Not master, no event sent - if !isMaster { - m.Logger().Debug("trying to fetch shard stats from a non-master node") + if shouldSkip { return nil } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 141a1cc79b2..457a2c2bfe7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -482,12 +482,9 @@ metricbeat.modules: #password: "changeme" #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - # Set to false to fetch all entries #index_recovery.active_only: true - - # Set to true to send data collected by module to X-Pack - # Monitoring instead of metricbeat-* indices. #xpack.enabled: false + #scope: node #------------------------------ Envoyproxy Module ------------------------------ - module: envoyproxy From 99191e9502ddc0b64f0e70f5be05e45ed624d374 Mon Sep 17 00:00:00 2001 From: jtinkus <35308202+jtinkus@users.noreply.github.com> Date: Tue, 4 Aug 2020 10:09:19 +0300 Subject: [PATCH 06/19] add_process_metadata processor adds container id even if process metadata not accessible (#19767) --- CHANGELOG.next.asciidoc | 1 + .../add_process_metadata.go | 38 +++++--- .../add_process_metadata_test.go | 86 ++++++++++++++++++- 3 files changed, 110 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 81e6a650ee7..d62ce252c6d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -364,6 +364,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562] - Add support for DNS over TLS for the dns_processor. {pull}19321[19321] - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215] +- Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767] *Auditbeat* diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index 4aff1df8576..c41ca9a73d6 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -190,15 +190,29 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul return nil, errors.Errorf("cannot parse field '%s' (not an integer or string)", pidField) } + var meta common.MapStr + metaPtr, err := p.provider.GetProcessMetadata(pid) if err != nil || metaPtr == nil { + // no process metadata, lets still try to get container id p.log.Debugf("failed to get process metadata for PID=%d: %v", pid, err) - return nil, ErrNoProcess + meta = common.MapStr{} + } else { + meta = metaPtr.fields } - meta := metaPtr.fields - if err = p.enrichContainerID(pid, meta); err != nil { - return nil, err + cid, err := p.getContainerID(pid) + if cid == "" || err != nil { + p.log.Debugf("failed to get container id for PID=%d: %v", pid, err) + } else { + if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { + return nil, err + } + } + + if len(meta) == 0 { + // no metadata nor container id + return nil, ErrNoProcess } result = event.Clone() @@ -216,8 +230,8 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul value, err := meta.GetValue(source) if err != nil { - // Should never happen - return nil, err + // skip missing values + continue } if _, err = result.Put(dest, value); err != nil { @@ -228,19 +242,15 @@ func (p *addProcessMetadata) enrich(event common.MapStr, pidField string) (resul return result, nil } -// enrichContainerID adds container.id into meta for mapping to pickup -func (p *addProcessMetadata) enrichContainerID(pid int, meta common.MapStr) error { +func (p *addProcessMetadata) getContainerID(pid int) (string, error) { if p.cidProvider == nil { - return nil + return "", nil } cid, err := p.cidProvider.GetCid(pid) if err != nil { - return err - } - if _, err = meta.Put("container", common.MapStr{"id": cid}); err != nil { - return err + return "", err } - return nil + return cid, nil } // String returns the processor representation formatted as a string diff --git a/libbeat/processors/add_process_metadata/add_process_metadata_test.go b/libbeat/processors/add_process_metadata/add_process_metadata_test.go index 8bb2cd4b625..f9b4aaa681c 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata_test.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata_test.go @@ -49,12 +49,42 @@ func TestAddProcessMetadata(t *testing.T) { ppid: 0, startTime: startTime, }, + 3: { + name: "systemd", + title: "/usr/lib/systemd/systemd --switched-root --system --deserialize 22", + exe: "/usr/lib/systemd/systemd", + args: []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"}, + env: map[string]string{ + "HOME": "/", + "TERM": "linux", + "BOOT_IMAGE": "/boot/vmlinuz-4.11.8-300.fc26.x86_64", + "LANG": "en_US.UTF-8", + }, + pid: 1, + ppid: 0, + startTime: startTime, + }, } // mock of the cgroup processCgroupPaths processCgroupPaths = func(_ string, pid int) (map[string]string, error) { testMap := map[int]map[string]string{ - 1: map[string]string{ + 1: { + "cpu": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "net_prio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "blkio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "perf_event": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "freezer": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "pids": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "hugetlb": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "cpuacct": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "cpuset": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "net_cls": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "devices": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "memory": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + "name=systemd": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + }, + 2: { "cpu": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", "net_prio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", "blkio": "/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", @@ -510,6 +540,60 @@ func TestAddProcessMetadata(t *testing.T) { }, }, }, + { + description: "no process metadata available", + config: common.MapStr{ + "match_pids": []string{"system.process.ppid"}, + "cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*", + }, + event: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "2", + }, + }, + }, + expected: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "2", + }, + }, + "container": common.MapStr{ + "id": "b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1", + }, + }, + }, + { + description: "no container id available", + config: common.MapStr{ + "match_pids": []string{"system.process.ppid"}, + "cgroup_regex": "\\/.+\\/.+\\/.+\\/([0-9a-f]{64}).*", + }, + event: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "3", + }, + }, + }, + expected: common.MapStr{ + "system": common.MapStr{ + "process": common.MapStr{ + "ppid": "3", + }, + }, + "process": common.MapStr{ + "name": "systemd", + "title": "/usr/lib/systemd/systemd --switched-root --system --deserialize 22", + "executable": "/usr/lib/systemd/systemd", + "args": []string{"/usr/lib/systemd/systemd", "--switched-root", "--system", "--deserialize", "22"}, + "pid": 1, + "ppid": 0, + "start_time": startTime, + }, + }, + }, { description: "without cgroup cache", config: common.MapStr{ From b3564940bfbd921fdf00009d26c124f7a2118514 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 4 Aug 2020 10:04:09 +0200 Subject: [PATCH 07/19] Group same timestamp metrics values in app_insights metricset (#20403) * mofidy doc * work on data * changelog * mage fmt update --- CHANGELOG.next.asciidoc | 1 + .../module/azure/app_insights/data.go | 67 +++++++++++++++---- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d62ce252c6d..c4f979f7f3b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -302,6 +302,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix k8s scheduler compatibility issue. {pull}19699[19699] - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185] +- Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403] *Packetbeat* diff --git a/x-pack/metricbeat/module/azure/app_insights/data.go b/x-pack/metricbeat/module/azure/app_insights/data.go index 62afa32163f..df7efdbeaba 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data.go +++ b/x-pack/metricbeat/module/azure/app_insights/data.go @@ -8,6 +8,8 @@ import ( "fmt" "strings" + "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" "github.com/elastic/beats/v7/libbeat/common" @@ -19,45 +21,80 @@ func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId s if metricValues.Value == nil { return events } + groupedAddProp := make(map[string][]insights.MetricsResultInfo) for _, item := range *metricValues.Value { if item.Body != nil && item.Body.Value != nil { if item.Body.Value.AdditionalProperties != nil { - events = append(events, createEvent(*item.Body.Value, insights.MetricsSegmentInfo{}, applicationId)) + groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)] = + append(groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)], *item.Body.Value) } else if item.Body.Value.Segments != nil { for _, segment := range *item.Body.Value.Segments { - events = append(events, createEvent(*item.Body.Value, segment, applicationId)) + event, ok := createSegmentEvent(*item.Body.Value.Start, *item.Body.Value.End, segment, applicationId) + if ok { + events = append(events, event) + } } } } } + if len(groupedAddProp) > 0 { + for _, val := range groupedAddProp { + event, ok := createEvent(val, applicationId) + if ok { + events = append(events, event) + } + } + } return events } -func createEvent(value insights.MetricsResultInfo, segment insights.MetricsSegmentInfo, applicationId string) mb.Event { +func createSegmentEvent(start date.Time, end date.Time, segment insights.MetricsSegmentInfo, applicationId string) (mb.Event, bool) { metricList := common.MapStr{} - if value.AdditionalProperties != nil { + metrics := getMetric(segment.AdditionalProperties) + if len(metrics) == 0 { + return mb.Event{}, false + } + for key, metric := range metrics { + metricList.Put(key, metric) + } + event := mb.Event{ + MetricSetFields: common.MapStr{ + "start_date": start, + "end_date": end, + "application_id": applicationId, + }, + Timestamp: end.Time, + } + event.RootFields = common.MapStr{} + event.RootFields.Put("cloud.provider", "azure") + event.MetricSetFields.Put("metrics", metricList) + return event, true +} + +func createEvent(values []insights.MetricsResultInfo, applicationId string) (mb.Event, bool) { + metricList := common.MapStr{} + for _, value := range values { metrics := getMetric(value.AdditionalProperties) for key, metric := range metrics { metricList.Put(key, metric) } - } else { - metrics := getMetric(segment.AdditionalProperties) - for key, metric := range metrics { - metricList.Put(key, metric) - } } + if len(metricList) == 0 { + return mb.Event{}, false + } + event := mb.Event{ MetricSetFields: common.MapStr{ - "start_date": value.Start, - "end_date": value.End, + "start_date": values[0].Start, + "end_date": values[0].End, "application_id": applicationId, }, - Timestamp: value.End.Time, + Timestamp: values[0].End.Time, } event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") event.MetricSetFields.Put("metrics", metricList) - return event + return event, true } func getMetric(addProp map[string]interface{}) map[string]interface{} { @@ -66,7 +103,9 @@ func getMetric(addProp map[string]interface{}) map[string]interface{} { switch val.(type) { case map[string]interface{}: for subKey, subVal := range val.(map[string]interface{}) { - metricNames[cleanMetricNames(fmt.Sprintf("%s.%s", key, subKey))] = subVal + if subVal != nil { + metricNames[cleanMetricNames(fmt.Sprintf("%s.%s", key, subKey))] = subVal + } } default: metricNames[cleanMetricNames(key)] = val From 8934bc53096ab8f3f1ce6c945756be00c0282586 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 4 Aug 2020 11:55:54 +0200 Subject: [PATCH 08/19] Update container name for the azure filesets (#19899) * update container name * update changelog --- CHANGELOG.next.asciidoc | 1 + .../filebeat/module/azure/activitylogs/config/azure-eventhub.yml | 1 + x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml | 1 + .../filebeat/module/azure/signinlogs/config/azure-eventhub.yml | 1 + 4 files changed, 4 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c4f979f7f3b..58adde400bf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -222,6 +222,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix memory leak in tcp and unix input sources. {pull}19459[19459] - Add missing `default_field: false` to aws filesets fields.yml. {pull}19568[19568] - Fix tls mapping in suricata module {issue}19492[19492] {pull}19494[19494] +- Update container name for the azure filesets. {pull}19899[19899] - Fix bug with empty filter values in system/service {pull}19812[19812] - Fix S3 input to trim delimiter /n from each log line. {pull}19972[19972] - Ignore missing in Zeek module when dropping unecessary fields. {pull}19984[19984] diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index 8c4c42d60cf..a4567959194 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -5,6 +5,7 @@ consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} +storage_account_container: filebeat-activitylogs-{{ .eventhub }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 239e7ef2249..3633cc4e5de 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -5,6 +5,7 @@ consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} +storage_account_container: filebeat-auditlogs-{{ .eventhub }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index 239e7ef2249..dd8e1473a68 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -5,6 +5,7 @@ consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} +storage_account_container: filebeat-signinlogs-{{ .eventhub }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: From 3a9ce715fc829f68c7cac902f7368897e2545c4d Mon Sep 17 00:00:00 2001 From: cachedout Date: Tue, 4 Aug 2020 13:14:52 +0200 Subject: [PATCH 09/19] Ensure install scripts only install if needed (#20349) * Ensure install scripts only install once * Add unmet dep str --- .ci/scripts/install-docker-compose.sh | 16 +++++++++++++++- .ci/scripts/install-go.sh | 14 +++++++++++++- .ci/scripts/install-kind.sh | 15 ++++++++++++++- .ci/scripts/install-terraform.sh | 15 ++++++++++++++- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/.ci/scripts/install-docker-compose.sh b/.ci/scripts/install-docker-compose.sh index a430eb1873f..72d889f216a 100755 --- a/.ci/scripts/install-docker-compose.sh +++ b/.ci/scripts/install-docker-compose.sh @@ -2,9 +2,23 @@ set -exuo pipefail -MSG="parameter missing." +MSG="environment variable missing: DOCKER_COMPOSE_VERSION." DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG} HOME=${HOME:?$MSG} + +if command -v docker-compose +then + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version|awk '{print $3}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ] + then + echo "Versions match. No need to install docker-compose. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing docker-compose" + DC_CMD="${HOME}/bin/docker-compose" mkdir -p "${HOME}/bin" diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh index 5af9f338ca1..49e12c7a18d 100755 --- a/.ci/scripts/install-go.sh +++ b/.ci/scripts/install-go.sh @@ -1,13 +1,25 @@ #!/usr/bin/env bash set -exuo pipefail -MSG="parameter missing." +MSG="environment variable missing" GO_VERSION=${GO_VERSION:?$MSG} PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} HOME=${HOME:?$MSG} ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') GVM_CMD="${HOME}/bin/gvm" +if command -v go +then + echo "Found Go. Checking version.." + FOUND_GO_VERSION=$(go version|awk '{print $3}'|sed s/go//) + if [ $FOUND_GO_VERSION == $GO_VERSION ] + then + echo "Versions match. No need to install Go. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing Go" mkdir -p "${HOME}/bin" curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" diff --git a/.ci/scripts/install-kind.sh b/.ci/scripts/install-kind.sh index dc83bb4cd2a..a53c4b3708a 100755 --- a/.ci/scripts/install-kind.sh +++ b/.ci/scripts/install-kind.sh @@ -1,12 +1,25 @@ #!/usr/bin/env bash set -exuo pipefail -MSG="parameter missing." +MSG="environment variable missing." DEFAULT_HOME="/usr/local" KIND_VERSION=${KIND_VERSION:?$MSG} HOME=${HOME:?$DEFAULT_HOME} KIND_CMD="${HOME}/bin/kind" +if command -v kind +then + echo "Found Kind. Checking version.." + FOUND_KIND_VERSION=$(kind --version 2>&1 >/dev/null | awk '{print $3}') + if [ $FOUND_KIND_VERSION == $KIND_VERSION ] + then + echo "Versions match. No need to install Kind. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing Kind" + mkdir -p "${HOME}/bin" curl -sSLo "${KIND_CMD}" "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" diff --git a/.ci/scripts/install-terraform.sh b/.ci/scripts/install-terraform.sh index 39aa684d0aa..4af2e91baab 100755 --- a/.ci/scripts/install-terraform.sh +++ b/.ci/scripts/install-terraform.sh @@ -2,13 +2,26 @@ set -exuo pipefail -MSG="parameter missing." +MSG="environment variable missing." TERRAFORM_VERSION=${TERRAFORM_VERSION:?$MSG} HOME=${HOME:?$MSG} TERRAFORM_CMD="${HOME}/bin/terraform" OS=$(uname -s | tr '[:upper:]' '[:lower:]') +if command -v terraform +then + echo "Found Terraform. Checking version.." + FOUND_TERRAFORM_VERSION=$(terraform --version | awk '{print $2}' | sed s/v//) + if [ $FOUND_TERRAFORM_VERSION == $TERRAFORM_VERSION ] + then + echo "Versions match. No need to install Terraform. Exiting." + exit 0 + fi +fi + +echo "UNMET DEP: Installing Terraform" + mkdir -p "${HOME}/bin" curl -sSLo - "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${OS}_amd64.zip" > ${TERRAFORM_CMD}.zip From 496e3c9a066120ef23a8a5efd12fe970c5a49a19 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 4 Aug 2020 13:52:47 +0200 Subject: [PATCH 10/19] Makes `metrics` config option required in app_insights (#20406) * mofidy doc * config * changelog * generate * regen * mage fmt --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/modules/azure.asciidoc | 3 ++- x-pack/metricbeat/metricbeat.reference.yml | 3 ++- x-pack/metricbeat/module/azure/_meta/config.reference.yml | 3 ++- x-pack/metricbeat/module/azure/_meta/config.yml | 2 ++ x-pack/metricbeat/module/azure/app_insights/app_insights.go | 2 +- x-pack/metricbeat/modules.d/azure.yml.disabled | 2 ++ 7 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 58adde400bf..1ed3adc4a83 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -303,6 +303,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix k8s scheduler compatibility issue. {pull}19699[19699] - Fix SQL module mapping NULL values as string {pull}18955[18955] {issue}18898[18898 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185] +- Add required option for `metrics` in app_insights. {pull}20406[20406] - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403] *Packetbeat* diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index 248350e3fb1..e4cd4835473 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -241,7 +241,8 @@ metricbeat.modules: period: 300s application_id: '' api_key: '' - + metrics: + - id: ["requests/count", "requests/duration"] ---- [float] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 457a2c2bfe7..a2183e895f4 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -340,7 +340,8 @@ metricbeat.modules: period: 300s application_id: '' api_key: '' - + metrics: + - id: ["requests/count", "requests/duration"] #--------------------------------- Beat Module --------------------------------- - module: beat diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 129384d1498..1f9ac04529e 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -100,4 +100,5 @@ period: 300s application_id: '' api_key: '' - + metrics: + - id: ["requests/count", "requests/duration"] diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 914f5ed5774..0f497af6fb4 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -109,3 +109,5 @@ # period: 300s # application_id: '' # api_key: '' +# metrics: +# - id: ["requests/count", "requests/duration"] diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights.go b/x-pack/metricbeat/module/azure/app_insights/app_insights.go index e3a087ad7ce..8ffe02eb860 100644 --- a/x-pack/metricbeat/module/azure/app_insights/app_insights.go +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights.go @@ -22,7 +22,7 @@ type Config struct { ApplicationId string `config:"application_id" validate:"required"` ApiKey string `config:"api_key" validate:"required"` Period time.Duration `config:"period" validate:"nonzero,required"` - Metrics []Metric `config:"metrics"` + Metrics []Metric `config:"metrics" validate:"required"` } // Metric struct used for configuration options diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 857bfd6619f..23211f47206 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -112,3 +112,5 @@ # period: 300s # application_id: '' # api_key: '' +# metrics: +# - id: ["requests/count", "requests/duration"] From e48f5c8e91506cf24410a65a6757b8fac83e07c1 Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 4 Aug 2020 16:27:42 +0300 Subject: [PATCH 11/19] Improve cgroup_regex docs with examples (#20425) Signed-off-by: chrismark --- .../docs/add_process_metadata.asciidoc | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/libbeat/processors/add_process_metadata/docs/add_process_metadata.asciidoc b/libbeat/processors/add_process_metadata/docs/add_process_metadata.asciidoc index 3066107a009..ddf5802a821 100644 --- a/libbeat/processors/add_process_metadata/docs/add_process_metadata.asciidoc +++ b/libbeat/processors/add_process_metadata/docs/add_process_metadata.asciidoc @@ -72,7 +72,7 @@ field will be present in the output. `host_path`:: (Optional) By default, the `host_path` field is set to the root directory of the host `/`. This is the path where `/proc` is mounted. For -different runtime configurations of Kubernetes or Docker, the `host_path` can +different runtime configurations of Kubernetes or Docker, the `host_path` can be set to overwrite the default. `cgroup_prefixes`:: (Optional) By default, the `cgroup_prefixes` field is set @@ -80,15 +80,21 @@ to `/kubepods` and `/docker`. This is the prefix where the container ID is inside cgroup. For different runtime configurations of Kubernetes or Docker, the `cgroup_prefixes` can be set to overwrite the defaults. -`cgroup_regex`:: (Optional) By default, the container id is extracted from -cgroup file based on `cgroup_prefixes`. This can be overwritten by specifying -regular expression with capture group for capturing container id from cgroup -path. For example: `^\/.+\/.+\/.+\/([0-9a-f]{64}).*` - -`cgroup_cache_expire_time`:: (Optional) By default, the +`cgroup_regex`:: (Optional) By default, the container id is extracted from +cgroup file based on `cgroup_prefixes`. This can be overwritten by specifying +regular expression with capture group for capturing container id from cgroup +path. Examples: +. `^\/.+\/.+\/.+\/([0-9a-f]{64}).*` will match the container id of a cgroup +like `/kubepods/besteffort/pod665fb997-575b-11ea-bfce-080027421ddf/b5285682fba7449c86452b89a800609440ecc88a7ba5f2d38bedfb85409b30b1` +. `^\/.+\/.+\/.+\/docker-([0-9a-f]{64}).scope` will match the container id of a cgroup +like `/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod69349abe_d645_11ea_9c4c_08002709c05c.slice/docker-80d85a3a585f1575028ebe468d83093c301eda20d37d1671ff2a0be50fc0e460.scope` +. `^\/.+\/.+\/.+\/crio-([0-9a-f]{64}).scope` will match the container id of a cgroup +like `/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod69349abe_d645_11ea_9c4c_08002709c05c.slice/crio-80d85a3a585f1575028ebe468d83093c301eda20d37d1671ff2a0be50fc0e460.scope` + +`cgroup_cache_expire_time`:: (Optional) By default, the `cgroup_cache_expire_time` is set to 30 seconds. This is the length of time before cgroup cache elements expire in seconds. It can be set to 0 to disable -the cgroup cache. In some container runtimes technology like runc, the +the cgroup cache. In some container runtimes technology like runc, the container's process is also process in the host kernel, and will be affected by -PID rollover/reuse. The expire time needs to set smaller than the PIDs wrap +PID rollover/reuse. The expire time needs to set smaller than the PIDs wrap around time to avoid wrong container id. From b1bd7b703fae6b5529c22b787cf78580ea1e974b Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Tue, 4 Aug 2020 10:32:47 -0400 Subject: [PATCH 12/19] [Elastic Agent] Fix agent control socket path to always be less than 107 characters (#20426) * Fix agent control socket path to always be less than 107 characters. * Use os.TempDir. * Don't use os.TempDir. --- x-pack/elastic-agent/pkg/agent/control/addr.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/control/addr.go b/x-pack/elastic-agent/pkg/agent/control/addr.go index 3416480a6a0..20bc1e6a005 100644 --- a/x-pack/elastic-agent/pkg/agent/control/addr.go +++ b/x-pack/elastic-agent/pkg/agent/control/addr.go @@ -7,8 +7,8 @@ package control import ( + "crypto/sha256" "fmt" - "path/filepath" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths" ) @@ -16,5 +16,7 @@ import ( // Address returns the address to connect to Elastic Agent daemon. func Address() string { data := paths.Data() - return fmt.Sprintf("unix://%s", filepath.Join(data, "agent.sock")) + // entire string cannot be longer than 107 characters, this forces the + // length to always be 88 characters (but unique per data path) + return fmt.Sprintf(`unix:///tmp/elastic-agent-%x.sock`, sha256.Sum256([]byte(data))) } From 829c3b7dcc6365161d83a3b10f05a9f9990f36c3 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 4 Aug 2020 16:37:12 -0400 Subject: [PATCH 13/19] Add event.ingested to all Filebeat modules (#20386) The event.ingested field defines time at which the event was ingested to Elasticsearch and it added by the Ingest Node pipeline. This field is important when trying to build alerts for activities that may have been reported long after they occurred (@timestamp is much older than event.ingested). This might happen if an agent was offline for a period of time or the processing was delayed. This adds a test to ensure all modules create event.ingested. Use Filebeat read time instead of ingest time as event.created in Zeek. Closes #20073 --- CHANGELOG.next.asciidoc | 1 + filebeat/module/apache/access/ingest/pipeline.yml | 3 +++ filebeat/module/apache/error/ingest/pipeline.yml | 3 +++ filebeat/module/auditd/log/ingest/pipeline.yml | 3 +++ .../module/elasticsearch/audit/ingest/pipeline.yml | 3 +++ .../elasticsearch/deprecation/ingest/pipeline.yml | 3 +++ .../module/elasticsearch/gc/ingest/pipeline.yml | 3 +++ .../module/elasticsearch/server/ingest/pipeline.yml | 3 +++ .../elasticsearch/slowlog/ingest/pipeline.yml | 3 +++ filebeat/module/haproxy/log/ingest/pipeline.yml | 3 +++ filebeat/module/icinga/debug/ingest/pipeline.yml | 3 +++ filebeat/module/icinga/main/ingest/pipeline.yml | 3 +++ filebeat/module/icinga/startup/ingest/pipeline.yml | 3 +++ filebeat/module/iis/access/ingest/pipeline.yml | 3 +++ filebeat/module/iis/error/ingest/pipeline.yml | 3 +++ filebeat/module/kafka/log/ingest/pipeline.yml | 3 +++ filebeat/module/kibana/log/ingest/pipeline.yml | 3 +++ filebeat/module/logstash/log/ingest/pipeline.yml | 3 +++ .../module/logstash/slowlog/ingest/pipeline.yml | 3 +++ filebeat/module/mongodb/log/ingest/pipeline.yml | 3 +++ filebeat/module/mysql/error/ingest/pipeline.yml | 3 +++ filebeat/module/mysql/slowlog/ingest/pipeline.json | 5 +++++ filebeat/module/nats/log/ingest/pipeline.yml | 3 +++ filebeat/module/nginx/access/ingest/pipeline.yml | 5 ++++- filebeat/module/nginx/error/ingest/pipeline.yml | 3 +++ .../nginx/ingress_controller/ingest/pipeline.yml | 3 +++ filebeat/module/osquery/result/ingest/pipeline.json | 5 +++++ filebeat/module/postgresql/log/ingest/pipeline.yml | 3 +++ filebeat/module/redis/log/ingest/pipeline.yml | 3 +++ filebeat/module/santa/log/ingest/pipeline.yml | 3 +++ filebeat/module/system/auth/ingest/pipeline.yml | 3 +++ filebeat/module/system/syslog/ingest/pipeline.yml | 3 +++ filebeat/module/traefik/access/ingest/pipeline.yml | 3 +++ filebeat/tests/system/test_modules.py | 4 ++++ .../module/activemq/audit/ingest/pipeline.yml | 3 +++ .../module/activemq/log/ingest/pipeline.yml | 3 +++ .../module/aws/cloudtrail/ingest/pipeline.yml | 5 ++++- .../module/aws/cloudwatch/ingest/pipeline.yml | 3 +++ x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml | 3 +++ x-pack/filebeat/module/aws/elb/ingest/pipeline.yml | 3 +++ .../module/aws/s3access/ingest/pipeline.yml | 3 +++ .../filebeat/module/aws/vpcflow/ingest/pipeline.yml | 4 ++++ .../module/azure/activitylogs/ingest/pipeline.yml | 3 +++ .../module/azure/auditlogs/ingest/pipeline.yml | 3 +++ .../module/azure/signinlogs/ingest/pipeline.yml | 3 +++ .../module/barracuda/waf/ingest/pipeline.yml | 4 ++++ .../module/bluecoat/director/ingest/pipeline.yml | 4 ++++ x-pack/filebeat/module/cef/log/ingest/pipeline.yml | 4 ++++ .../module/checkpoint/firewall/ingest/pipeline.yml | 13 ++++++++----- .../filebeat/module/cisco/ios/ingest/pipeline.yml | 3 +++ .../module/cisco/shared/ingest/asa-ftd-pipeline.yml | 3 +++ .../module/coredns/log/ingest/pipeline-entry.yml | 3 +++ .../module/cylance/protect/ingest/pipeline.yml | 4 ++++ .../module/envoyproxy/log/ingest/pipeline-entry.yml | 3 +++ .../filebeat/module/f5/bigipapm/ingest/pipeline.yml | 4 ++++ .../fortinet/clientendpoint/ingest/pipeline.yml | 4 ++++ .../module/fortinet/firewall/ingest/pipeline.yml | 3 +++ .../module/googlecloud/audit/ingest/pipeline.yml | 3 +++ .../module/googlecloud/firewall/ingest/pipeline.yml | 4 ++++ .../module/googlecloud/vpcflow/ingest/pipeline.yml | 4 ++++ .../module/ibmmq/errorlog/ingest/pipeline.yml | 3 +++ .../module/imperva/securesphere/ingest/pipeline.yml | 4 ++++ .../module/infoblox/nios/ingest/pipeline.yml | 4 ++++ .../module/iptables/log/ingest/pipeline.yml | 3 +++ .../microsoft/defender_atp/ingest/pipeline.yml | 8 ++++---- .../module/microsoft/dhcp/ingest/pipeline.yml | 4 ++++ .../module/misp/threat/ingest/pipeline.json | 6 ++++++ .../filebeat/module/mssql/log/ingest/pipeline.yml | 3 +++ .../module/netscout/sightline/ingest/pipeline.yml | 4 ++++ .../filebeat/module/o365/audit/ingest/pipeline.yml | 3 +++ .../filebeat/module/okta/system/ingest/pipeline.yml | 5 ++++- .../filebeat/module/panw/panos/ingest/pipeline.yml | 3 +++ .../module/rabbitmq/log/ingest/pipeline.yml | 3 +++ .../module/rapid7/nexpose/ingest/pipeline.yml | 4 ++++ .../module/sonicwall/firewall/ingest/pipeline.yml | 4 ++++ .../filebeat/module/sophos/xg/ingest/pipeline.yml | 3 +++ .../filebeat/module/squid/log/ingest/pipeline.yml | 4 ++++ .../filebeat/module/tomcat/log/ingest/pipeline.yml | 4 ++++ .../module/zeek/capture_loss/ingest/pipeline.yml | 5 ++++- .../module/zeek/connection/ingest/pipeline.yml | 5 ++++- .../module/zeek/dce_rpc/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/dhcp/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/dnp3/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml | 4 ++++ x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/files/ingest/pipeline.yml | 7 +++++-- x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/http/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/intel/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml | 5 ++++- .../module/zeek/kerberos/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/modbus/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/mysql/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/notice/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/ntlm/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/ocsp/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/radius/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml | 5 ++++- .../module/zeek/smb_cmd/ingest/pipeline.yml | 5 ++++- .../module/zeek/smb_files/ingest/pipeline.yml | 5 ++++- .../module/zeek/smb_mapping/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/smtp/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/snmp/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/socks/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml | 5 ++++- x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/stats/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/syslog/ingest/pipeline.yml | 5 ++++- .../module/zeek/traceroute/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/tunnel/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/weird/ingest/pipeline.yml | 5 ++++- .../filebeat/module/zeek/x509/ingest/pipeline.json | 8 +++++++- .../filebeat/module/zscaler/zia/ingest/pipeline.yml | 4 ++++ 116 files changed, 422 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 1ed3adc4a83..41709892e44 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -505,6 +505,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add event.ingested for CrowdStrike module {pull}20138[20138] - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] - Add event.ingested for Suricata module {pull}20220[20220] +- Add event.ingested to all Filebeat modules. {pull}20386[20386] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index 6311bfef12b..a9f23eb4a10 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index 967f7a34b69..aad4c3f4a5f 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing apache error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/auditd/log/ingest/pipeline.yml b/filebeat/module/auditd/log/ingest/pipeline.yml index 2b7c114f10a..26a8bf2ab91 100644 --- a/filebeat/module/auditd/log/ingest/pipeline.yml +++ b/filebeat/module/auditd/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing Linux auditd logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 8f1093f5eea..ec3873d2b9f 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch audit logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index 59b8cf882f9..e1f4838df9b 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch deprecation logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index fc8ec5c73e3..d0980763ecc 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Elasticsearch JVM garbage collection logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 6e09a9dbde8..4d4e634cc4b 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch server logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index 360e86d9d77..ea501d9b3e0 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing elasticsearch slow logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/haproxy/log/ingest/pipeline.yml b/filebeat/module/haproxy/log/ingest/pipeline.yml index fdcfc828701..d9315df0f02 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.yml +++ b/filebeat/module/haproxy/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/debug/ingest/pipeline.yml b/filebeat/module/icinga/debug/ingest/pipeline.yml index ee25b38e90e..dbe9f1ee39d 100644 --- a/filebeat/module/icinga/debug/ingest/pipeline.yml +++ b/filebeat/module/icinga/debug/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga debug logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/main/ingest/pipeline.yml b/filebeat/module/icinga/main/ingest/pipeline.yml index 5db480e07ab..654e8c3c4e7 100644 --- a/filebeat/module/icinga/main/ingest/pipeline.yml +++ b/filebeat/module/icinga/main/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga main logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/icinga/startup/ingest/pipeline.yml b/filebeat/module/icinga/startup/ingest/pipeline.yml index 61e0e6fef27..aee7377b140 100644 --- a/filebeat/module/icinga/startup/ingest/pipeline.yml +++ b/filebeat/module/icinga/startup/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing icinga startup logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 8344cccac1b..84fabdc59b8 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index 4611744d3c9..a16fde841da 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing IIS error logs. Requires the geoip plugin. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/kafka/log/ingest/pipeline.yml b/filebeat/module/kafka/log/ingest/pipeline.yml index a1072489122..aa72addb642 100644 --- a/filebeat/module/kafka/log/ingest/pipeline.yml +++ b/filebeat/module/kafka/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Kafka log messages processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message trace_match: true diff --git a/filebeat/module/kibana/log/ingest/pipeline.yml b/filebeat/module/kibana/log/ingest/pipeline.yml index 0112e09fcfc..ced76d42c23 100644 --- a/filebeat/module/kibana/log/ingest/pipeline.yml +++ b/filebeat/module/kibana/log/ingest/pipeline.yml @@ -4,6 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/log/ingest/pipeline.yml b/filebeat/module/logstash/log/ingest/pipeline.yml index 0a416e5758e..e7dc228a76d 100644 --- a/filebeat/module/logstash/log/ingest/pipeline.yml +++ b/filebeat/module/logstash/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash node logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline.yml b/filebeat/module/logstash/slowlog/ingest/pipeline.yml index 061a4f8c636..949ffdcb91e 100644 --- a/filebeat/module/logstash/slowlog/ingest/pipeline.yml +++ b/filebeat/module/logstash/slowlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing logstash slow logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: '@timestamp' target_field: event.created diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml index 6460a2b02c6..9355e031802 100644 --- a/filebeat/module/mongodb/log/ingest/pipeline.yml +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MongoDB logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml index b11f280d1ea..baf4c11aa40 100644 --- a/filebeat/module/mysql/error/ingest/pipeline.yml +++ b/filebeat/module/mysql/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MySQL error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json index 93ce577a330..d3fbe49707c 100644 --- a/filebeat/module/mysql/slowlog/ingest/pipeline.json +++ b/filebeat/module/mysql/slowlog/ingest/pipeline.json @@ -1,6 +1,11 @@ { "description": "Pipeline for parsing MySQL slow logs.", "processors": [{ + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "grok": { "field": "message", "patterns":[ diff --git a/filebeat/module/nats/log/ingest/pipeline.yml b/filebeat/module/nats/log/ingest/pipeline.yml index 53c4f774b5e..bece77c1b8e 100644 --- a/filebeat/module/nats/log/ingest/pipeline.yml +++ b/filebeat/module/nats/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing nats log logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index f07e82f2b60..57fe9031b55 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -145,7 +148,7 @@ processors: - set: field: event.outcome value: failure - if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" + if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" - append: field: related.ip value: "{{source.ip}}" diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 5a33c34710c..05691eeb737 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing the Nginx error logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 74118b7405e..c9f4a5860c7 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Nginx ingress controller access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/osquery/result/ingest/pipeline.json b/filebeat/module/osquery/result/ingest/pipeline.json index cbc45c202f9..c14b9664d1e 100644 --- a/filebeat/module/osquery/result/ingest/pipeline.json +++ b/filebeat/module/osquery/result/ingest/pipeline.json @@ -2,6 +2,11 @@ "description": "Pipeline for parsing osquery result logs", "processors": [ { + "set":{ + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "rename": { "field": "@timestamp", "target_field": "event.created" diff --git a/filebeat/module/postgresql/log/ingest/pipeline.yml b/filebeat/module/postgresql/log/ingest/pipeline.yml index bd208d1eb72..9233ed95c5f 100644 --- a/filebeat/module/postgresql/log/ingest/pipeline.yml +++ b/filebeat/module/postgresql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing PostgreSQL logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/redis/log/ingest/pipeline.yml b/filebeat/module/redis/log/ingest/pipeline.yml index d1c08cab378..472c3398e36 100644 --- a/filebeat/module/redis/log/ingest/pipeline.yml +++ b/filebeat/module/redis/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing redis logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/santa/log/ingest/pipeline.yml b/filebeat/module/santa/log/ingest/pipeline.yml index 9b68cce3644..e914253f8ee 100644 --- a/filebeat/module/santa/log/ingest/pipeline.yml +++ b/filebeat/module/santa/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Google Santa logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/system/auth/ingest/pipeline.yml b/filebeat/module/system/auth/ingest/pipeline.yml index 3f45705416a..a958855936a 100644 --- a/filebeat/module/system/auth/ingest/pipeline.yml +++ b/filebeat/module/system/auth/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing system authorisation/secure logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message ignore_missing: true diff --git a/filebeat/module/system/syslog/ingest/pipeline.yml b/filebeat/module/system/syslog/ingest/pipeline.yml index e0c80b9aad6..2963ba410b0 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.yml +++ b/filebeat/module/system/syslog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing Syslog messages. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index ce489a4a92c..dd5de1b0b0b 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - dissect: field: message pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index 9360abf51af..b02a98b2f51 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -161,6 +161,10 @@ def run_on_file(self, module, fileset, test_file, cfgfile): assert obj["event"]["module"] == module, "expected event.module={} but got {}".format( module, obj["event"]["module"]) + # All modules must include a set processor that adds the time that + # the event was ingested to Elasticsearch + assert "ingested" in obj["event"], "missing event.ingested timestamp" + assert "error" not in obj, "not error expected but got: {}".format( obj) diff --git a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml index afc4c50f3dc..c5cb5ee8ed1 100644 --- a/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ audit logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml index c33d77295e5..b84807be893 100644 --- a/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/activemq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing ActiveMQ logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml index 36773124439..42395228853 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for AWS CloudTrail Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: "message" target_field: "event.original" @@ -614,7 +617,7 @@ processors: if (ctx.event.action == 'ConsoleLogin' && ctx?.aws?.cloudtrail?.flattened?.response_elements.ConsoleLogin != null) { ctx.event.outcome = Processors.lowercase(ctx.aws.cloudtrail.flattened.response_elements.ConsoleLogin); } - + def hm = new HashMap(params.get(ctx.event.action)); hm.forEach((k, v) -> ctx.event[k] = v); diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index ff7e20d1c3d..05f25463414 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for CloudWatch logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 0ada24c6f77..878aa14aef5 100644 --- a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for EC2 logs in CloudWatch" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index fbd1195dcae..de772ccdf01 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for ELB logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message # Classic ELB patterns documented in https://docs.aws.amazon.com/elasticloadbalancing/latest/classic/access-log-collection.html diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index efd1a9d358a..dd8613a904a 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: "Pipeline for s3 server access logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml index 0ad04419cbd..bd9b1d32769 100644 --- a/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/vpcflow/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for AWS VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # Convert Unix epoch to timestamp - date: field: "aws.vpcflow.end" diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index dac11495608..f8f10132a0d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml index 2bf26322faf..e6a29f6cc13 100644 --- a/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/auditlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure activity logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml index 9d5351bf36a..77ccfa32dec 100644 --- a/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/signinlogs/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing azure signin logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - rename: field: azure target_field: azure-eventhub diff --git a/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml b/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml index dffea972086..1616836706d 100644 --- a/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml +++ b/x-pack/filebeat/module/barracuda/waf/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Barracuda Web Application Firewall processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml b/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml index e26891a1ad0..8f8064017d4 100644 --- a/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml +++ b/x-pack/filebeat/module/bluecoat/director/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Blue Coat Director processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 75a86ea2758..7dab1ca3382 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Filebeat CEF processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index 9a74b0b7c72..d21d421ce0f 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing checkpoint firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: @@ -157,7 +160,7 @@ processors: target_field: source.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true if: "ctx.checkpoint?.xlatesport != '0'" - rename: field: checkpoint.mac_source_address @@ -691,7 +694,7 @@ processors: field: client.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: client.bytes type: long @@ -711,7 +714,7 @@ processors: field: server.nat.port type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - convert: field: server.bytes type: long @@ -721,7 +724,7 @@ processors: field: server.packets type: long ignore_failure: true - ignore_missing: true + ignore_missing: true - script: lang: painless source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" @@ -797,4 +800,4 @@ processors: on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' \ No newline at end of file + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml index 6ffe20df8f5..a09d2b31c5e 100644 --- a/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/ios/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Cisco IOS logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index d34f3562e68..28a2750b6d4 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1,6 +1,9 @@ --- description: "Pipeline for Cisco {< .internal_PREFIX >} logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # # Parse the syslog header # diff --git a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml index f25d3417836..0a14b12f4c1 100644 --- a/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/coredns/log/ingest/pipeline-entry.yml @@ -1,6 +1,9 @@ --- description: Pipeline for normalizing Kubernetes CoreDNS logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - pipeline: if: ctx.message.charAt(0) == (char)("{") name: '{< IngestPipeline "pipeline-json" >}' diff --git a/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml b/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml index d6bca1e8c47..286058aea62 100644 --- a/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cylance/protect/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for CylanceProtect processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml index 2bc7e14fb4f..296d932f2ce 100644 --- a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.yml @@ -1,5 +1,8 @@ description: Pipeline for normalizing envoyproxy logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - pipeline: if: ctx.message.charAt(0) != (char)("{") name: '{< IngestPipeline "pipeline-plaintext" >}' diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 0ea72c6ba4d..2de20fc1a50 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Big-IP Access Policy Manager processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml index 1897a785e50..1fd14e58bd6 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/clientendpoint/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Fortinet FortiClient Endpoint Security processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 2aaf7065ec1..eeb5368db55 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing fortinet firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml index 6c148a0c07c..8e0d3ac6fdb 100644 --- a/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Google Cloud audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml index 8d68de684a6..b01435b7b62 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for Google Cloud Firewall Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml b/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml index 161de8ea031..a8af06f2f4b 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/ingest/pipeline.yml @@ -1,6 +1,10 @@ description: Pipeline for Google Cloud VPC Flow Logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml index 80db3a86a86..87c3deacb97 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing MQ error logs. processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - gsub: field: message pattern: ^[\-]{5}[a-z0-9\. :]*[\-]{5,} diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index 4a84f2a8bc8..63671e09e97 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Imperva SecureSphere processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml b/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml index 5693b4aea49..cc784492797 100644 --- a/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml +++ b/x-pack/filebeat/module/infoblox/nios/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Infoblox NIOS processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml index 4eb24ff7d03..ecaa40ce67c 100644 --- a/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/iptables/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for IPTables processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index 6dedd5e8a1f..392f3a441a7 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing microsoft atp logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - remove: field: - message @@ -76,9 +79,6 @@ processors: - set: field: event.provider value: defender_atp -- set: - field: event.ingested - value: '{{_ingest.timestamp}}' - set: field: event.created value: '{{json.alertCreationTime}}' @@ -284,7 +284,7 @@ processors: ## Cleanup ## ############# - remove: - field: + field: - json.alertCreationTime - json.severity - json.relatedUser diff --git a/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml index 184e6c3e4a9..fd43032ff6e 100644 --- a/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/dhcp/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Microsoft DHCP processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json index 0d710feeb24..59abc2fc21e 100644 --- a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json +++ b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json @@ -1,6 +1,12 @@ { "description": "Pipeline for normalizing MISP threat", "processors": [ + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + }, { "geoip": { "field": "destination.ip", diff --git a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml index 39a10a9ff99..cae8f53ab34 100644 --- a/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/mssql/log/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline to parse MSSQL logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 66f9ab7bcc1..5525c2ba70f 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Arbor Peakflow SP processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 98fd4f0ff58..49371346925 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Office 365 Audit logs processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml index 78f6fa37047..e3d92540d5f 100644 --- a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml +++ b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml @@ -1,6 +1,9 @@ description: Pipeline for Okta system logs. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - user_agent: field: user_agent.original ignore_missing: true @@ -44,7 +47,7 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - + on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml index 7cc44f287b6..25d1ba1681b 100644 --- a/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/panw/panos/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: "Pipeline for Palo Alto Networks PAN-OS Logs" processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' # keep message as log.original. - rename: diff --git a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml index 58097c578d8..e69402c6a95 100644 --- a/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rabbitmq/log/ingest/pipeline.yml @@ -1,6 +1,9 @@ --- description: Pipeline for parsing RabbitMQ logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message pattern_definitions: diff --git a/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml b/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml index d558e7071ea..816d612b6a7 100644 --- a/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml +++ b/x-pack/filebeat/module/rapid7/nexpose/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Rapid7 NeXpose processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml index 75670b6f441..fdfb0f7f9a0 100644 --- a/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sonicwall/firewall/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Sonicwall-FW processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml index f408b6f01cd..8102bb92514 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml @@ -1,5 +1,8 @@ description: Pipeline for parsing sophosxg firewall logs processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' - grok: field: message patterns: diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index caeba41fcbc..574cfafde0a 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Squid processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml b/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml index e5cd87682ea..16a25fde6f2 100644 --- a/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/tomcat/log/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Apache Tomcat processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original diff --git a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml index 3c6171bc045..76e5178572e 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek capture_loss.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.capture_loss.ts formats: diff --git a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml index b660079324a..a243e057420 100644 --- a/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/connection/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek conn.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.connection.ts formats: diff --git a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml index 1ecda252cc8..e77cd5646f0 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dce_rpc.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dce_rpc.ts formats: diff --git a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml index 49df687ecc3..49216c077c2 100644 --- a/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dhcp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dhcp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dhcp.ts formats: diff --git a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml index ad4670dc350..f1a1e527cfc 100644 --- a/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dnp3/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dnp3.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dnp3.ts formats: diff --git a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml index db603d93dbb..77ea898c66b 100644 --- a/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dns/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Filebeat Zeek dns.log processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml index f30ff172fa8..32d1852c3e2 100644 --- a/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/dpd/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek dpd.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.dpd.ts formats: diff --git a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml index 0d5abf9bdda..754720e9209 100644 --- a/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/files/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek files.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.files.ts formats: @@ -47,7 +50,7 @@ processors: - set: field: client.ip value: "{{zeek.files.rx_host}}" - if: "ctx?.zeek?.files?.rx_host != null" + if: "ctx?.zeek?.files?.rx_host != null" - append: field: related.hash value: "{{file.hash.md5}}" diff --git a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml index 7c15dce3ac5..f1f7d0b4f52 100644 --- a/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ftp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ftp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ftp.ts formats: diff --git a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml index a382c25a74d..a2c4a85b994 100644 --- a/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/http/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek http.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.http.ts formats: diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml index 6a2bd6382ad..c6e64e016b8 100644 --- a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml @@ -1,9 +1,12 @@ --- description: Pipeline for normalizing Zeek intel.log. processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' - set: field: event.created - value: "{{_ingest.timestamp}}" + value: '{{@timestamp}}' # IP Geolocation Lookup - geoip: diff --git a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml index ec04f4e7c93..dd1e37a7035 100644 --- a/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/irc/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek irc.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.irc.ts formats: diff --git a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml index 05005491115..3604287cb5e 100644 --- a/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/kerberos/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek kerberos.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.kerberos.ts formats: diff --git a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml index d053a541ef5..d918b2de09a 100644 --- a/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/modbus/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek modbus.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.modbus.ts formats: diff --git a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml index ca2c6c57172..ce2de353549 100644 --- a/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/mysql/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek mysql.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.mysql.ts formats: diff --git a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml index c4dee6b78f2..c741d355361 100644 --- a/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/notice/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek notice.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.notice.ts formats: diff --git a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml index 9f76d461392..690fd54a54b 100644 --- a/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ntlm/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ntlm.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ntlm.ts formats: diff --git a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml index 63a878825d7..462c1f36612 100644 --- a/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ocsp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ocsp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ocsp.ts formats: diff --git a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml index 6a7fa7dca87..6e1272a8ab2 100644 --- a/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/pe/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek pe.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.pe.ts formats: diff --git a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml index c69dfaefbb4..acc7fad2f03 100644 --- a/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/radius/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek radius.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.radius.ts formats: diff --git a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml index d6b70dd92e6..bbe4abcee9f 100644 --- a/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/rdp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek rdp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.rdp.ts formats: diff --git a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml index 8cf2cebdf4d..2ce5fda4e16 100644 --- a/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/rfb/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek rfb.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.rfb.ts formats: diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index 9982cb82d87..d8408c51133 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek sip.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.sip.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml index 838e9f2e8bc..0a853104351 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_cmd.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_cmd.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml index b2c7f52a29b..b1c0d3a6992 100644 --- a/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_files/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_files.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_files.ts formats: diff --git a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml index b5752120267..e116e1bfb60 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smb_mapping.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smb_mapping.ts formats: diff --git a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml index 4424d3674ff..03e2ffb6a25 100644 --- a/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/smtp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek smtp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.smtp.ts formats: diff --git a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml index f0070ef790d..1aefc539733 100644 --- a/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/snmp/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek snmp.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.snmp.ts formats: diff --git a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml index 04a84b13177..210d9771023 100644 --- a/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/socks/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek socks.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.socks.ts formats: diff --git a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml index 019a44b89e0..26980d26f3d 100644 --- a/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssh/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek ssh.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ssh.ts formats: diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml index bbeaa24d1bd..c40801f4243 100644 --- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml @@ -2,8 +2,11 @@ description: Pipeline for normalizing Zeek ssl.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.ssl.ts formats: diff --git a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml index c0347161190..04e851e14a9 100644 --- a/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/stats/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek stats.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.stats.ts formats: diff --git a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml index 7fd848682b1..5f3432ec488 100644 --- a/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/syslog/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek syslog.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.syslog.ts formats: diff --git a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml index 6fa5a0bc993..f4744c540d7 100644 --- a/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/traceroute/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek traceroute.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.traceroute.ts formats: diff --git a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml index 402bce5fa5d..9ca83da3305 100644 --- a/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/tunnel/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek tunnel.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.tunnel.ts formats: diff --git a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml index e0325d9a1c5..d791eb77a09 100644 --- a/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/weird/ingest/pipeline.yml @@ -1,8 +1,11 @@ description: Pipeline for normalizing Zeek weird.log processors: - set: - field: event.created + field: event.ingested value: '{{_ingest.timestamp}}' +- set: + field: event.created + value: '{{@timestamp}}' - date: field: zeek.weird.ts formats: diff --git a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json index e35b8bbbafc..aeb1dbf36fb 100644 --- a/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json +++ b/x-pack/filebeat/module/zeek/x509/ingest/pipeline.json @@ -3,10 +3,16 @@ "processors": [ { "set": { - "field": "event.created", + "field": "event.ingested", "value": "{{_ingest.timestamp}}" } }, + { + "set": { + "field": "event.created", + "value": "{{@timestamp}}" + } + }, { "date": { "field": "zeek.x509.ts", diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index 3354fb0674a..884dd6392a5 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -2,6 +2,10 @@ description: Pipeline for Zscaler NSS processors: + - set: + field: event.ingested + value: '{{_ingest.timestamp}}' + # User agent - user_agent: field: user_agent.original From fc0e1928bae2188049976e63eb40db1343b4876e Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Wed, 5 Aug 2020 11:09:19 +0200 Subject: [PATCH 14/19] [Ingest Manager] Send datastreams fields (#20402) [Ingest Manager] Send datastreams fields (#20402) --- .../elastic-agent/pkg/agent/operation/monitoring.go | 12 ++++++------ .../program/testdata/constraints_config-filebeat.yml | 4 ++-- .../testdata/enabled_output_true-filebeat.yml | 4 ++-- .../agent/program/testdata/enabled_true-filebeat.yml | 4 ++-- .../program/testdata/single_config-filebeat.yml | 8 ++++---- .../program/testdata/single_config-metricbeat.yml | 12 ++++++------ x-pack/elastic-agent/pkg/agent/transpiler/rules.go | 6 ++++-- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 62372cc3f54..cd9d3d95a2c 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -190,10 +190,10 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "logs", - "name": "elastic.agent", + "dataset": "elastic.agent", "namespace": "default", }, }, @@ -224,10 +224,10 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "logs", - "name": fmt.Sprintf("elastic.agent.%s", name), + "dataset": fmt.Sprintf("elastic.agent.%s", name), "namespace": "default", }, }, @@ -274,10 +274,10 @@ func (o *Operator) getMonitoringMetricbeatConfig(output interface{}) (map[string "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ - "target": "dataset", + "target": "data_stream", "fields": map[string]interface{}{ "type": "metrics", - "name": fmt.Sprintf("elastic.agent.%s", name), + "dataset": fmt.Sprintf("elastic.agent.%s", name), "namespace": "default", }, }, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml index 15f6b71a953..15013da3377 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/constraints_config-filebeat.yml @@ -7,10 +7,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index c2e8c0d26ec..8edc27061b0 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -7,10 +7,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index 1da1c701d81..8bd5d93a3b9 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -8,10 +8,10 @@ filebeat: index: logs-generic-default processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index 0fb1a4356b5..b996e13b531 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -9,10 +9,10 @@ filebeat: var: value processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: logs - name: generic + dataset: generic namespace: default - add_fields: target: "event" @@ -27,10 +27,10 @@ filebeat: var: value processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: testtype - name: generic + dataset: generic namespace: default - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index 67a3815e4a7..c62882ff6da 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -6,10 +6,10 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: docker.status + dataset: docker.status namespace: default - add_fields: target: "event" @@ -21,10 +21,10 @@ metricbeat: hosts: ["http://127.0.0.1:8080"] processors: - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: generic + dataset: generic namespace: default - add_fields: target: "event" @@ -39,10 +39,10 @@ metricbeat: fields: should_be: first - add_fields: - target: "dataset" + target: "data_stream" fields: type: metrics - name: generic + dataset: generic namespace: testing - add_fields: target: "event" diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index fe98386a150..a9fc8f4497d 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -632,16 +632,18 @@ func (r *InjectStreamProcessorRule) Apply(ast *AST) error { return errors.New("InjectStreamProcessorRule: processors is not a list") } + // datastream processorMap := &Dict{value: make([]Node, 0)} - processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "dataset"}}) + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "data_stream"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ &Key{name: "type", value: &StrVal{value: datasetType}}, &Key{name: "namespace", value: &StrVal{value: namespace}}, - &Key{name: "name", value: &StrVal{value: dataset}}, + &Key{name: "dataset", value: &StrVal{value: dataset}}, }}}) addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} processorsList.value = mergeStrategy(r.OnConflict).InjectItem(processorsList.value, addFieldsMap) + // event processorMap = &Dict{value: make([]Node, 0)} processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "event"}}) processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ From 6a9a20e47a8a28a0368c1f992ce585a271d09b07 Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Wed, 5 Aug 2020 11:40:02 +0200 Subject: [PATCH 15/19] [Filebeat][ATP Module]Setting user agent field required by the API (#20440) Setting user-agent field required by ATP API for monitoring purposes --- .../filebeat/module/microsoft/defender_atp/config/atp.yml | 2 +- x-pack/filebeat/module/microsoft/defender_atp/manifest.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml index 5210fc53e75..5108ebdad07 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml @@ -11,7 +11,7 @@ url: {{ .url }} oauth2: {{ .oauth2 | tojson }} oauth2.provider: azure oauth2.azure.resource: https://api.securitycenter.windows.com/ - +http_headers: {{ .http_headers | tojson }} date_cursor.field: lastUpdateTime date_cursor.url_field: '$filter' date_cursor.value_template: {{ .date_cursor.value_template }} diff --git a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml index 99cca9da1af..22db3448710 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/manifest.yml @@ -6,14 +6,17 @@ var: - name: interval default: 5m - name: date_cursor - default: + default: value_template: "lastUpdateTime gt {{.}}" - name: tags default: [defender-atp, forwarded] + - name: http_headers + default: + User-Agent: MdatpPartner-Elastic-Filebeat/1.0.0 - name: url default: "https://api.securitycenter.windows.com/api/alerts?$expand=evidence" - name: oauth2 - + ingest_pipeline: ingest/pipeline.yml input: config/atp.yml From 3cc97ffe18e63b7af3aaceab5e85e397d90d3be7 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 5 Aug 2020 12:16:08 +0200 Subject: [PATCH 16/19] Remove pillow from testing requirements.txt (#20407) --- libbeat/tests/system/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 0d8d0f081a2..8cbb1c57393 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -19,7 +19,6 @@ nose==1.3.7 nose-timer==0.7.1 pycodestyle==2.4.0 PyYAML==4.2b1 -Pillow>=7.1.0 redis==2.10.6 requests==2.20.0 six==1.11.0 From 6d8acd0f48d10ea4d1c1d03ab6dfa31747576c62 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 5 Aug 2020 12:27:47 +0200 Subject: [PATCH 17/19] Disable logging when pulling on python integration tests (#20397) Docker compose library is quite verbose, and it prints many messages when logging is enabled. On integration tests we make a pull before trying to build the images in case the image is already pre-built. If this pull doesn't work, the image is built, so we ignore errors on this pull. But, even when ignoring errors, these errors are logged, and when investigating problems with tests this may lead to think that the problem is with the unavailability of some image. Disable logging on the compose logger while this previous pull is being done. --- libbeat/tests/system/beat/compose.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py index f50ab299dff..9e9f36d849b 100644 --- a/libbeat/tests/system/beat/compose.py +++ b/libbeat/tests/system/beat/compose.py @@ -1,8 +1,11 @@ +import io +import logging import os import sys import tarfile import time -import io + +from contextlib import contextmanager INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -54,9 +57,12 @@ def is_healthy(container): return container.inspect()['State']['Health']['Status'] == 'healthy' project = cls.compose_project() - project.pull( - ignore_pull_failures=True, - service_names=cls.COMPOSE_SERVICES) + + with disabled_logger('compose.service'): + project.pull( + ignore_pull_failures=True, + service_names=cls.COMPOSE_SERVICES) + project.up( strategy=ConvergenceStrategy.always, service_names=cls.COMPOSE_SERVICES, @@ -231,3 +237,14 @@ def service_log_contains(cls, service, msg): if line.find(msg.encode("utf-8")) >= 0: counter += 1 return counter > 0 + + +@contextmanager +def disabled_logger(name): + logger = logging.getLogger(name) + old_level = logger.getEffectiveLevel() + logger.setLevel(logging.CRITICAL) + try: + yield logger + finally: + logger.setLevel(old_level) From 945da26f73599adeed41e80855604bde53d0c3db Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Wed, 5 Aug 2020 16:15:38 +0200 Subject: [PATCH 18/19] Upgrade elasticsearch client library used in tests (#20405) Upgrade elasticsearch client library used in python tests. Old version uses deprecated Python functions that will be removed in Python 3.9. --- libbeat/tests/system/requirements.txt | 2 +- metricbeat/module/elasticsearch/test_elasticsearch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 8cbb1c57393..0829cbd0164 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -8,7 +8,7 @@ docker-compose==1.25.3 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 -elasticsearch==7.1.0 +elasticsearch==7.8.1 enum34==1.1.6 idna==2.6 ipaddress==1.0.19 diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index f8e2762b8cf..88e064c6d11 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -24,7 +24,7 @@ class Test(metricbeat.BaseTest): def setUp(self): super(Test, self).setUp() self.es = Elasticsearch(self.get_hosts()) - self.ml_es = client.xpack.ml.MlClient(self.es) + self.ml_es = client.ml.MlClient(self.es) es_version = self.get_version() if es_version["major"] < 7: From 44fc8e3302138dd7bd0be8d6ac2ff6754cc63c20 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Wed, 5 Aug 2020 18:22:53 -0700 Subject: [PATCH 19/19] [docs] Promote ingest management to beta (#20295) --- x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc | 2 +- .../docs/elastic-agent-configuration-example.asciidoc | 2 +- x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc | 2 +- x-pack/elastic-agent/docs/elastic-agent.asciidoc | 2 +- x-pack/elastic-agent/docs/install-elastic-agent.asciidoc | 2 +- x-pack/elastic-agent/docs/run-elastic-agent.asciidoc | 2 +- x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc | 2 -- 7 files changed, 6 insertions(+), 8 deletions(-) diff --git a/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc index 679e7c26e0c..aa47b8505e7 100644 --- a/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc @@ -2,7 +2,7 @@ [role="xpack"] = Command line options -experimental[] +beta[] The `elastic-agent run` command provides flags that alter the behavior of an agent: diff --git a/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc index 464712c6a27..b5f0ed0aef6 100644 --- a/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent-configuration-example.asciidoc @@ -2,7 +2,7 @@ [role="xpack"] = Configuration example -experimental[] +beta[] The following example shows a full list of configuration options: diff --git a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc index 860b109c779..9dcf1dc7a5b 100644 --- a/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent-configuration.asciidoc @@ -2,7 +2,7 @@ [role="xpack"] = Configuration settings -experimental[] +beta[] By default {agent} runs in standalone mode to ingest system data and send it to a local {es} instance running on port 9200. It uses the demo credentials of the diff --git a/x-pack/elastic-agent/docs/elastic-agent.asciidoc b/x-pack/elastic-agent/docs/elastic-agent.asciidoc index a3736a8e944..f68138f9d53 100644 --- a/x-pack/elastic-agent/docs/elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/elastic-agent.asciidoc @@ -3,7 +3,7 @@ = Manage your {agent}s -experimental[] +beta[] // tag::agent-install-intro[] {agent} is a single, unified agent that you can deploy to hosts or containers to diff --git a/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc index c7112ac3a32..dc661e6308c 100644 --- a/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/install-elastic-agent.asciidoc @@ -2,7 +2,7 @@ [role="xpack"] = Install {agent} -experimental[] +beta[] Download and install the Agent on each system you want to monitor. diff --git a/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc index 9cc31bfc44a..9c7ce91e2ec 100644 --- a/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/run-elastic-agent.asciidoc @@ -2,7 +2,7 @@ [role="xpack"] = Run {agent} -experimental[] +beta[] {agent} runs in two modes: standalone or fleet. The two modes differ in how you configure and manage the Agent. diff --git a/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc b/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc index 913254d688b..1c90d4c3f60 100644 --- a/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc +++ b/x-pack/elastic-agent/docs/stop-elastic-agent.asciidoc @@ -5,8 +5,6 @@ To stop {agent} and its related executables, stop the {agent} process. Use the commands that work for your system. -//TODO: Replace with tabbed panel when it's out of experimental phase. - *Windows:* If you installed the Agent as a service, stop the service. If