From f3cbcadda5a8e38b2580b1746f9a3b95c072a6df Mon Sep 17 00:00:00 2001 From: Mathieu Martin Date: Fri, 4 Dec 2020 15:28:40 -0500 Subject: [PATCH 1/2] Apply the RFC 0005 stage 2 (host metrics) changes in the experimental artifacts (#1159) # Conflicts: # experimental/generated/csv/fields.csv --- experimental/generated/beats/fields.ecs.yml | 46 ++++++++++++ experimental/generated/ecs/ecs_flat.yml | 74 +++++++++++++++++++ experimental/generated/ecs/ecs_nested.yml | 74 +++++++++++++++++++ .../generated/elasticsearch/7/template.json | 50 +++++++++++++ experimental/schemas/host.yml | 61 +++++++++++++++ 5 files changed, 305 insertions(+) diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 16c38aefca..6c7bab42a6 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -2047,6 +2047,28 @@ ignore_above: 1024 description: Operating system architecture. example: x86_64 + - name: cpu.usage + level: extended + type: scaled_float + description: 'Percent CPU used which is normalized by the number of CPU cores + and it ranges from 0 to 1. Scaling factor: 1000. + + For example: For a two core host, this value should be the average of the + two cores, between 0 and 1.' + scaling_factor: 1000 + default_field: false + - name: disk.read.bytes + level: extended + type: long + description: The total number of bytes (gauge) read successfully (aggregated + from all disks) since the last metric collection. + default_field: false + - name: disk.write.bytes + level: extended + type: long + description: The total number of bytes (gauge) written successfully (aggregated + from all disks) since the last metric collection. + default_field: false - name: domain level: extended type: keyword @@ -2143,6 +2165,30 @@ It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: network.egress.bytes + level: extended + type: long + description: The number of bytes (gauge) sent out on all network interfaces + by the host since the last metric collection. + default_field: false + - name: network.egress.packets + level: extended + type: long + description: The number of packets (gauge) sent out on all network interfaces + by the host since the last metric collection. + default_field: false + - name: network.ingress.bytes + level: extended + type: long + description: The number of bytes received (gauge) on all network interfaces + by the host since the last metric collection. + default_field: false + - name: network.ingress.packets + level: extended + type: long + description: The number of packets (gauge) received on all network interfaces + by the host since the last metric collection. + default_field: false - name: os.family level: extended type: keyword diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index 255173741f..9f8aa0a1b6 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -3182,6 +3182,40 @@ host.architecture: normalize: [] short: Operating system architecture. type: keyword +host.cpu.usage: + dashed_name: host-cpu-usage + description: 'Percent CPU used which is normalized by the number of CPU cores and + it ranges from 0 to 1. Scaling factor: 1000. + + For example: For a two core host, this value should be the average of the two + cores, between 0 and 1.' + flat_name: host.cpu.usage + level: extended + name: cpu.usage + normalize: [] + scaling_factor: 1000 + short: Percent CPU used, between 0 and 1. + type: scaled_float +host.disk.read.bytes: + dashed_name: host-disk-read-bytes + description: The total number of bytes (gauge) read successfully (aggregated from + all disks) since the last metric collection. + flat_name: host.disk.read.bytes + level: extended + name: disk.read.bytes + normalize: [] + short: The number of bytes read by all disks. + type: long +host.disk.write.bytes: + dashed_name: host-disk-write-bytes + description: The total number of bytes (gauge) written successfully (aggregated + from all disks) since the last metric collection. + flat_name: host.disk.write.bytes + level: extended + name: disk.write.bytes + normalize: [] + short: The number of bytes written on all disks. + type: long host.domain: dashed_name: host-domain description: 'Name of the domain of which the host is a member. @@ -3355,6 +3389,46 @@ host.name: normalize: [] short: Name of the host. type: keyword +host.network.egress.bytes: + dashed_name: host-network-egress-bytes + description: The number of bytes (gauge) sent out on all network interfaces by the + host since the last metric collection. + flat_name: host.network.egress.bytes + level: extended + name: network.egress.bytes + normalize: [] + short: The number of bytes sent on all network interfaces. + type: long +host.network.egress.packets: + dashed_name: host-network-egress-packets + description: The number of packets (gauge) sent out on all network interfaces by + the host since the last metric collection. + flat_name: host.network.egress.packets + level: extended + name: network.egress.packets + normalize: [] + short: The number of packets sent on all network interfaces. + type: long +host.network.ingress.bytes: + dashed_name: host-network-ingress-bytes + description: The number of bytes received (gauge) on all network interfaces by the + host since the last metric collection. + flat_name: host.network.ingress.bytes + level: extended + name: network.ingress.bytes + normalize: [] + short: The number of bytes received on all network interfaces. + type: long +host.network.ingress.packets: + dashed_name: host-network-ingress-packets + description: The number of packets (gauge) received on all network interfaces by + the host since the last metric collection. + flat_name: host.network.ingress.packets + level: extended + name: network.ingress.packets + normalize: [] + short: The number of packets received on all network interfaces. + type: long host.os.family: dashed_name: host-os-family description: OS family (such as redhat, debian, freebsd, windows). diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 5072c1b3de..f6c475532c 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -3843,6 +3843,40 @@ host: normalize: [] short: Operating system architecture. type: keyword + host.cpu.usage: + dashed_name: host-cpu-usage + description: 'Percent CPU used which is normalized by the number of CPU cores + and it ranges from 0 to 1. Scaling factor: 1000. + + For example: For a two core host, this value should be the average of the + two cores, between 0 and 1.' + flat_name: host.cpu.usage + level: extended + name: cpu.usage + normalize: [] + scaling_factor: 1000 + short: Percent CPU used, between 0 and 1. + type: scaled_float + host.disk.read.bytes: + dashed_name: host-disk-read-bytes + description: The total number of bytes (gauge) read successfully (aggregated + from all disks) since the last metric collection. + flat_name: host.disk.read.bytes + level: extended + name: disk.read.bytes + normalize: [] + short: The number of bytes read by all disks. + type: long + host.disk.write.bytes: + dashed_name: host-disk-write-bytes + description: The total number of bytes (gauge) written successfully (aggregated + from all disks) since the last metric collection. + flat_name: host.disk.write.bytes + level: extended + name: disk.write.bytes + normalize: [] + short: The number of bytes written on all disks. + type: long host.domain: dashed_name: host-domain description: 'Name of the domain of which the host is a member. @@ -4018,6 +4052,46 @@ host: normalize: [] short: Name of the host. type: keyword + host.network.egress.bytes: + dashed_name: host-network-egress-bytes + description: The number of bytes (gauge) sent out on all network interfaces + by the host since the last metric collection. + flat_name: host.network.egress.bytes + level: extended + name: network.egress.bytes + normalize: [] + short: The number of bytes sent on all network interfaces. + type: long + host.network.egress.packets: + dashed_name: host-network-egress-packets + description: The number of packets (gauge) sent out on all network interfaces + by the host since the last metric collection. + flat_name: host.network.egress.packets + level: extended + name: network.egress.packets + normalize: [] + short: The number of packets sent on all network interfaces. + type: long + host.network.ingress.bytes: + dashed_name: host-network-ingress-bytes + description: The number of bytes received (gauge) on all network interfaces + by the host since the last metric collection. + flat_name: host.network.ingress.bytes + level: extended + name: network.ingress.bytes + normalize: [] + short: The number of bytes received on all network interfaces. + type: long + host.network.ingress.packets: + dashed_name: host-network-ingress-packets + description: The number of packets (gauge) received on all network interfaces + by the host since the last metric collection. + flat_name: host.network.ingress.packets + level: extended + name: network.ingress.packets + normalize: [] + short: The number of packets received on all network interfaces. + type: long host.os.family: dashed_name: host-os-family description: OS family (such as redhat, debian, freebsd, windows). diff --git a/experimental/generated/elasticsearch/7/template.json b/experimental/generated/elasticsearch/7/template.json index dfa18031da..73e91eb32c 100644 --- a/experimental/generated/elasticsearch/7/template.json +++ b/experimental/generated/elasticsearch/7/template.json @@ -1046,6 +1046,32 @@ "ignore_above": 1024, "type": "keyword" }, + "cpu": { + "properties": { + "usage": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "disk": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, "domain": { "ignore_above": 1024, "type": "keyword" @@ -1102,6 +1128,30 @@ "ignore_above": 1024, "type": "keyword" }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, "os": { "properties": { "family": { diff --git a/experimental/schemas/host.yml b/experimental/schemas/host.yml index 91f3d1bbc2..1185d03e0b 100644 --- a/experimental/schemas/host.yml +++ b/experimental/schemas/host.yml @@ -2,3 +2,64 @@ fields: - name: hostname type: wildcard + + # RFC 0005 + - name: cpu.usage + type: scaled_float + scaling_factor: 1000 + level: extended + short: Percent CPU used, between 0 and 1. + description: > + Percent CPU used which is normalized by the number of CPU cores and it + ranges from 0 to 1. Scaling factor: 1000. + + For example: For a two core host, this value should be the average of the + two cores, between 0 and 1. + + - name: network.ingress.bytes + type: long + level: extended + short: The number of bytes received on all network interfaces. + description: > + The number of bytes received (gauge) on all network interfaces by the + host since the last metric collection. + + - name: network.ingress.packets + type: long + level: extended + short: The number of packets received on all network interfaces. + description: > + The number of packets (gauge) received on all network interfaces by the + host since the last metric collection. + + - name: network.egress.bytes + type: long + level: extended + short: The number of bytes sent on all network interfaces. + description: > + The number of bytes (gauge) sent out on all network interfaces by the + host since the last metric collection. + + - name: network.egress.packets + type: long + level: extended + short: The number of packets sent on all network interfaces. + description: > + The number of packets (gauge) sent out on all network interfaces by the + host since the last metric collection. + + - name: disk.read.bytes + type: long + level: extended + short: The number of bytes read by all disks. + description: > + The total number of bytes (gauge) read successfully (aggregated from all + disks) since the last metric collection. + + - name: disk.write.bytes + type: long + level: extended + short: The number of bytes written on all disks. + description: > + The total number of bytes (gauge) written successfully (aggregated from + all disks) since the last metric collection. From 534b21323133c8f353f1d0b74916808ca2ae7ad2 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Thu, 10 Dec 2020 14:05:22 -0600 Subject: [PATCH 2/2] update artifacts --- experimental/generated/csv/fields.csv | 7 +++ .../elasticsearch/component/host.json | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 2c83d5823d..1912a88568 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -230,6 +230,9 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.8.0-dev+exp,true,group,group.id,keyword,extended,,,Unique identifier for the group on the system/platform. 1.8.0-dev+exp,true,group,group.name,keyword,extended,,,Name of the group. 1.8.0-dev+exp,true,host,host.architecture,keyword,core,,x86_64,Operating system architecture. +1.8.0-dev+exp,true,host,host.cpu.usage,scaled_float,extended,,,"Percent CPU used, between 0 and 1." +1.8.0-dev+exp,true,host,host.disk.read.bytes,long,extended,,,The number of bytes read by all disks. +1.8.0-dev+exp,true,host,host.disk.write.bytes,long,extended,,,The number of bytes written on all disks. 1.8.0-dev+exp,true,host,host.domain,keyword,extended,,CONTOSO,Name of the directory the group is a member of. 1.8.0-dev+exp,true,host,host.geo.city_name,keyword,core,,Montreal,City name. 1.8.0-dev+exp,true,host,host.geo.continent_name,keyword,core,,North America,Name of the continent. @@ -244,6 +247,10 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 1.8.0-dev+exp,true,host,host.ip,ip,core,array,,Host ip addresses. 1.8.0-dev+exp,true,host,host.mac,keyword,core,array,,Host mac addresses. 1.8.0-dev+exp,true,host,host.name,keyword,core,,,Name of the host. +1.8.0-dev+exp,true,host,host.network.egress.bytes,long,extended,,,The number of bytes sent on all network interfaces. +1.8.0-dev+exp,true,host,host.network.egress.packets,long,extended,,,The number of packets sent on all network interfaces. +1.8.0-dev+exp,true,host,host.network.ingress.bytes,long,extended,,,The number of bytes received on all network interfaces. +1.8.0-dev+exp,true,host,host.network.ingress.packets,long,extended,,,The number of packets received on all network interfaces. 1.8.0-dev+exp,true,host,host.os.family,keyword,extended,,debian,"OS family (such as redhat, debian, freebsd, windows)." 1.8.0-dev+exp,true,host,host.os.full,wildcard,extended,,Mac OS Mojave,"Operating system name, including the version or code name." 1.8.0-dev+exp,true,host,host.os.full.text,text,extended,,Mac OS Mojave,"Operating system name, including the version or code name." diff --git a/experimental/generated/elasticsearch/component/host.json b/experimental/generated/elasticsearch/component/host.json index 19c9898702..72a4bf410b 100644 --- a/experimental/generated/elasticsearch/component/host.json +++ b/experimental/generated/elasticsearch/component/host.json @@ -12,6 +12,32 @@ "ignore_above": 1024, "type": "keyword" }, + "cpu": { + "properties": { + "usage": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "disk": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, "domain": { "ignore_above": 1024, "type": "keyword" @@ -68,6 +94,30 @@ "ignore_above": 1024, "type": "keyword" }, + "network": { + "properties": { + "egress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "ingress": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, "os": { "properties": { "family": {