[breaking] Make default_field: false the default for all fields (#28596…

…) (#28855) Changes the default value of the default_field flag in fields definitions to false. This means that only fields that are explicitly marked with default_fields: true (or their subfields) will be added to the index template's setting.index.query.default_field list. After this PR, all fields are excluded from default_field, except: - Selected fields from ECS. The ECS team maintains the list of fields that are included. - Fields for processors. - Fields for Filebeat inputs. (cherry picked from commit 84e668c) Co-authored-by: Adrian Serrano <adrisr83@gmail.com>
elastic · Nov 8, 2021 · ada8eae · ada8eae
1 parent 0d6a448
commit ada8eae
Show file tree

Hide file tree

Showing 162 changed files with 87 additions and 250 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -25,6 +25,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671]
 - add_process_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
+- Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
 
 *Auditbeat*
 

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/pensando/dfw/_meta/fields.yml b/filebeat/module/pensando/dfw/_meta/fields.yml
@@ -1,7 +1,6 @@
 - name: dfw 
   type: group
   release: beta
-  default_field: false
   description: >
    Fields for Pensando DFW
   fields:

diff --git a/filebeat/module/pensando/fields.go b/filebeat/module/pensando/fields.go
diff --git a/heartbeat/_meta/fields.common.yml b/heartbeat/_meta/fields.common.yml
@@ -478,7 +478,6 @@
               ignore_above: 1024
               description: Version of x509 format.
               example: 3
-              default_field: false
 
 - key: icmp
   title: "ICMP"

diff --git a/heartbeat/include/fields.go b/heartbeat/include/fields.go
diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go
diff --git a/libbeat/autodiscover/providers/jolokia/_meta/fields.yml b/libbeat/autodiscover/providers/jolokia/_meta/fields.yml
@@ -4,30 +4,37 @@
     Metadata from Jolokia Discovery added by the jolokia provider.
   fields:
     - name: jolokia.agent.version
+      default_field: true
       type: keyword
       description: >
         Version number of jolokia agent.
     - name: jolokia.agent.id
+      default_field: true
       type: keyword
       description: >
         Each agent has a unique id which can be either provided during startup of the agent in form of a configuration parameter or being autodetected. If autodected, the id has several parts: The IP, the process id, hashcode of the agent and its type.
     - name: jolokia.server.product
+      default_field: true
       type: keyword
       description: >
         The container product if detected.
     - name: jolokia.server.version
+      default_field: true
       type: keyword
       description: >
         The container's version (if detected).
     - name: jolokia.server.vendor
+      default_field: true
       type: keyword
       description: >
         The vendor of the container the agent is running in.
     - name: jolokia.url
+      default_field: true
       type: keyword
       description: >
         The URL how this agent can be contacted.
     - name: jolokia.secured
+      default_field: true
       type: boolean
       description: >
         Whether the agent was configured for authentication or not.
diff --git a/libbeat/processors/add_cloud_metadata/_meta/fields.yml b/libbeat/processors/add_cloud_metadata/_meta/fields.yml
@@ -5,42 +5,50 @@
   fields:
 
     - name: cloud.image.id
+      default_field: true
       example: ami-abcd1234
       description: >
         Image ID for the cloud instance.
 
     # Alias for old fields
     - name: meta.cloud.provider
+      default_field: true
       type: alias
       path: cloud.provider
       migration: true
 
     - name: meta.cloud.instance_id
+      default_field: true
       type: alias
       path: cloud.instance.id
       migration: true
 
     - name: meta.cloud.instance_name
+      default_field: true
       type: alias
       path: cloud.instance.name
       migration: true
 
     - name: meta.cloud.machine_type
+      default_field: true
       type: alias
       path: cloud.machine.type
       migration: true
 
     - name: meta.cloud.availability_zone
+      default_field: true
       type: alias
       path: cloud.availability_zone
       migration: true
 
     - name: meta.cloud.project_id
+      default_field: true
       type: alias
       path: cloud.project.id
       migration: true
 
     - name: meta.cloud.region
+      default_field: true
       type: alias
       path: cloud.region
       migration: true

diff --git a/libbeat/processors/add_docker_metadata/_meta/fields.yml b/libbeat/processors/add_docker_metadata/_meta/fields.yml
@@ -6,6 +6,7 @@
   anchor: docker-processor
   fields:
     - name: docker
+      default_field: true
       type: group
       fields:
         - name: container.id

diff --git a/libbeat/processors/add_host_metadata/_meta/fields.yml b/libbeat/processors/add_host_metadata/_meta/fields.yml
@@ -1,4 +1,5 @@
 - key: host
+  default_field: true
   title: Host
   description: >
     Info collected for the host machine.
@@ -8,6 +9,7 @@
     # ECS fields are in fields.ecs.yml.
     # These are the non-ECS fields.
     - name: host
+      default_field: true
       type: group
       fields:
 

diff --git a/libbeat/processors/add_kubernetes_metadata/_meta/fields.yml b/libbeat/processors/add_kubernetes_metadata/_meta/fields.yml
@@ -6,6 +6,7 @@
   anchor: kubernetes-processor
   fields:
     - name: kubernetes
+      default_field: true
       type: group
       fields:
         - name: pod.name

diff --git a/libbeat/processors/add_process_metadata/_meta/fields.yml b/libbeat/processors/add_process_metadata/_meta/fields.yml
@@ -4,6 +4,7 @@
     Process metadata fields
   fields:
     - name: process
+      default_field: true
       type: group
       fields:
         - name: exe
@@ -25,7 +26,6 @@
               - name: text
                 type: text
                 norms: false
-                default_field: false
               description: Short name or login of the user.
               example: albert
 
diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go
@@ -27,7 +27,7 @@ import (
 )
 
 // DefaultField controls the default value for the default_field flag.
-const DefaultField = true
+const DefaultField = false
 
 var (
 	minVersionAlias                   = common.MustNewVersion("6.4.0")

diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go
@@ -646,7 +646,7 @@ func TestProcessDefaultField(t *testing.T) {
 	)
 
 	fields := mapping.Fields{
-		// By default foo will be included in default_field.
+		// By default foo will be excluded in default_field.
 		mapping.Field{
 			Name: "foo",
 			Type: "keyword",
@@ -694,8 +694,9 @@ func TestProcessDefaultField(t *testing.T) {
 		},
 		// Check that multi_fields are correctly stored in defaultFields.
 		mapping.Field{
-			Name: "qux",
-			Type: "keyword",
+			Name:         "qux",
+			Type:         "keyword",
+			DefaultField: &enableDefaultField,
 			MultiFields: []mapping.Field{
 				{
 					Name: "text",
@@ -742,7 +743,6 @@ func TestProcessDefaultField(t *testing.T) {
 
 	expectedFields := []string{
 		"bar",
-		"foo",
 		"nested.bar",
 		"nested.foo",
 		"qux",
@@ -752,7 +752,7 @@ func TestProcessDefaultField(t *testing.T) {
 	}
 	sort.Strings(defaultFields)
 	sort.Strings(expectedFields)
-	assert.Equal(t, defaultFields, expectedFields)
+	assert.Equal(t, expectedFields, defaultFields)
 }
 
 func TestProcessWildcardOSS(t *testing.T) {

diff --git a/metricbeat/include/fields/fields.go b/metricbeat/include/fields/fields.go
diff --git a/packetbeat/include/fields.go b/packetbeat/include/fields.go
diff --git a/winlogbeat/include/fields.go b/winlogbeat/include/fields.go
diff --git a/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml b/x-pack/filebeat/input/awscloudwatch/_meta/fields.yml
@@ -4,8 +4,8 @@
     Fields from AWS CloudWatch logs.
   fields:
     - name: aws-cloudwatch
+      default_field: true
       type: group
-      default_field: false
       description: >
         Fields from AWS CloudWatch logs.
       fields:

diff --git a/x-pack/filebeat/input/awscloudwatch/fields.go b/x-pack/filebeat/input/awscloudwatch/fields.go
diff --git a/x-pack/filebeat/input/awss3/_meta/fields.yml b/x-pack/filebeat/input/awss3/_meta/fields.yml
@@ -5,18 +5,22 @@
   release: ga
   fields:
     - name: bucket.name
+      default_field: true
       type: keyword
       description: >
         Name of the S3 bucket that this log retrieved from.
     - name: bucket.arn
+      default_field: true
       type: keyword
       description: >
         ARN of the S3 bucket that this log retrieved from.
     - name: object.key
+      default_field: true
       type: keyword
       description: >
         Name of the S3 object that this log retrieved from.
     - name: metadata
+      default_field: true
       type: flattened
       description:
         AWS S3 object metadata values.
diff --git a/x-pack/filebeat/input/awss3/fields.go b/x-pack/filebeat/input/awss3/fields.go
diff --git a/x-pack/filebeat/input/netflow/_meta/fields.header.yml b/x-pack/filebeat/input/netflow/_meta/fields.header.yml
@@ -5,7 +5,6 @@
   fields:
     - name: netflow
       type: group
-      default_field: false
       description: >
         Fields from NetFlow and IPFIX.
       fields:

diff --git a/x-pack/filebeat/input/netflow/_meta/fields.yml b/x-pack/filebeat/input/netflow/_meta/fields.yml
@@ -8,7 +8,6 @@
   fields:
     - name: netflow
       type: group
-      default_field: false
       description: >
         Fields from NetFlow and IPFIX.
       fields: