From 00ef2300ceacbed42557f36b72d2f239a53530ef Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Thu, 16 Dec 2021 09:04:32 -0600 Subject: [PATCH 1/2] [Filebeat] Enable dynamic inputs (TCP) for Cisco syslog modules (#26159) - Add tcp option to asa, ftd & ios filesets - Add SSL option Closes #28821 Co-authored-by: Lee E. Hinman --- CHANGELOG.next.asciidoc | 31 ++++++++++++++++ x-pack/filebeat/filebeat.reference.yml | 37 ++++++++++++++----- x-pack/filebeat/module/cisco/_meta/config.yml | 37 ++++++++++++++----- .../module/cisco/asa/config/input.yml | 14 +++---- x-pack/filebeat/module/cisco/asa/manifest.yml | 3 +- .../module/cisco/ftd/config/input.yml | 13 ++++--- x-pack/filebeat/module/cisco/ftd/manifest.yml | 3 +- .../module/cisco/ios/config/input.yml | 14 +++---- x-pack/filebeat/module/cisco/ios/manifest.yml | 2 + x-pack/filebeat/modules.d/cisco.yml.disabled | 37 ++++++++++++++----- 10 files changed, 139 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c807a2fe233..2b0c4f4e7fb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -67,6 +67,37 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* +- `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358] +- Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010] +- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836] +- Work on e2e ACK's for the azure-eventhub input {issue}15671[15671] {pull}16215[16215] +- Add a TLS test and more debug output to httpjson input {pull}16315[16315] +- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] +- Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] +- Add a TLS test and more debug output to httpjson input {pull}16315[16315] +- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] +- Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] +- Release Google Cloud module as GA. {pull}17511[17511] +- Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] +- Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729] +- Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] +- Release Google Cloud module as GA. {pull}17511[17511] +- Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] +- Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] +- Add support for array parsing in azure-eventhub input. {pull}18585[18585] +- Add support for array parsing in azure-eventhub input. {pull}18585[18585] +- Improved performance of PANW sample dashboards. {issue}19031[19031] {pull}19032[19032] +- Add event.ingested for CrowdStrike module {pull}20138[20138] +- Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] +- Azure signinlogs - Add support for ManagedIdentitySignInLogs, NonInteractiveUserSignInLogs, and ServicePrincipalSignInLogs. {issue}23653[23653] +- Add `text/csv` decoder to `httpjson` input {pull}28564[28564] +- Update `aws-s3` input to connect to non AWS S3 buckets {issue}28222[28222] {pull}28234[28234] +- Add support for '/var/log/pods/' path for add_kubernetes_metadata processor with `resource_type: pod`. {pull}28868[28868] +- Add documentation for add_kubernetes_metadata processors `log_path` matcher. {pull}28868[28868] +- Add support for parsers on journald input {pull}29070[29070] +- Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087] +- Update Cisco module to enable TCP input. {issue}26118[26118] {issue}28821[28821] {pull}26159[26159] + *Heartbeat* diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 2e54a6404ac..2cfb04dc75c 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -625,16 +625,23 @@ filebeat.modules: asa: enabled: true - # Set which input to use between syslog (default) or file. - #var.input: syslog + # Set which input to use between udp (default), tcp or file. + #var.input: udp - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to udp or tcp syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9001. + # The port to listen for udp or tcp syslog traffic. Defaults to 9001. #var.syslog_port: 9001 + # With tcp input, set the optional tls configuration: + #var.ssl: + # enabled: true + # certificate: /path/to/cert.pem + # key: /path/to/privatekey.pem + # key_passphrase: 'password for my key' + # Set the log level from 1 (alerts only) to 7 (include all messages). # Messages with a log level higher than the specified will be dropped. # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html @@ -651,16 +658,23 @@ filebeat.modules: ftd: enabled: true - # Set which input to use between syslog (default) or file. - #var.input: syslog + # Set which input to use between udp (default), tcp or file. + #var.input: udp - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to tcp or udp syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9003. + # The UDP port to listen for tcp or udp syslog traffic. Defaults to 9003. #var.syslog_port: 9003 + # With tcp input, set the optional tls configuration: + #var.ssl: + # enabled: true + # certificate: /path/to/cert.pem + # key: /path/to/privatekey.pem + # key_passphrase: 'password for my key' + # Set the log level from 1 (alerts only) to 7 (include all messages). # Messages with a log level higher than the specified will be dropped. # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html @@ -680,13 +694,16 @@ filebeat.modules: # Set which input to use between syslog (default) or file. #var.input: syslog - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9002. + # The port to listen on for syslog traffic. Defaults to 9002. #var.syslog_port: 9002 + # Set which protocol to use between udp (default) or tcp. + #var.syslog_protocol: udp + # Set custom paths for the log files when using file input. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: diff --git a/x-pack/filebeat/module/cisco/_meta/config.yml b/x-pack/filebeat/module/cisco/_meta/config.yml index 3af897a1225..04be301586b 100644 --- a/x-pack/filebeat/module/cisco/_meta/config.yml +++ b/x-pack/filebeat/module/cisco/_meta/config.yml @@ -2,16 +2,23 @@ asa: enabled: true - # Set which input to use between syslog (default) or file. - #var.input: syslog + # Set which input to use between udp (default), tcp or file. + #var.input: udp - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to udp or tcp syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9001. + # The port to listen for udp or tcp syslog traffic. Defaults to 9001. #var.syslog_port: 9001 + # With tcp input, set the optional tls configuration: + #var.ssl: + # enabled: true + # certificate: /path/to/cert.pem + # key: /path/to/privatekey.pem + # key_passphrase: 'password for my key' + # Set the log level from 1 (alerts only) to 7 (include all messages). # Messages with a log level higher than the specified will be dropped. # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html @@ -28,16 +35,23 @@ ftd: enabled: true - # Set which input to use between syslog (default) or file. - #var.input: syslog + # Set which input to use between udp (default), tcp or file. + #var.input: udp - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to tcp or udp syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9003. + # The UDP port to listen for tcp or udp syslog traffic. Defaults to 9003. #var.syslog_port: 9003 + # With tcp input, set the optional tls configuration: + #var.ssl: + # enabled: true + # certificate: /path/to/cert.pem + # key: /path/to/privatekey.pem + # key_passphrase: 'password for my key' + # Set the log level from 1 (alerts only) to 7 (include all messages). # Messages with a log level higher than the specified will be dropped. # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html @@ -57,13 +71,16 @@ # Set which input to use between syslog (default) or file. #var.input: syslog - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9002. + # The port to listen on for syslog traffic. Defaults to 9002. #var.syslog_port: 9002 + # Set which protocol to use between udp (default) or tcp. + #var.syslog_protocol: udp + # Set custom paths for the log files when using file input. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: diff --git a/x-pack/filebeat/module/cisco/asa/config/input.yml b/x-pack/filebeat/module/cisco/asa/config/input.yml index 4237b4d9ae2..cb9df5bd6ec 100644 --- a/x-pack/filebeat/module/cisco/asa/config/input.yml +++ b/x-pack/filebeat/module/cisco/asa/config/input.yml @@ -1,10 +1,4 @@ -{{ if eq .input "syslog" }} - -type: udp -udp: -host: "{{.syslog_host}}:{{.syslog_port}}" - -{{ else if eq .input "file" }} +{{ if eq .input "file" }} type: log paths: @@ -13,6 +7,12 @@ paths: {{ end }} exclude_files: [".gz$"] +{{ else }} + +type: {{.input}} +host: "{{.syslog_host}}:{{.syslog_port}}" +ssl: {{ .ssl | tojson }} + {{ end }} tags: {{.tags | tojson}} diff --git a/x-pack/filebeat/module/cisco/asa/manifest.yml b/x-pack/filebeat/module/cisco/asa/manifest.yml index 3c185f7980c..184df5404ad 100644 --- a/x-pack/filebeat/module/cisco/asa/manifest.yml +++ b/x-pack/filebeat/module/cisco/asa/manifest.yml @@ -11,7 +11,8 @@ var: - name: syslog_port default: 9001 - name: input - default: syslog + default: udp + - name: ssl - name: log_level default: 7 # if ES < 6.1.0, this flag switches to false automatically when evaluating the diff --git a/x-pack/filebeat/module/cisco/ftd/config/input.yml b/x-pack/filebeat/module/cisco/ftd/config/input.yml index b29aa4c725f..cb9df5bd6ec 100644 --- a/x-pack/filebeat/module/cisco/ftd/config/input.yml +++ b/x-pack/filebeat/module/cisco/ftd/config/input.yml @@ -1,9 +1,4 @@ -{{ if eq .input "syslog" }} - -type: udp -host: "{{.syslog_host}}:{{.syslog_port}}" - -{{ else if eq .input "file" }} +{{ if eq .input "file" }} type: log paths: @@ -12,6 +7,12 @@ paths: {{ end }} exclude_files: [".gz$"] +{{ else }} + +type: {{.input}} +host: "{{.syslog_host}}:{{.syslog_port}}" +ssl: {{ .ssl | tojson }} + {{ end }} tags: {{.tags | tojson}} diff --git a/x-pack/filebeat/module/cisco/ftd/manifest.yml b/x-pack/filebeat/module/cisco/ftd/manifest.yml index 31eb9659a6b..d681ff4d323 100644 --- a/x-pack/filebeat/module/cisco/ftd/manifest.yml +++ b/x-pack/filebeat/module/cisco/ftd/manifest.yml @@ -11,7 +11,8 @@ var: - name: syslog_port default: 9003 - name: input - default: syslog + default: udp + - name: ssl - name: log_level default: 7 # if ES < 6.1.0, this flag switches to false automatically when evaluating the diff --git a/x-pack/filebeat/module/cisco/ios/config/input.yml b/x-pack/filebeat/module/cisco/ios/config/input.yml index d911aa3ed9e..979f9cf380b 100644 --- a/x-pack/filebeat/module/cisco/ios/config/input.yml +++ b/x-pack/filebeat/module/cisco/ios/config/input.yml @@ -1,10 +1,4 @@ -{{ if eq .input "syslog" }} - -type: syslog -protocol.udp: - host: "{{.syslog_host}}:{{.syslog_port}}" - -{{ else if eq .input "file" }} +{{ if eq .input "file" }} type: log paths: @@ -13,6 +7,12 @@ paths: {{ end }} exclude_files: [".gz$"] +{{ else if eq .input "syslog" }} + +type: syslog +protocol.{{.syslog_protocol}}: + host: "{{.syslog_host}}:{{.syslog_port}}" + {{ end }} tags: {{.tags | tojson}} diff --git a/x-pack/filebeat/module/cisco/ios/manifest.yml b/x-pack/filebeat/module/cisco/ios/manifest.yml index e67f5c2f729..169e909fd89 100644 --- a/x-pack/filebeat/module/cisco/ios/manifest.yml +++ b/x-pack/filebeat/module/cisco/ios/manifest.yml @@ -10,6 +10,8 @@ var: default: localhost - name: syslog_port default: 9002 + - name: syslog_protocol + default: udp - name: input default: syslog diff --git a/x-pack/filebeat/modules.d/cisco.yml.disabled b/x-pack/filebeat/modules.d/cisco.yml.disabled index 32d1d5ecac6..a4ef5c7d706 100644 --- a/x-pack/filebeat/modules.d/cisco.yml.disabled +++ b/x-pack/filebeat/modules.d/cisco.yml.disabled @@ -5,16 +5,23 @@ asa: enabled: true - # Set which input to use between syslog (default) or file. - #var.input: syslog + # Set which input to use between udp (default), tcp or file. + #var.input: udp - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to udp or tcp syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9001. + # The port to listen for udp or tcp syslog traffic. Defaults to 9001. #var.syslog_port: 9001 + # With tcp input, set the optional tls configuration: + #var.ssl: + # enabled: true + # certificate: /path/to/cert.pem + # key: /path/to/privatekey.pem + # key_passphrase: 'password for my key' + # Set the log level from 1 (alerts only) to 7 (include all messages). # Messages with a log level higher than the specified will be dropped. # See https://www.cisco.com/c/en/us/td/docs/security/asa/syslog/b_syslog/syslogs-sev-level.html @@ -31,16 +38,23 @@ ftd: enabled: true - # Set which input to use between syslog (default) or file. - #var.input: syslog + # Set which input to use between udp (default), tcp or file. + #var.input: udp - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to tcp or udp syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9003. + # The UDP port to listen for tcp or udp syslog traffic. Defaults to 9003. #var.syslog_port: 9003 + # With tcp input, set the optional tls configuration: + #var.ssl: + # enabled: true + # certificate: /path/to/cert.pem + # key: /path/to/privatekey.pem + # key_passphrase: 'password for my key' + # Set the log level from 1 (alerts only) to 7 (include all messages). # Messages with a log level higher than the specified will be dropped. # See https://www.cisco.com/c/en/us/td/docs/security/firepower/Syslogs/b_fptd_syslog_guide/syslogs-sev-level.html @@ -60,13 +74,16 @@ # Set which input to use between syslog (default) or file. #var.input: syslog - # The interface to listen to UDP based syslog traffic. Defaults to + # The interface to listen to syslog traffic. Defaults to # localhost. Set to 0.0.0.0 to bind to all available interfaces. #var.syslog_host: localhost - # The UDP port to listen for syslog traffic. Defaults to 9002. + # The port to listen on for syslog traffic. Defaults to 9002. #var.syslog_port: 9002 + # Set which protocol to use between udp (default) or tcp. + #var.syslog_protocol: udp + # Set custom paths for the log files when using file input. If left empty, # Filebeat will choose the paths depending on your OS. #var.paths: From 04195a271ec796a2d2b5230aed669292580e4927 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 10 Feb 2022 00:51:01 -0500 Subject: [PATCH 2/2] [Filebeat] Fix Cisco ASA/FTD configs that used var.input syslog (#30072) and FTD that specified `var.input: syslog`. `syslog` was effectively an alias for the UDP input and the alias support was removed. This change allows `var.input: syslog` to continue working as it did before. --- CHANGELOG.next.asciidoc | 33 ++----------------- .../module/cisco/asa/config/input.yml | 4 +++ .../module/cisco/ftd/config/input.yml | 4 +++ 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2b0c4f4e7fb..33da349015e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -38,7 +38,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* + - Fix broken Kafka input {issue}29746[29746] {pull}30277[30277] +- cisco module: Fix change the broke ASA and FTD configs that used `var.input: syslog`. {pull}30072[30072] *Heartbeat* @@ -67,37 +69,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Filebeat* -- `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358] -- Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010] -- move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836] -- Work on e2e ACK's for the azure-eventhub input {issue}15671[15671] {pull}16215[16215] -- Add a TLS test and more debug output to httpjson input {pull}16315[16315] -- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] -- Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] -- Add a TLS test and more debug output to httpjson input {pull}16315[16315] -- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320] -- Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] -- Release Google Cloud module as GA. {pull}17511[17511] -- Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] -- Enhance `elasticsearch/slowlog` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17729[17729] -- Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] -- Release Google Cloud module as GA. {pull}17511[17511] -- Update filebeat httpjson input to support pagination via Header and Okta module. {pull}16354[16354] -- Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958] -- Add support for array parsing in azure-eventhub input. {pull}18585[18585] -- Add support for array parsing in azure-eventhub input. {pull}18585[18585] -- Improved performance of PANW sample dashboards. {issue}19031[19031] {pull}19032[19032] -- Add event.ingested for CrowdStrike module {pull}20138[20138] -- Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138] -- Azure signinlogs - Add support for ManagedIdentitySignInLogs, NonInteractiveUserSignInLogs, and ServicePrincipalSignInLogs. {issue}23653[23653] -- Add `text/csv` decoder to `httpjson` input {pull}28564[28564] -- Update `aws-s3` input to connect to non AWS S3 buckets {issue}28222[28222] {pull}28234[28234] -- Add support for '/var/log/pods/' path for add_kubernetes_metadata processor with `resource_type: pod`. {pull}28868[28868] -- Add documentation for add_kubernetes_metadata processors `log_path` matcher. {pull}28868[28868] -- Add support for parsers on journald input {pull}29070[29070] -- Add support in httpjson input for oAuth2ProviderDefault of password grant_type. {pull}29087[29087] -- Update Cisco module to enable TCP input. {issue}26118[26118] {issue}28821[28821] {pull}26159[26159] - *Heartbeat* diff --git a/x-pack/filebeat/module/cisco/asa/config/input.yml b/x-pack/filebeat/module/cisco/asa/config/input.yml index cb9df5bd6ec..b28c6bf9aab 100644 --- a/x-pack/filebeat/module/cisco/asa/config/input.yml +++ b/x-pack/filebeat/module/cisco/asa/config/input.yml @@ -7,6 +7,10 @@ paths: {{ end }} exclude_files: [".gz$"] +{{ else if eq .input "syslog" }} +type: udp +host: "{{.syslog_host}}:{{.syslog_port}}" + {{ else }} type: {{.input}} diff --git a/x-pack/filebeat/module/cisco/ftd/config/input.yml b/x-pack/filebeat/module/cisco/ftd/config/input.yml index cb9df5bd6ec..b28c6bf9aab 100644 --- a/x-pack/filebeat/module/cisco/ftd/config/input.yml +++ b/x-pack/filebeat/module/cisco/ftd/config/input.yml @@ -7,6 +7,10 @@ paths: {{ end }} exclude_files: [".gz$"] +{{ else if eq .input "syslog" }} +type: udp +host: "{{.syslog_host}}:{{.syslog_port}}" + {{ else }} type: {{.input}}