diff --git a/packages/mongodb/_dev/build/docs/README.md b/packages/mongodb/_dev/build/docs/README.md index c139450bc43c..a6bf5db9a94c 100644 --- a/packages/mongodb/_dev/build/docs/README.md +++ b/packages/mongodb/_dev/build/docs/README.md @@ -4,7 +4,8 @@ This integration is used to fetch logs and metrics from [MongoDB](https://www.mo ## Compatibility -The `log` dataset is tested with logs from versions v3.2.11 on Debian. +The `log` dataset is tested with logs from versions v3.2.11 and v4.4.4 in +plaintext and json formats. The `collstats`, `dbstats`, `metrics`, `replstatus` and `status` datasets are tested with MongoDB 3.4 and 3.0 and are expected to work with all versions >= 2.8. diff --git a/packages/mongodb/changelog.yml b/packages/mongodb/changelog.yml index a3ccdf57caa2..7786455ec051 100644 --- a/packages/mongodb/changelog.yml +++ b/packages/mongodb/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.4.0" + changes: + - description: Add support for logs in JSON format + type: enhancement + link: https://github.com/elastic/integrations/pull/1138 - version: "0.3.0" changes: - description: update to ECS 1.10.0 and add event.original option diff --git a/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-debian.log-expected.json b/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-debian.log-expected.json index b289341ad743..4b01ca833683 100644 --- a/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-debian.log-expected.json +++ b/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-debian.log-expected.json @@ -2,25 +2,21 @@ "expected": [ { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "git version: 009580ad490190ba33d1c6253ebd8d91808923e4", "event": { - "ingested": "2021-06-09T12:02:02.902214600Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] git version: 009580ad490190ba33d1c6253ebd8d91808923e4", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812084646Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "git version: 009580ad490190ba33d1c6253ebd8d91808923e4", "mongodb": { "log": { "context": "initandlisten", @@ -33,25 +29,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "modules: none", "event": { - "ingested": "2021-06-09T12:02:02.902237700Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] modules: none", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812088978Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "modules: none", "mongodb": { "log": { "context": "initandlisten", @@ -64,25 +56,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "OpenSSL version: OpenSSL 1.0.2l 25 May 2017", "event": { - "ingested": "2021-06-09T12:02:02.902245500Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] OpenSSL version: OpenSSL 1.0.2l 25 May 2017", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812090853Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "OpenSSL version: OpenSSL 1.0.2l 25 May 2017", "mongodb": { "log": { "context": "initandlisten", @@ -95,25 +83,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.677Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", "event": { - "ingested": "2021-06-09T12:02:02.902274900Z", - "original": "2018-02-05T13:44:56.677+0100 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812092720Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", "mongodb": { "log": { "context": "initandlisten", @@ -126,25 +110,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.724Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", "event": { - "ingested": "2021-06-09T12:02:02.902282300Z", - "original": "2018-02-05T13:44:56.724+0100 I FTDC [initandlisten] Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812094420Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", "mongodb": { "log": { "context": "initandlisten", @@ -157,25 +137,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.724Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "Starting hostname canonicalization worker", "event": { - "ingested": "2021-06-09T12:02:02.902288500Z", - "original": "2018-02-05T13:44:56.724+0100 I NETWORK [HostnameCanonicalizationWorker] Starting hostname canonicalization worker", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812096116Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Starting hostname canonicalization worker", "mongodb": { "log": { "context": "HostnameCanonicalizationWorker", @@ -188,25 +164,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.744Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "waiting for connections on port 27017", "event": { - "ingested": "2021-06-09T12:02:02.902295200Z", - "original": "2018-02-05T13:44:56.744+0100 I NETWORK [initandlisten] waiting for connections on port 27017", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812097795Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "waiting for connections on port 27017", "mongodb": { "log": { "context": "initandlisten", @@ -219,25 +191,21 @@ }, { "@timestamp": "2018-02-05T12:50:55.170Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "end connection 127.0.0.1:55404 (0 connections now open)", "event": { - "ingested": "2021-06-09T12:02:02.902300800Z", - "original": "2018-02-05T13:50:55.170+0100 I NETWORK [conn1] end connection 127.0.0.1:55404 (0 connections now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812099461Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "end connection 127.0.0.1:55404 (0 connections now open)", "mongodb": { "log": { "context": "conn1", @@ -250,25 +218,21 @@ }, { "@timestamp": "2018-02-05T12:50:55.487Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", "event": { - "ingested": "2021-06-09T12:02:02.902306300Z", - "original": "2018-02-05T13:50:55.487+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812101148Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", "mongodb": { "log": { "context": "initandlisten", @@ -281,25 +245,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "now exiting", "event": { - "ingested": "2021-06-09T12:02:02.902311400Z", - "original": "2018-02-05T14:49:45.606+0100 I CONTROL [signalProcessingThread] now exiting", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812102827Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "now exiting", "mongodb": { "log": { "context": "signalProcessingThread", @@ -312,25 +272,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "closing listening socket: 7", "event": { - "ingested": "2021-06-09T12:02:02.902318300Z", - "original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] closing listening socket: 7", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812104511Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "closing listening socket: 7", "mongodb": { "log": { "context": "signalProcessingThread", @@ -343,25 +299,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "removing socket file: /run/mongodb/mongodb-27017.sock", "event": { - "ingested": "2021-06-09T12:02:02.902337300Z", - "original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] removing socket file: /run/mongodb/mongodb-27017.sock", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812106466Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "removing socket file: /run/mongodb/mongodb-27017.sock", "mongodb": { "log": { "context": "signalProcessingThread", @@ -374,25 +326,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "shutdown: going to flush diaglog...", "event": { - "ingested": "2021-06-09T12:02:02.902343500Z", - "original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] shutdown: going to flush diaglog...", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812108148Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "shutdown: going to flush diaglog...", "mongodb": { "log": { "context": "signalProcessingThread", @@ -405,25 +353,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "shutdown: going to close sockets...", "event": { - "ingested": "2021-06-09T12:02:02.902348900Z", - "original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] shutdown: going to close sockets...", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812109910Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "shutdown: going to close sockets...", "mongodb": { "log": { "context": "signalProcessingThread", @@ -436,25 +380,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.688Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "shutdown: removing fs lock...", "event": { - "ingested": "2021-06-09T12:02:02.902354Z", - "original": "2018-02-05T14:49:45.688+0100 I STORAGE [signalProcessingThread] shutdown: removing fs lock...", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812111624Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "shutdown: removing fs lock...", "mongodb": { "log": { "context": "signalProcessingThread", @@ -467,25 +407,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "db version v3.2.11", "event": { - "ingested": "2021-06-09T12:02:02.902359200Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] db version v3.2.11", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812113280Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "db version v3.2.11", "mongodb": { "log": { "context": "initandlisten", @@ -498,25 +434,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "build environment:", "event": { - "ingested": "2021-06-09T12:02:02.902364600Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] build environment:", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812115548Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "build environment:", "mongodb": { "log": { "context": "initandlisten", @@ -529,25 +461,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "distarch: x86_64", "event": { - "ingested": "2021-06-09T12:02:02.902369600Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] distarch: x86_64", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812118348Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "distarch: x86_64", "mongodb": { "log": { "context": "initandlisten", @@ -560,25 +488,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", "event": { - "ingested": "2021-06-09T12:02:02.902374500Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812120999Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", "mongodb": { "log": { "context": "initandlisten", @@ -591,25 +515,21 @@ }, { "@timestamp": "2018-02-05T12:50:55.170Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", "event": { - "ingested": "2021-06-09T12:02:02.902380500Z", - "original": "2018-02-05T13:50:55.170+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812123109Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", "mongodb": { "log": { "context": "initandlisten", @@ -622,25 +542,21 @@ }, { "@timestamp": "2018-02-05T12:50:56.180Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "end connection 127.0.0.1:55414 (0 connections now open)", "event": { - "ingested": "2021-06-09T12:02:02.902396Z", - "original": "2018-02-05T13:50:56.180+0100 I NETWORK [conn3] end connection 127.0.0.1:55414 (0 connections now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812124846Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "end connection 127.0.0.1:55414 (0 connections now open)", "mongodb": { "log": { "context": "conn3", @@ -653,25 +569,21 @@ }, { "@timestamp": "2018-02-05T13:15:42.095Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "end connection 127.0.0.1:58336 (0 connections now open)", "event": { - "ingested": "2021-06-09T12:02:02.902413500Z", - "original": "2018-02-05T14:15:42.095+0100 I NETWORK [conn4] end connection 127.0.0.1:58336 (0 connections now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812126591Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "end connection 127.0.0.1:58336 (0 connections now open)", "mongodb": { "log": { "context": "conn4", @@ -684,25 +596,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "shutdown: going to close listening sockets...", "event": { - "ingested": "2021-06-09T12:02:02.902421800Z", - "original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] shutdown: going to close listening sockets...", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812128346Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "shutdown: going to close listening sockets...", "mongodb": { "log": { "context": "signalProcessingThread", @@ -715,25 +623,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "WiredTigerKVEngine shutting down", "event": { - "ingested": "2021-06-09T12:02:02.902467400Z", - "original": "2018-02-05T14:49:45.606+0100 I STORAGE [signalProcessingThread] WiredTigerKVEngine shutting down", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812130151Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "WiredTigerKVEngine shutting down", "mongodb": { "log": { "context": "signalProcessingThread", @@ -746,25 +650,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.688Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "dbexit: rc: 0", "event": { - "ingested": "2021-06-09T12:02:02.902474500Z", - "original": "2018-02-05T14:49:45.688+0100 I CONTROL [signalProcessingThread] dbexit: rc: 0", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812131856Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "dbexit: rc: 0", "mongodb": { "log": { "context": "signalProcessingThread", @@ -777,25 +677,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", "event": { - "ingested": "2021-06-09T12:02:02.902480400Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812133558Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", "mongodb": { "log": { "context": "initandlisten", @@ -808,25 +704,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "allocator: tcmalloc", "event": { - "ingested": "2021-06-09T12:02:02.902485900Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] allocator: tcmalloc", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812135229Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "allocator: tcmalloc", "mongodb": { "log": { "context": "initandlisten", @@ -839,25 +731,21 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "target_arch: x86_64", "event": { - "ingested": "2021-06-09T12:02:02.902491600Z", - "original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] target_arch: x86_64", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812136905Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "target_arch: x86_64", "mongodb": { "log": { "context": "initandlisten", @@ -870,25 +758,21 @@ }, { "@timestamp": "2018-02-05T12:50:55.487Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "end connection 127.0.0.1:55406 (0 connections now open)", "event": { - "ingested": "2021-06-09T12:02:02.902496800Z", - "original": "2018-02-05T13:50:55.487+0100 I NETWORK [conn2] end connection 127.0.0.1:55406 (0 connections now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812138594Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "end connection 127.0.0.1:55406 (0 connections now open)", "mongodb": { "log": { "context": "conn2", @@ -901,25 +785,21 @@ }, { "@timestamp": "2018-02-05T12:50:56.180Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", "event": { - "ingested": "2021-06-09T12:02:02.902501100Z", - "original": "2018-02-05T13:50:56.180+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812140354Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", "mongodb": { "log": { "context": "initandlisten", @@ -932,25 +812,21 @@ }, { "@timestamp": "2018-02-05T13:11:41.401Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", "event": { - "ingested": "2021-06-09T12:02:02.902505900Z", - "original": "2018-02-05T14:11:41.401+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812142074Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", "mongodb": { "log": { "context": "initandlisten", @@ -963,25 +839,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.605Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "got signal 15 (Terminated), will terminate after current cmd ends", "event": { - "ingested": "2021-06-09T12:02:02.902510500Z", - "original": "2018-02-05T14:49:45.605+0100 I CONTROL [signalProcessingThread] got signal 15 (Terminated), will terminate after current cmd ends", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812143769Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "got signal 15 (Terminated), will terminate after current cmd ends", "mongodb": { "log": { "context": "signalProcessingThread", @@ -994,25 +866,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.605Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "Shutting down full-time diagnostic data capture", "event": { - "ingested": "2021-06-09T12:02:02.902514800Z", - "original": "2018-02-05T14:49:45.605+0100 I FTDC [signalProcessingThread] Shutting down full-time diagnostic data capture", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812145497Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Shutting down full-time diagnostic data capture", "mongodb": { "log": { "context": "signalProcessingThread", @@ -1025,25 +893,21 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "closing listening socket: 6", "event": { - "ingested": "2021-06-09T12:02:02.902525400Z", - "original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] closing listening socket: 6", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812147163Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "closing listening socket: 6", "mongodb": { "log": { "context": "signalProcessingThread", @@ -1056,25 +920,21 @@ }, { "@timestamp": "2019-03-07T15:10:26.960Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "I" }, + "message": "Successfully connected to dbbox7:27017, took 10ms (1 connections now open to dbbox7:27017)", "event": { - "ingested": "2021-06-09T12:02:02.902530400Z", - "original": "2019-03-07T15:10:26.960+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Successfully connected to dbbox7:27017, took 10ms (1 connections now open to dbbox7:27017)", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812148999Z", "type": [ "info" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "Successfully connected to dbbox7:27017, took 10ms (1 connections now open to dbbox7:27017)", "mongodb": { "log": { "context": "NetworkInterfaceASIO-Replication-0", @@ -1087,26 +947,22 @@ }, { "@timestamp": "2020-03-31T21:19:46.942Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "E" }, + "message": "** ERROR: A write operation resulted in an error. E11000 duplicate key error index: test.people.$_id_ dup key: { : 0 }", "event": { - "ingested": "2021-06-09T12:02:02.902535100Z", - "original": "2020-03-31T21:19:46.942+0000 E WRITE [initandlisten] ** ERROR: A write operation resulted in an error. E11000 duplicate key error index: test.people.$_id_ dup key: { : 0 }", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812150688Z", "type": [ "change", "error" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "** ERROR: A write operation resulted in an error. E11000 duplicate key error index: test.people.$_id_ dup key: { : 0 }", "mongodb": { "log": { "context": "initandlisten", @@ -1119,26 +975,22 @@ }, { "@timestamp": "2020-03-31T21:19:47.420Z", - "ecs": { - "version": "1.10.0" - }, "log": { "level": "E" }, + "message": "** ERROR: No connection could be made because the target machine actively refused it 127.0.0.1:27017 at System.Net.Sockets.Socket.EndConnect", "event": { - "ingested": "2021-06-09T12:02:02.902539400Z", - "original": "2020-03-31T21:19:47.420+0000 E NETWORK [initandlisten] ** ERROR: No connection could be made because the target machine actively refused it 127.0.0.1:27017 at System.Net.Sockets.Socket.EndConnect", - "category": [ - "database" - ], + "ingested": "2021-06-17T09:12:47.812152429Z", "type": [ "info", "error" ], + "category": [ + "database" + ], "created": "2020-04-28T11:07:58.223Z", "kind": "event" }, - "message": "** ERROR: No connection could be made because the target machine actively refused it 127.0.0.1:27017 at System.Net.Sockets.Socket.EndConnect", "mongodb": { "log": { "context": "initandlisten", diff --git a/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-ubuntu-4-4-4.log b/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-ubuntu-4-4-4.log new file mode 100644 index 000000000000..9391f225742f --- /dev/null +++ b/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-ubuntu-4-4-4.log @@ -0,0 +1,7 @@ +{"t":{"$date":"2021-03-22T21:21:20.349+00:00"},"s":"I", "c":"STORAGE", "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":1,"port":27017,"dbPath":"/data/db","architecture":"64-bit","host":"6150fe65a89c"}} +{"t":{"$date":"2021-03-22T21:21:20.350+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"4.4.4","gitVersion":"8db30a63db1a9d84bdcad0c83369623f708e0397","openSSLVersion":"OpenSSL 1.1.1 11 Sep 2018","modules":[],"allocator":"tcmalloc","environment":{"distmod":"ubuntu1804","distarch":"x86_64","target_arch":"x86_64"}}}} +{"t":{"$date":"2021-03-22T21:21:26.240+00:00"},"s":"I", "c":"RECOVERY", "id":23987, "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":0,"i":0}}}} +{"t":{"$date":"2021-03-22T21:21:26.363+00:00"},"s":"I", "c":"STORAGE", "id":20320, "ctx":"initandlisten","msg":"createCollection","attr":{"namespace":"admin.system.version","uuidDisposition":"provided","uuid":{"uuid":{"$uuid":"b383f03c-b97c-4584-87ae-ab1b8ea399c3"}},"options":{"uuid":{"$uuid":"b383f03c-b97c-4584-87ae-ab1b8ea399c3"}}}} +{"t":{"$date":"2021-03-22T21:21:26.410+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"initandlisten","msg":"Index build: done building","attr":{"buildUUID":null,"namespace":"admin.system.version","index":"_id_","commitTimestamp":{"$timestamp":{"t":0,"i":0}}}} +{"t":{"$date":"2021-03-22T21:21:26.412+00:00"},"s":"I", "c":"COMMAND", "id":20459, "ctx":"initandlisten","msg":"Setting featureCompatibilityVersion","attr":{"newVersion":"4.4"}} +{"t":{"$date":"2021-03-22T21:21:26.451+00:00"},"s":"I", "c":"FTDC", "id":20625, "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"/data/db/diagnostic.data"}} diff --git a/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-ubuntu-4-4-4.log-expected.json b/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-ubuntu-4-4-4.log-expected.json new file mode 100644 index 000000000000..c6862e79299a --- /dev/null +++ b/packages/mongodb/data_stream/log/_dev/test/pipeline/test-mongodb-ubuntu-4-4-4.log-expected.json @@ -0,0 +1,207 @@ +{ + "expected": [ + { + "@timestamp": "2021-03-22T21:21:20.349Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524854135Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.349+00:00\"},\"s\":\"I\", \"c\":\"STORAGE\", \"id\":4615611, \"ctx\":\"initandlisten\",\"msg\":\"MongoDB starting\",\"attr\":{\"pid\":1,\"port\":27017,\"dbPath\":\"/data/db\",\"architecture\":\"64-bit\",\"host\":\"6150fe65a89c\"}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "MongoDB starting", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "STORAGE", + "id": 4615611 + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-03-22T21:21:20.350Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524861128Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.350+00:00\"},\"s\":\"I\", \"c\":\"CONTROL\", \"id\":23403, \"ctx\":\"initandlisten\",\"msg\":\"Build Info\",\"attr\":{\"buildInfo\":{\"version\":\"4.4.4\",\"gitVersion\":\"8db30a63db1a9d84bdcad0c83369623f708e0397\",\"openSSLVersion\":\"OpenSSL 1.1.1 11 Sep 2018\",\"modules\":[],\"allocator\":\"tcmalloc\",\"environment\":{\"distmod\":\"ubuntu1804\",\"distarch\":\"x86_64\",\"target_arch\":\"x86_64\"}}}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Build Info", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "CONTROL", + "id": 23403 + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-03-22T21:21:26.240Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524885169Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.240+00:00\"},\"s\":\"I\", \"c\":\"RECOVERY\", \"id\":23987, \"ctx\":\"initandlisten\",\"msg\":\"WiredTiger recoveryTimestamp\",\"attr\":{\"recoveryTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "WiredTiger recoveryTimestamp", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "RECOVERY", + "id": 23987 + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-03-22T21:21:26.363Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524888351Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.363+00:00\"},\"s\":\"I\", \"c\":\"STORAGE\", \"id\":20320, \"ctx\":\"initandlisten\",\"msg\":\"createCollection\",\"attr\":{\"namespace\":\"admin.system.version\",\"uuidDisposition\":\"provided\",\"uuid\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}},\"options\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}}}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "createCollection", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "STORAGE", + "id": 20320 + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-03-22T21:21:26.410Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524891317Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.410+00:00\"},\"s\":\"I\", \"c\":\"INDEX\", \"id\":20345, \"ctx\":\"initandlisten\",\"msg\":\"Index build: done building\",\"attr\":{\"buildUUID\":null,\"namespace\":\"admin.system.version\",\"index\":\"_id_\",\"commitTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Index build: done building", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "INDEX", + "id": 20345 + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-03-22T21:21:26.412Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524894043Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.412+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":20459, \"ctx\":\"initandlisten\",\"msg\":\"Setting featureCompatibilityVersion\",\"attr\":{\"newVersion\":\"4.4\"}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Setting featureCompatibilityVersion", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "COMMAND", + "id": 20459 + } + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-03-22T21:21:26.451Z", + "log": { + "level": "I" + }, + "event": { + "ingested": "2021-06-17T09:12:48.524896808Z", + "original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.451+00:00\"},\"s\":\"I\", \"c\":\"FTDC\", \"id\":20625, \"ctx\":\"initandlisten\",\"msg\":\"Initializing full-time diagnostic data capture\",\"attr\":{\"dataDirectory\":\"/data/db/diagnostic.data\"}}", + "type": [ + "info" + ], + "category": [ + "database" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event" + }, + "message": "Initializing full-time diagnostic data capture", + "mongodb": { + "log": { + "context": "initandlisten", + "component": "FTDC", + "id": 20625 + } + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 7f9d73f721d3..662d9e3ba92c 100644 --- a/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -1,59 +1,34 @@ --- description: Pipeline for parsing MongoDB logs processors: - - set: - field: event.ingested - value: '{{_ingest.timestamp}}' - - set: - field: ecs.version - value: "1.10.0" - - rename: - field: message - target_field: event.original - ignore_missing: true - - grok: - field: event.original - patterns: - - '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}' - ignore_missing: true - - rename: - field: '@timestamp' - target_field: event.created - - date: - field: mongodb.log.timestamp - target_field: '@timestamp' - formats: - - yyyy-MM-dd'T'HH:mm:ss.SSSZZ - - remove: - field: mongodb.log.timestamp - - set: - field: event.kind - value: event - - append: - field: event.category - value: database - - append: - field: event.type - value: access - if: "ctx?.mongodb?.log?.component == 'ACCESS'" - - append: - field: event.type - value: change - if: "ctx?.mongodb?.log?.component == 'WRITE'" - - append: - field: event.type - value: info - if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'" - - append: - field: event.type - value: error - if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'" - - remove: - field: event.original - if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" - ignore_failure: true - ignore_missing: true +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: '@timestamp' + target_field: event.created +- grok: + field: message + patterns: + - ^%{CHAR:first_char} + pattern_definitions: + CHAR: . +- pipeline: + if: ctx.first_char != '{' + name: '{{ IngestPipeline "pipeline-plaintext" }}' +- pipeline: + if: ctx.first_char == '{' + name: '{{ IngestPipeline "pipeline-json" }}' +- set: + field: event.kind + value: event +- append: + field: event.category + value: database +- remove: + field: + - first_char on_failure: - - set: - field: error.message - value: '{{ _ingest.on_failure_message }}' +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/pipeline-json.yml b/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/pipeline-json.yml new file mode 100644 index 000000000000..ca550988b105 --- /dev/null +++ b/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/pipeline-json.yml @@ -0,0 +1,54 @@ +--- +description: Pipeline for parsing MongoDB logs in JSON format +processors: +- json: + field: message + target_field: mongodb.log +- date: + field: mongodb.log.t.$date + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss.SSSZZZZZ +- rename: + field: mongodb.log.s + target_field: log.level +- rename: + field: mongodb.log.c + target_field: mongodb.log.component +- rename: + field: mongodb.log.ctx + target_field: mongodb.log.context +- rename: + field: message + target_field: event.original +- rename: + field: mongodb.log.msg + target_field: message +- append: + field: event.type + value: access + if: ctx.mongodb.log.component == 'ACCESS' +- append: + field: event.type + value: change + if: ctx.mongodb.log.component == 'WRITE' +- append: + field: event.type + value: info + if: ctx.mongodb.log.component != 'WRITE' && ctx.mongodb.log.component != 'ACCESS' +- append: + field: event.type + value: error + if: ctx.log.level == 'F' || ctx.log.level == 'E' +- remove: + field: + - mongodb.log.t + - mongodb.log.attr + - mongodb.log.tags + - mongodb.log.truncated + - mongodb.log.size + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/pipeline-plaintext.yml b/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/pipeline-plaintext.yml new file mode 100644 index 000000000000..9f50cf5ce149 --- /dev/null +++ b/packages/mongodb/data_stream/log/elasticsearch/ingest_pipeline/pipeline-plaintext.yml @@ -0,0 +1,35 @@ +--- +description: Pipeline for parsing MongoDB logs in plaintext +processors: +- grok: + field: message + patterns: + - '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}' + ignore_missing: true +- date: + field: mongodb.log.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss.SSSZZ +- remove: + field: mongodb.log.timestamp +- append: + field: event.type + value: access + if: "ctx?.mongodb?.log?.component == 'ACCESS'" +- append: + field: event.type + value: change + if: "ctx?.mongodb?.log?.component == 'WRITE'" +- append: + field: event.type + value: info + if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'" +- append: + field: event.type + value: error + if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'" +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/packages/mongodb/data_stream/log/fields/fields.yml b/packages/mongodb/data_stream/log/fields/fields.yml index 0b1b6585bf4f..fcf6296df861 100644 --- a/packages/mongodb/data_stream/log/fields/fields.yml +++ b/packages/mongodb/data_stream/log/fields/fields.yml @@ -9,3 +9,8 @@ type: keyword description: | Context of message + - name: id + description: | + Integer representing the unique identifier of the log statement + example: 4615611 + type: long diff --git a/packages/mongodb/docs/README.md b/packages/mongodb/docs/README.md index 004d5025c846..b9f736dc6d27 100644 --- a/packages/mongodb/docs/README.md +++ b/packages/mongodb/docs/README.md @@ -4,7 +4,8 @@ This integration is used to fetch logs and metrics from [MongoDB](https://www.mo ## Compatibility -The `log` dataset is tested with logs from versions v3.2.11 on Debian. +The `log` dataset is tested with logs from versions v3.2.11 and v4.4.4 in +plaintext and json formats. The `collstats`, `dbstats`, `metrics`, `replstatus` and `status` datasets are tested with MongoDB 3.4 and 3.0 and are expected to work with all versions >= 2.8. @@ -86,6 +87,7 @@ The `log` dataset collects the MongoDB logs. | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | text | | mongodb.log.component | Functional categorization of message | keyword | | mongodb.log.context | Context of message | keyword | +| mongodb.log.id | Integer representing the unique identifier of the log statement | long | | service.address | Service address | keyword | | service.type | Service type | keyword | | tags | List of keywords used to tag each event. | keyword | diff --git a/packages/mongodb/manifest.yml b/packages/mongodb/manifest.yml index c1b852f63aba..8df1c1e5e2ff 100644 --- a/packages/mongodb/manifest.yml +++ b/packages/mongodb/manifest.yml @@ -1,6 +1,6 @@ name: mongodb title: MongoDB -version: 0.3.0 +version: 0.4.0 description: MongoDB Integration type: integration categories: