Store message from MongoDB json logs in message field (#26338) (#26344)

(cherry picked from commit b18d1f2)

Co-authored-by: Jaime Soriano Pastor <jaime.soriano@elastic.co>

filebeat/docs/fields.asciidoc

@@ -95635,18 +95635,6 @@ alias to: message
 
 --
 
-*`mongodb.log.msg`*::
-+
---
-String representing the raw log output message as passed from the server or driver
-
-
-type: text
-
-example: MongoDB starting
-
---
-
 *`mongodb.log.id`*::
 +
 --

filebeat/module/mongodb/log/_meta/fields.yml

@@ -13,7 +13,6 @@
         Context of message
     example: initandlisten
     type: keyword
-
   - name: severity
     type: alias
     path: log.level
@@ -22,11 +21,6 @@
     type: alias
     path: message
     migration: true
-  - name: msg
-    description: >
-        String representing the raw log output message as passed from the server or driver
-    example: MongoDB starting
-    type: text
   - name: id
     description: >
         Integer representing the unique identifier of the log statement

filebeat/module/mongodb/log/ingest/pipeline-json.yml

@@ -17,6 +17,12 @@ processors:
 - rename:
     field: mongodb.log.ctx
     target_field: mongodb.log.context
+- rename:
+    field: message
+    target_field: event.original
+- rename:
+    field: mongodb.log.msg
+    target_field: message
 - append:
     field: event.type
     value: access

filebeat/module/mongodb/log/test/mongodb-ubuntu-4.4.4-json.log-expected.json

@@ -7,18 +7,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.349+00:00\"},\"s\":\"I\",  \"c\":\"STORAGE\",  \"id\":4615611, \"ctx\":\"initandlisten\",\"msg\":\"MongoDB starting\",\"attr\":{\"pid\":1,\"port\":27017,\"dbPath\":\"/data/db\",\"architecture\":\"64-bit\",\"host\":\"6150fe65a89c\"}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 0,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.349+00:00\"},\"s\":\"I\",  \"c\":\"STORAGE\",  \"id\":4615611, \"ctx\":\"initandlisten\",\"msg\":\"MongoDB starting\",\"attr\":{\"pid\":1,\"port\":27017,\"dbPath\":\"/data/db\",\"architecture\":\"64-bit\",\"host\":\"6150fe65a89c\"}}",
+        "message": "MongoDB starting",
         "mongodb.log.component": "STORAGE",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 4615611,
-        "mongodb.log.msg": "MongoDB starting",
         "service.type": "mongodb"
     },
     {
@@ -29,18 +29,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.350+00:00\"},\"s\":\"I\",  \"c\":\"CONTROL\",  \"id\":23403,   \"ctx\":\"initandlisten\",\"msg\":\"Build Info\",\"attr\":{\"buildInfo\":{\"version\":\"4.4.4\",\"gitVersion\":\"8db30a63db1a9d84bdcad0c83369623f708e0397\",\"openSSLVersion\":\"OpenSSL 1.1.1  11 Sep 2018\",\"modules\":[],\"allocator\":\"tcmalloc\",\"environment\":{\"distmod\":\"ubuntu1804\",\"distarch\":\"x86_64\",\"target_arch\":\"x86_64\"}}}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 231,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:20.350+00:00\"},\"s\":\"I\",  \"c\":\"CONTROL\",  \"id\":23403,   \"ctx\":\"initandlisten\",\"msg\":\"Build Info\",\"attr\":{\"buildInfo\":{\"version\":\"4.4.4\",\"gitVersion\":\"8db30a63db1a9d84bdcad0c83369623f708e0397\",\"openSSLVersion\":\"OpenSSL 1.1.1  11 Sep 2018\",\"modules\":[],\"allocator\":\"tcmalloc\",\"environment\":{\"distmod\":\"ubuntu1804\",\"distarch\":\"x86_64\",\"target_arch\":\"x86_64\"}}}}",
+        "message": "Build Info",
         "mongodb.log.component": "CONTROL",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 23403,
-        "mongodb.log.msg": "Build Info",
         "service.type": "mongodb"
     },
     {
@@ -51,18 +51,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.240+00:00\"},\"s\":\"I\",  \"c\":\"RECOVERY\", \"id\":23987,   \"ctx\":\"initandlisten\",\"msg\":\"WiredTiger recoveryTimestamp\",\"attr\":{\"recoveryTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 621,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.240+00:00\"},\"s\":\"I\",  \"c\":\"RECOVERY\", \"id\":23987,   \"ctx\":\"initandlisten\",\"msg\":\"WiredTiger recoveryTimestamp\",\"attr\":{\"recoveryTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
+        "message": "WiredTiger recoveryTimestamp",
         "mongodb.log.component": "RECOVERY",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 23987,
-        "mongodb.log.msg": "WiredTiger recoveryTimestamp",
         "service.type": "mongodb"
     },
     {
@@ -73,18 +73,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.363+00:00\"},\"s\":\"I\",  \"c\":\"STORAGE\",  \"id\":20320,   \"ctx\":\"initandlisten\",\"msg\":\"createCollection\",\"attr\":{\"namespace\":\"admin.system.version\",\"uuidDisposition\":\"provided\",\"uuid\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}},\"options\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}}}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 826,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.363+00:00\"},\"s\":\"I\",  \"c\":\"STORAGE\",  \"id\":20320,   \"ctx\":\"initandlisten\",\"msg\":\"createCollection\",\"attr\":{\"namespace\":\"admin.system.version\",\"uuidDisposition\":\"provided\",\"uuid\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}},\"options\":{\"uuid\":{\"$uuid\":\"b383f03c-b97c-4584-87ae-ab1b8ea399c3\"}}}}",
+        "message": "createCollection",
         "mongodb.log.component": "STORAGE",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 20320,
-        "mongodb.log.msg": "createCollection",
         "service.type": "mongodb"
     },
     {
@@ -95,18 +95,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.410+00:00\"},\"s\":\"I\",  \"c\":\"INDEX\",    \"id\":20345,   \"ctx\":\"initandlisten\",\"msg\":\"Index build: done building\",\"attr\":{\"buildUUID\":null,\"namespace\":\"admin.system.version\",\"index\":\"_id_\",\"commitTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 1167,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.410+00:00\"},\"s\":\"I\",  \"c\":\"INDEX\",    \"id\":20345,   \"ctx\":\"initandlisten\",\"msg\":\"Index build: done building\",\"attr\":{\"buildUUID\":null,\"namespace\":\"admin.system.version\",\"index\":\"_id_\",\"commitTimestamp\":{\"$timestamp\":{\"t\":0,\"i\":0}}}}",
+        "message": "Index build: done building",
         "mongodb.log.component": "INDEX",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 20345,
-        "mongodb.log.msg": "Index build: done building",
         "service.type": "mongodb"
     },
     {
@@ -117,18 +117,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.412+00:00\"},\"s\":\"I\",  \"c\":\"COMMAND\",  \"id\":20459,   \"ctx\":\"initandlisten\",\"msg\":\"Setting featureCompatibilityVersion\",\"attr\":{\"newVersion\":\"4.4\"}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 1435,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.412+00:00\"},\"s\":\"I\",  \"c\":\"COMMAND\",  \"id\":20459,   \"ctx\":\"initandlisten\",\"msg\":\"Setting featureCompatibilityVersion\",\"attr\":{\"newVersion\":\"4.4\"}}",
+        "message": "Setting featureCompatibilityVersion",
         "mongodb.log.component": "COMMAND",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 20459,
-        "mongodb.log.msg": "Setting featureCompatibilityVersion",
         "service.type": "mongodb"
     },
     {
@@ -139,18 +139,18 @@
         "event.dataset": "mongodb.log",
         "event.kind": "event",
         "event.module": "mongodb",
+        "event.original": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.451+00:00\"},\"s\":\"I\",  \"c\":\"FTDC\",     \"id\":20625,   \"ctx\":\"initandlisten\",\"msg\":\"Initializing full-time diagnostic data capture\",\"attr\":{\"dataDirectory\":\"/data/db/diagnostic.data\"}}",
         "event.type": [
             "info"
         ],
         "fileset.name": "log",
         "input.type": "log",
         "log.level": "I",
         "log.offset": 1617,
-        "message": "{\"t\":{\"$date\":\"2021-03-22T21:21:26.451+00:00\"},\"s\":\"I\",  \"c\":\"FTDC\",     \"id\":20625,   \"ctx\":\"initandlisten\",\"msg\":\"Initializing full-time diagnostic data capture\",\"attr\":{\"dataDirectory\":\"/data/db/diagnostic.data\"}}",
+        "message": "Initializing full-time diagnostic data capture",
         "mongodb.log.component": "FTDC",
         "mongodb.log.context": "initandlisten",
         "mongodb.log.id": 20625,
-        "mongodb.log.msg": "Initializing full-time diagnostic data capture",
         "service.type": "mongodb"
     }
-]
+]