filter_lua: new 'time_as_table' property for timestamp handling (flue…

…nt#641 fluent#2519 fluent#2015)

Long time ago we got some reports that using timestamps as double/floats might lost
precision when the values are converted back from Lua. Actually there is no exact
way to have 100% precision in doubles in our use case 'C > Lua > C'. Community
suggested that we might workaround this with another solution.

This patch considering backward compatibility, implements a new configuration
property called 'time_as_table', which passes the timestamp as a Lua table
with the following keys

  ['sec']  => timestamp seconds
  ['nsec'] => timestamp nanoseconds

for users looking for 100% timestamp precision and specifically when dealing
with nanoseconds, this option ensures timestamp integrity. If the option is enabled
the user just need to adjust the script to use the new format if they touch or
use the timestamp value.

By default the option 'time_as_table' is disabled and in the future we might
consider to enable it by default.

Signed-off-by: Eduardo Silva <eduardo@treasure-data.com>
Signed-off-by: xmcqueen <bmcqueen@linkedin.com>

plugins/filter_lua/lua.c

@@ -31,6 +31,22 @@
 
 #include "lua_config.h"
 
+/* Push timestamp as a Lua table into the stack */
+static void lua_pushtimetable(lua_State *l, struct flb_time *tm)
+{
+    lua_createtable(l, 0, 2);
+
+    /* seconds */
+    lua_pushlstring(l, "sec", 3);
+    lua_pushinteger(l, tm->tm.tv_sec);
+    lua_settable(l, -3);
+
+    /* nanoseconds */
+    lua_pushlstring(l, "nsec", 4);
+    lua_pushinteger(l, tm->tm.tv_nsec);
+    lua_settable(l, -3);
+}
+
 static void lua_pushmsgpack(lua_State *l, msgpack_object *o)
 {
     int i;
@@ -439,8 +455,13 @@ static int cb_lua_filter(const void *data, size_t bytes,
         lua_pushstring(ctx->lua->state, tag);
 
         /* Timestamp */
-        ts = flb_time_to_double(&t);
-        lua_pushnumber(ctx->lua->state, ts);
+        if (ctx->time_as_table == FLB_TRUE) {
+            lua_pushtimetable(ctx->lua->state, &t);
+        }
+        else {
+            ts = flb_time_to_double(&t);
+            lua_pushnumber(ctx->lua->state, ts);
+        }
 
         lua_pushmsgpack(ctx->lua->state, p);
         if (ctx->protected_mode) {
@@ -466,8 +487,28 @@ static int cb_lua_filter(const void *data, size_t bytes,
         lua_tomsgpack(ctx, &data_pck, 0);
         lua_pop(ctx->lua->state, 1);
 
-        l_timestamp = (double) lua_tonumber(ctx->lua->state, -1);
-        lua_pop(ctx->lua->state, 1);
+        /* Lua table */
+        if (ctx->time_as_table == FLB_TRUE) {
+            if (lua_type(ctx->lua->state, -1) == LUA_TTABLE) {
+                /* Retrieve seconds */
+                lua_getfield(ctx->lua->state, -1, "sec");
+                t.tm.tv_sec = lua_tointeger(ctx->lua->state, -1);
+                lua_pop(ctx->lua->state, 1);
+
+                /* Retrieve nanoseconds */
+                lua_getfield(ctx->lua->state, -1, "nsec");
+                t.tm.tv_nsec = lua_tointeger(ctx->lua->state, -1);
+                lua_pop(ctx->lua->state, 2);
+            }
+            else {
+                flb_plg_error(ctx->ins, "invalid lua timestamp type returned");
+                t = t_orig;
+            }
+        }
+        else {
+            l_timestamp = (double) lua_tonumber(ctx->lua->state, -1);
+            lua_pop(ctx->lua->state, 1);
+        }
 
         l_code = (int) lua_tointeger(ctx->lua->state, -1);
         lua_pop(ctx->lua->state, 1);
@@ -481,7 +522,9 @@ static int cb_lua_filter(const void *data, size_t bytes,
         }
         else if (l_code == 1 || l_code == 2) { /* Modified, pack new data */
             if (l_code == 1) {
-                flb_time_from_double(&t, l_timestamp);
+                if (ctx->time_as_table == FLB_FALSE) {
+                    flb_time_from_double(&t, l_timestamp);
+                }
             }
             else if (l_code == 2) {
                 /* Keep the timestamp */

plugins/filter_lua/lua_config.c

@@ -146,6 +146,12 @@ struct lua_filter *lua_config_create(struct flb_filter_instance *ins,
         lf->protected_mode = flb_utils_bool(tmp);
     }
 
+    lf->time_as_table = FLB_FALSE;
+    tmp = flb_filter_get_property("time_as_table", ins);
+    if (tmp) {
+        lf->time_as_table = flb_utils_bool(tmp);
+    }
+
     return lf;
 }
 

plugins/filter_lua/lua_config.h

@@ -40,6 +40,7 @@ struct lua_filter {
     flb_sds_t buffer;                 /* json dec buffer */
     int    l2c_types_num;             /* number of l2c_types */
     int    protected_mode;            /* exec lua function in protected mode */
+    int    time_as_table;             /* timestamp as a Lua table */
     struct mk_list l2c_types;         /* data types (lua -> C) */
     struct flb_luajit *lua;           /* state context   */
     struct flb_filter_instance *ins;  /* filter instance */