Dump and Reset stats data on demand using SIGUSR1 signal

Documentation/performance.rst

@@ -103,6 +103,12 @@ How to read this output:
    counters should be compared against "golden runs" to deduce any interesting
    trends.
 
+It is also possible to dump and reset SGX-related statistics interactively, using
+``SIGUSR1`` signal. This helps to collect SGX-related statistics only for a
+particular period, e.g. skipping the Gramine startup and application
+initialization time and concentrating only on the actual application processing.
+Send ``SIGUSR1`` using command ``kill -SIGUSR1 <gramine-pid>``.
+
 Effects of system calls / ocalls
 --------------------------------
 

pal/src/host/linux-sgx/host_exception.c

@@ -14,16 +14,16 @@
                      * ../../../include/arch/x86_64/linux/ucontext.h:136:5: error: unknown type name ‘__sigset_t’
                      *      __sigset_t uc_sigmask;
                      */
-
-
 #include <linux/signal.h>
 #include <stdbool.h>
 
 #include "api.h"
 #include "cpu.h"
 #include "debug_map.h"
+#include "gdb_integration/sgx_gdb.h"
 #include "host_internal.h"
 #include "pal_rpc_queue.h"
+#include "pal_tcb.h"
 #include "sigreturn.h"
 #include "ucontext.h"
 
@@ -188,6 +188,52 @@ static void handle_dummy_signal(int signum, siginfo_t* info, struct ucontext* uc
     /* we need this handler to interrupt blocking syscalls in RPC threads */
 }
 
+static size_t send_sigusr1_signal_to_children(pid_t main_tid) {
+    size_t no_of_signals_sent = 0;
+
+    for (size_t i = 0; i < MAX_DBG_THREADS; i++) {
+        int child_tid = ((struct enclave_dbginfo*)DBGINFO_ADDR)->thread_tids[i];
+        if (child_tid == main_tid)
+            continue;
+
+        if (child_tid) {
+            DO_SYSCALL(tkill, child_tid, SIGUSR1);
+            no_of_signals_sent++;
+        }
+    }
+    return no_of_signals_sent;
+}
+
+static void dump_and_reset_stats(void) {
+    static size_t no_of_children_visited = 0;
+
+    if (DO_SYSCALL(gettid) == g_host_pid) {
+        size_t no_of_children = send_sigusr1_signal_to_children(g_host_pid);
+
+        log_always("----- DUMPING and RESETTING SGX STATS -----");
+        while ((__atomic_load_n(&no_of_children_visited, __ATOMIC_ACQUIRE)) < no_of_children) {
+            DO_SYSCALL(sched_yield);
+        }
+
+        update_and_print_stats(/*process_wide=*/true);
+        __atomic_store_n(&no_of_children_visited, 0, __ATOMIC_RELEASE);
+    } else {
+        update_and_print_stats(/*process_wide=*/false);
+        __atomic_fetch_add(&no_of_children_visited, 1, __ATOMIC_ACQ_REL);
+    }
+
+    pal_host_tcb_reset_stats();
+}
+
+static void handle_sigusr1(int signum, siginfo_t* info, struct ucontext* uc) {
+    __UNUSED(signum);
+    __UNUSED(info);
+    __UNUSED(uc);
+
+    if (g_sgx_enable_stats)
+        dump_and_reset_stats();
+}
+
 int sgx_signal_setup(void) {
     int ret;
 
@@ -226,6 +272,10 @@ int sgx_signal_setup(void) {
     if (ret < 0)
         goto err;
 
+    ret = set_signal_handler(SIGUSR1, handle_sigusr1);
+    if (ret < 0)
+        goto err;
+
     /* SIGUSR2 is reserved for Gramine usage: interrupting blocking syscalls in RPC threads.
      * We block SIGUSR2 in enclave threads; it is unblocked by each RPC thread explicitly. */
     ret = set_signal_handler(SIGUSR2, handle_dummy_signal);

pal/src/host/linux-sgx/host_thread.c

@@ -50,11 +50,11 @@ void update_and_print_stats(bool process_wide) {
                tid, tcb->eenter_cnt, tcb->eexit_cnt, tcb->aex_cnt,
                tcb->sync_signal_cnt, tcb->async_signal_cnt);
 
-    g_eenter_cnt       += tcb->eenter_cnt;
-    g_eexit_cnt        += tcb->eexit_cnt;
-    g_aex_cnt          += tcb->aex_cnt;
-    g_sync_signal_cnt  += tcb->sync_signal_cnt;
-    g_async_signal_cnt += tcb->async_signal_cnt;
+    __atomic_fetch_add(&g_eenter_cnt, tcb->eenter_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_eexit_cnt, tcb->eexit_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_aex_cnt, tcb->aex_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_sync_signal_cnt, tcb->sync_signal_cnt, __ATOMIC_ACQ_REL);
+    __atomic_fetch_add(&g_async_signal_cnt, tcb->async_signal_cnt, __ATOMIC_ACQ_REL);
 
     if (process_wide) {
         int pid = g_host_pid;
@@ -65,8 +65,17 @@ void update_and_print_stats(bool process_wide) {
                    "  # of AEXs:           %lu\n"
                    "  # of sync signals:   %lu\n"
                    "  # of async signals:  %lu",
-                   pid, g_eenter_cnt, g_eexit_cnt, g_aex_cnt,
-                   g_sync_signal_cnt, g_async_signal_cnt);
+                   pid, __atomic_load_n(&g_eenter_cnt, __ATOMIC_ACQUIRE),
+                   __atomic_load_n(&g_eexit_cnt, __ATOMIC_ACQUIRE),
+                   __atomic_load_n(&g_aex_cnt, __ATOMIC_ACQUIRE),
+                   __atomic_load_n(&g_sync_signal_cnt, __ATOMIC_ACQUIRE),
+                   __atomic_load_n(&g_async_signal_cnt, __ATOMIC_ACQUIRE));
+
+        __atomic_store_n(&g_eenter_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_eexit_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_aex_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_sync_signal_cnt, 0, __ATOMIC_RELEASE);
+        __atomic_store_n(&g_async_signal_cnt, 0, __ATOMIC_RELEASE);
     }
 }
 
@@ -87,6 +96,15 @@ void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack) {
     tcb->last_async_event = PAL_EVENT_NO_EVENT;
 }
 
+void pal_host_tcb_reset_stats(void) {
+    PAL_HOST_TCB* tcb = pal_get_host_tcb();
+    tcb->eenter_cnt       = 0;
+    tcb->eexit_cnt        = 0;
+    tcb->aex_cnt          = 0;
+    tcb->sync_signal_cnt  = 0;
+    tcb->async_signal_cnt = 0;
+}
+
 int create_tcs_mapper(void* tcs_base, unsigned int thread_num) {
     sgx_arch_tcs_t* enclave_tcs = tcs_base;
 

pal/src/host/linux-sgx/pal_tcb.h

@@ -108,6 +108,7 @@ typedef struct pal_host_tcb {
 } PAL_HOST_TCB;
 
 extern void pal_host_tcb_init(PAL_HOST_TCB* tcb, void* stack, void* alt_stack);
+extern void pal_host_tcb_reset_stats(void);
 
 static inline PAL_HOST_TCB* pal_get_host_tcb(void) {
     PAL_HOST_TCB* tcb;