network: ping pong keepalive for tcp connections

To make the tcp connections keepalive and better handle the timeout
of connections

con_timeout indicate the connection timeout and it is configurable

Signed-off-by: Chen Minqiang <ptpt52@gmail.com>

CONFIGURE.md

@@ -263,7 +263,7 @@ grep 'CONFIG-T:' `find . -type f -name "*.[ch]"`|sed 's/^.*CONFIG-.: *\(.*\)$/|\
 -->
 |Parameter|Purpose|Notes [Default is bracketed]|
 |---------|-------|-----|
-|client_timeout|Timespan until a client is seen as disconnected|[60]|
+|con_timeout|Timespan until a connection is seen as disconnected|[60]|
 |remove_ap|Timer to remove expired AP entries from core data set|[460]|
 |remove_client|Timer to remove expired client entries from core data set|[15]|
 |remove_probe|Timer to remove expired PROBE and BEACON entries from core data set|[30]|

dawn-config

@@ -17,7 +17,7 @@ config hostapd
 	option hostapd_dir '/var/run/hostapd'
 
 config times
-	option client_timeout '60'
+	option con_timeout '60'
 	option update_client '10'
 	option remove_client '15'
 	option remove_probe '30'

src/include/datastorage.h

@@ -109,7 +109,7 @@ struct time_config_s {
     time_t update_tcp_con; // Refresh network connections
     time_t update_chan_util; // Refresh per radio / SSID channel util info
     time_t update_beacon_reports; // Request BEACON from capable clients
-    time_t client_timeout; // Check for client timeouts.
+    time_t con_timeout; // Check for connection timeouts.
 };
 
 struct local_config_s {

src/include/tcpsocket.h

@@ -5,7 +5,7 @@
 #include <netinet/in.h>
 
 #define ARRAY_NETWORK_LEN 50
-#define CHECK_CLIENT_TIMEOUT 5
+#define CHECK_TIMEOUT 10
 
 struct network_con_s {
     struct list_head list;
@@ -14,6 +14,7 @@ struct network_con_s {
     struct ustream_fd stream;
     struct sockaddr_in sock_addr;
     int connected;
+    time_t time_alive;
 };
 
 /**
@@ -38,9 +39,14 @@ int run_server(int port);
 void send_tcp(char *msg);
 
 /**
- * Check sockets for client timeouts.
+ * Send ping to clients
  */
-void check_client_timeout(int timeout);
+void server_to_clients_ping(void);
+
+/**
+ * Check sockets timeouts.
+ */
+void check_timeout(int timeout);
 
 /**
  * Debug message.

src/network/tcpsocket.c

@@ -12,6 +12,10 @@
 #define STR_QUOTE(x) STR_EVAL(x)
 
 #define HEADER_SIZE sizeof(uint32_t)
+#define PING_STR "ping"
+#define PONG_STR "pong"
+#define PING_SIZE (strlen(PING_STR)+1)
+#define PONG_SIZE (strlen(PONG_STR)+1)
 
 LIST_HEAD(tcp_sock_list);
 LIST_HEAD(cli_list);
@@ -183,6 +187,12 @@ static void client_read_cb(struct ustream *s, int bytes) {
         if (cl->state == READ_STATUS_COMPLETE)
         {
             dawnlog_debug("tcp_socket: processing message...\n");
+
+            /* received pong */
+            if (cl->final_len == HEADER_SIZE + PONG_SIZE && memcmp(cl->str + HEADER_SIZE, PONG_STR, PONG_SIZE) == 0) {
+                goto process_done;
+            }
+
             if (network_config.use_symm_enc) {
                 char *dec = gcrypt_decrypt_msg(cl->str + HEADER_SIZE, cl->final_len - HEADER_SIZE);//len of str is final_len
                 if (!dec) {
@@ -198,6 +208,7 @@ static void client_read_cb(struct ustream *s, int bytes) {
                 handle_network_msg(cl->str + HEADER_SIZE);//len of str is final_len
             }
 
+process_done:
             cl->state = READ_STATUS_READY;
             cl->curr_len = 0;
             cl->final_len = 0;
@@ -258,15 +269,48 @@ int run_server(int port) {
     return 0;
 }
 
-static void client_not_be_used_read_cb(struct ustream *s, int bytes) {
+static void client_ping_read_cb(struct ustream *s, int bytes) {
     int len;
     char buf[2048];
+    uint32_t ping_len = HEADER_SIZE + PING_SIZE;
 
     dawnlog_debug_func("Entering...");
 
-    len = ustream_read(s, buf, sizeof(buf));
+    len = ustream_read(s, buf, ping_len);
-    buf[len] = '\0';
+
-    dawnlog_debug("Read %d bytes from SSL connection: %s\n", len, buf);
+    /* client received ping, send pong back to server */
+    if (len == ping_len && ntohl(*(uint32_t *)buf) == ping_len && memcmp(buf + HEADER_SIZE, PING_STR, PING_SIZE) == 0) {
+        struct network_con_s *con = container_of(s, struct network_con_s, stream.stream);
+        int len_ustream;
+        const char *msg = PONG_STR;
+        size_t msglen = PONG_SIZE;
+        uint32_t final_len = msglen + HEADER_SIZE;
+        char final_str[HEADER_SIZE + PONG_SIZE];
+        uint32_t *msg_header = (uint32_t *)final_str;
+
+        con->time_alive = time(0);
+
+        *msg_header = htonl(final_len);
+        memcpy(final_str + HEADER_SIZE, msg, msglen);
+        len_ustream = ustream_write(&con->stream.stream, final_str, final_len, 0);
+        if (len_ustream <= 0) {
+            dawnlog_error("Ustream error(" STR_QUOTE(__LINE__) ")!\n");
+            //ERROR HANDLING!
+            if (con->stream.stream.write_error) {
+                ustream_free(&con->stream.stream);
+                dawn_unregmem(&con->stream.stream);
+                close(con->fd.fd);
+                list_del(&con->list);
+                dawn_free(con);
+                con = NULL;
+            }
+        }
+
+    } else {
+        buf[len] = 0;
+        dawnlog_error("Read %d bytes upexpected: %s\n", len, buf);
+    }
+
 }
 
 static void connect_cb(struct uloop_fd *f, unsigned int events) {
@@ -287,7 +331,7 @@ static void connect_cb(struct uloop_fd *f, unsigned int events) {
     dawnlog_debug("Connection established\n");
     uloop_fd_delete(&entry->fd);
 
-    entry->stream.stream.notify_read = client_not_be_used_read_cb;
+    entry->stream.stream.notify_read = client_ping_read_cb;
     entry->stream.stream.notify_state = client_to_server_state;
 
     ustream_fd_init(&entry->stream, entry->fd.fd);
@@ -336,6 +380,7 @@ int add_tcp_connection(char *ipv4, int port) {
     uloop_fd_add(&tcp_entry->fd, ULOOP_WRITE | ULOOP_EDGE_TRIGGER);
 
     dawnlog_debug("New TCP connection to %s:%d\n", ipv4, port);
+    tcp_entry->time_alive = time(0);
     list_add(&tcp_entry->list, &tcp_sock_list);
 
     return 0;
@@ -429,18 +474,59 @@ void send_tcp(char *msg) {
     }
 }
 
-void check_client_timeout(int timeout) {
+void server_to_clients_ping(void)
+{
     struct client *cl, *tmp;
-    time_t now = time(0);
+    const char *msg = PING_STR;
+    size_t msglen = PING_SIZE;
+    uint32_t final_len = msglen + HEADER_SIZE;
+    char final_str[HEADER_SIZE + PING_SIZE];
+    uint32_t *msg_header = (uint32_t *)final_str;
+    *msg_header = htonl(final_len);
+    memcpy(final_str + HEADER_SIZE, msg, msglen);
+
     list_for_each_entry_safe(cl, tmp, &cli_list, list)
     {
-        if (now - cl->time_alive > timeout || now - cl->time_alive < -timeout) {
+        int len_ustream = ustream_write(&cl->s.stream, final_str, final_len, 0);
-            dawnlog_debug("Ustream client_close: timeout=%d\n", (int)(now - cl->time_alive));
+        if (len_ustream <= 0) {
-            client_close(&cl->s.stream);
+            dawnlog_error("Ustream error(" STR_QUOTE(__LINE__) ")!\n");
+            if (cl->s.stream.write_error) {
+                client_close(&cl->s.stream);
+            }
         }
     }
 }
 
+void check_timeout(int timeout) {
+    do {
+        struct client *cl, *tmp;
+        time_t now = time(0);
+        list_for_each_entry_safe(cl, tmp, &cli_list, list)
+        {
+            if (now - cl->time_alive > timeout || now - cl->time_alive < -timeout) {
+                dawnlog_info("server: close client connection! timeout=%d\n", (int)(now - cl->time_alive));
+                client_close(&cl->s.stream);
+            }
+        }
+    } while (0);
+
+    do {
+        struct network_con_s *con, *tmp;
+        time_t now = time(0);
+        list_for_each_entry_safe(con, tmp, &cli_list, list)
+        {
+            if (now - con->time_alive > timeout || now - con->time_alive < -timeout) {
+                dawnlog_info("client: close client_to_server connection! timeout=%d\n", (int)(now - con->time_alive));
+                ustream_free(&con->stream.stream);
+                dawn_unregmem(&con->stream.stream);
+                close(con->fd.fd);
+                list_del(&con->list);
+                dawn_free(con);
+            }
+        }
+    } while (0);
+}
+
 struct network_con_s* tcp_list_contains_address(struct sockaddr_in entry) {
     struct network_con_s *con;
 

src/utils/dawn_uci.c

@@ -79,7 +79,7 @@ struct time_config_s uci_get_time_config() {
         .update_tcp_con = 10,
         .update_chan_util = 5,
         .update_beacon_reports = 20,
-        .client_timeout = 60,
+        .con_timeout = 60,
     };
 
     dawnlog_debug_func("Entering...");
@@ -106,8 +106,8 @@ struct time_config_s uci_get_time_config() {
             DAWN_SET_CONFIG_TIME(ret, s, update_chan_util);
             //CONFIG-T: update_beacon_reports|Timer to ask all connected clients for a new BEACON REPORT|[20]
             DAWN_SET_CONFIG_TIME(ret, s, update_beacon_reports);
-            //CONFIG-T: client_timeout|Timespan to check if a client timed out|[60]
+            //CONFIG-T: con_timeout|Timespan to check if a connection timed out|[60]
-            DAWN_SET_CONFIG_TIME(ret, s, client_timeout);
+            DAWN_SET_CONFIG_TIME(ret, s, con_timeout);
             return ret;
         }
     }

src/utils/msghandler.c

@@ -656,7 +656,7 @@ enum {
     UCI_UPDATE_TCP_CON,
     UCI_UPDATE_CHAN_UTIL,
     UCI_UPDATE_BEACON_REPORTS,
-    UCI_CLIENT_TIMEOUT,
+    UCI_CON_TIMEOUT,
     __UCI_TIMES_MAX,
 };
 
@@ -714,7 +714,7 @@ static const struct blobmsg_policy uci_times_policy[__UCI_TIMES_MAX] = {
         [UCI_UPDATE_TCP_CON] = {.name = "update_tcp_con", .type = BLOBMSG_TYPE_INT32},
         [UCI_UPDATE_CHAN_UTIL] = {.name = "update_chan_util", .type = BLOBMSG_TYPE_INT32},
         [UCI_UPDATE_BEACON_REPORTS] = {.name = "update_beacon_reports", .type = BLOBMSG_TYPE_INT32},
-        [UCI_CLIENT_TIMEOUT] = {.name = "client_timeout", .type = BLOBMSG_TYPE_INT32},
+        [UCI_CON_TIMEOUT] = {.name = "con_timeout", .type = BLOBMSG_TYPE_INT32},
 };
 
 static void set_uci_item(char* m, struct blob_attr* a)
@@ -897,7 +897,7 @@ static int handle_uci_config(struct blob_attr* msg) {
 
     set_uci_item("dawn.@times[0].update_beacon_reports=%d", tb_times[UCI_UPDATE_BEACON_REPORTS]);
 
-    set_uci_item("dawn.@times[0].client_timeout=%d", tb_times[UCI_CLIENT_TIMEOUT]);
+    set_uci_item("dawn.@times[0].con_timeout=%d", tb_times[UCI_CON_TIMEOUT]);
 
     uci_reset();
     dawn_metric = uci_get_dawn_metric();

src/utils/ubus.c

@@ -23,7 +23,9 @@ void update_clients(struct uloop_timeout *t);
 
 void update_tcp_connections(struct uloop_timeout *t);
 
-void check_client_timeouts(struct uloop_timeout *t);
+void server_ping_clients(struct uloop_timeout *t);
+
+void check_timeouts(struct uloop_timeout *t);
 
 void update_channel_utilization(struct uloop_timeout *t);
 
@@ -40,8 +42,11 @@ struct uloop_timeout hostapd_timer = {
 struct uloop_timeout tcp_con_timer = {
         .cb = update_tcp_connections
 };
-struct uloop_timeout client_timeout_timer = {
+struct uloop_timeout server_ping_timer = {
-        .cb = check_client_timeouts
+        .cb = server_ping_clients
+};
+struct uloop_timeout check_timeout_timer = {
+        .cb = check_timeouts
 };
 struct uloop_timeout channel_utilization_timer = {
         .cb = update_channel_utilization
@@ -1153,20 +1158,29 @@ void update_tcp_connections(struct uloop_timeout *t) {
     uloop_timeout_set(&tcp_con_timer, timeout_config.update_tcp_con * 1000);
 }
 
-void check_client_timeouts(struct uloop_timeout *t) {
+void server_ping_clients(struct uloop_timeout *t) {
     dawnlog_debug_func("Entering...");
 
-    check_client_timeout(timeout_config.client_timeout);
+    server_to_clients_ping();
 
-    uloop_timeout_set(&client_timeout_timer, CHECK_CLIENT_TIMEOUT * 1000);
+    uloop_timeout_set(&server_ping_timer, 20 * 1000);
+}
+
+void check_timeouts(struct uloop_timeout *t) {
+    dawnlog_debug_func("Entering...");
+
+    check_timeout(timeout_config.con_timeout);
+
+    uloop_timeout_set(&check_timeout_timer, CHECK_TIMEOUT * 1000);
 }
 
 void start_tcp_con_update() {
     dawnlog_debug_func("Entering...");
 
+    uloop_timeout_add(&server_ping_timer);
     // update connections
     uloop_timeout_add(&tcp_con_timer); // callback = update_tcp_connections
-    uloop_timeout_add(&client_timeout_timer); // callback = client_timeout
+    uloop_timeout_add(&check_timeout_timer); // callback = con_timeout
 }
 
 void update_hostapd_sockets(struct uloop_timeout *t) {
@@ -1902,7 +1916,7 @@ int uci_send_via_network()
     blobmsg_add_u32(&b, "update_tcp_con", timeout_config.update_tcp_con);
     blobmsg_add_u32(&b, "update_chan_util", timeout_config.update_chan_util);
     blobmsg_add_u32(&b, "update_beacon_reports", timeout_config.update_beacon_reports);
-    blobmsg_add_u32(&b, "client_timeout", timeout_config.client_timeout);
+    blobmsg_add_u32(&b, "con_timeout", timeout_config.con_timeout);
     blobmsg_close_table(&b, times);
 
     send_blob_attr_via_network(b.head, "uci");