From 47e98efed6ccb878227bc1c94e4e74b4ca97d8d5 Mon Sep 17 00:00:00 2001 From: Chen Minqiang Date: Thu, 21 Jul 2022 06:03:58 +0800 Subject: [PATCH] network: ping pong keepalive for tcp connections To make the tcp connections keepalive and better handle the timeout of connections con_timeout indicate the connection timeout and it is configurable Signed-off-by: Chen Minqiang --- CONFIGURE.md | 2 +- dawn-config | 2 +- src/include/datastorage.h | 2 +- src/include/tcpsocket.h | 12 +++-- src/network/tcpsocket.c | 106 ++++++++++++++++++++++++++++++++++---- src/utils/dawn_uci.c | 6 +-- src/utils/msghandler.c | 6 +-- src/utils/ubus.c | 30 ++++++++--- 8 files changed, 136 insertions(+), 30 deletions(-) diff --git a/CONFIGURE.md b/CONFIGURE.md index a653fbe..fb40487 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -263,7 +263,7 @@ grep 'CONFIG-T:' `find . -type f -name "*.[ch]"`|sed 's/^.*CONFIG-.: *\(.*\)$/|\ --> |Parameter|Purpose|Notes [Default is bracketed]| |---------|-------|-----| -|client_timeout|Timespan until a client is seen as disconnected|[60]| +|con_timeout|Timespan until a connection is seen as disconnected|[60]| |remove_ap|Timer to remove expired AP entries from core data set|[460]| |remove_client|Timer to remove expired client entries from core data set|[15]| |remove_probe|Timer to remove expired PROBE and BEACON entries from core data set|[30]| diff --git a/dawn-config b/dawn-config index 5e226a3..011d1f5 100644 --- a/dawn-config +++ b/dawn-config @@ -17,7 +17,7 @@ config hostapd option hostapd_dir '/var/run/hostapd' config times - option client_timeout '60' + option con_timeout '60' option update_client '10' option remove_client '15' option remove_probe '30' diff --git a/src/include/datastorage.h b/src/include/datastorage.h index eefc95c..d817e59 100644 --- a/src/include/datastorage.h +++ b/src/include/datastorage.h @@ -109,7 +109,7 @@ struct time_config_s { time_t update_tcp_con; // Refresh network connections time_t update_chan_util; // Refresh per radio / SSID channel util info time_t update_beacon_reports; // Request BEACON from capable clients - time_t client_timeout; // Check for client timeouts. + time_t con_timeout; // Check for connection timeouts. }; struct local_config_s { diff --git a/src/include/tcpsocket.h b/src/include/tcpsocket.h index 1fb335d..6d9cd9f 100644 --- a/src/include/tcpsocket.h +++ b/src/include/tcpsocket.h @@ -5,7 +5,7 @@ #include #define ARRAY_NETWORK_LEN 50 -#define CHECK_CLIENT_TIMEOUT 5 +#define CHECK_TIMEOUT 10 struct network_con_s { struct list_head list; @@ -14,6 +14,7 @@ struct network_con_s { struct ustream_fd stream; struct sockaddr_in sock_addr; int connected; + time_t time_alive; }; /** @@ -38,9 +39,14 @@ int run_server(int port); void send_tcp(char *msg); /** - * Check sockets for client timeouts. + * Send ping to clients */ -void check_client_timeout(int timeout); +void server_to_clients_ping(void); + +/** + * Check sockets timeouts. + */ +void check_timeout(int timeout); /** * Debug message. diff --git a/src/network/tcpsocket.c b/src/network/tcpsocket.c index b539286..7c64de7 100644 --- a/src/network/tcpsocket.c +++ b/src/network/tcpsocket.c @@ -12,6 +12,10 @@ #define STR_QUOTE(x) STR_EVAL(x) #define HEADER_SIZE sizeof(uint32_t) +#define PING_STR "ping" +#define PONG_STR "pong" +#define PING_SIZE (strlen(PING_STR)+1) +#define PONG_SIZE (strlen(PONG_STR)+1) LIST_HEAD(tcp_sock_list); LIST_HEAD(cli_list); @@ -183,6 +187,12 @@ static void client_read_cb(struct ustream *s, int bytes) { if (cl->state == READ_STATUS_COMPLETE) { dawnlog_debug("tcp_socket: processing message...\n"); + + /* received pong */ + if (cl->final_len == HEADER_SIZE + PONG_SIZE && memcmp(cl->str + HEADER_SIZE, PONG_STR, PONG_SIZE) == 0) { + goto process_done; + } + if (network_config.use_symm_enc) { char *dec = gcrypt_decrypt_msg(cl->str + HEADER_SIZE, cl->final_len - HEADER_SIZE);//len of str is final_len if (!dec) { @@ -198,6 +208,7 @@ static void client_read_cb(struct ustream *s, int bytes) { handle_network_msg(cl->str + HEADER_SIZE);//len of str is final_len } +process_done: cl->state = READ_STATUS_READY; cl->curr_len = 0; cl->final_len = 0; @@ -258,15 +269,48 @@ int run_server(int port) { return 0; } -static void client_not_be_used_read_cb(struct ustream *s, int bytes) { +static void client_ping_read_cb(struct ustream *s, int bytes) { int len; char buf[2048]; + uint32_t ping_len = HEADER_SIZE + PING_SIZE; dawnlog_debug_func("Entering..."); - len = ustream_read(s, buf, sizeof(buf)); - buf[len] = '\0'; - dawnlog_debug("Read %d bytes from SSL connection: %s\n", len, buf); + len = ustream_read(s, buf, ping_len); + + /* client received ping, send pong back to server */ + if (len == ping_len && ntohl(*(uint32_t *)buf) == ping_len && memcmp(buf + HEADER_SIZE, PING_STR, PING_SIZE) == 0) { + struct network_con_s *con = container_of(s, struct network_con_s, stream.stream); + int len_ustream; + const char *msg = PONG_STR; + size_t msglen = PONG_SIZE; + uint32_t final_len = msglen + HEADER_SIZE; + char final_str[HEADER_SIZE + PONG_SIZE]; + uint32_t *msg_header = (uint32_t *)final_str; + + con->time_alive = time(0); + + *msg_header = htonl(final_len); + memcpy(final_str + HEADER_SIZE, msg, msglen); + len_ustream = ustream_write(&con->stream.stream, final_str, final_len, 0); + if (len_ustream <= 0) { + dawnlog_error("Ustream error(" STR_QUOTE(__LINE__) ")!\n"); + //ERROR HANDLING! + if (con->stream.stream.write_error) { + ustream_free(&con->stream.stream); + dawn_unregmem(&con->stream.stream); + close(con->fd.fd); + list_del(&con->list); + dawn_free(con); + con = NULL; + } + } + + } else { + buf[len] = 0; + dawnlog_error("Read %d bytes upexpected: %s\n", len, buf); + } + } static void connect_cb(struct uloop_fd *f, unsigned int events) { @@ -287,7 +331,7 @@ static void connect_cb(struct uloop_fd *f, unsigned int events) { dawnlog_debug("Connection established\n"); uloop_fd_delete(&entry->fd); - entry->stream.stream.notify_read = client_not_be_used_read_cb; + entry->stream.stream.notify_read = client_ping_read_cb; entry->stream.stream.notify_state = client_to_server_state; ustream_fd_init(&entry->stream, entry->fd.fd); @@ -336,6 +380,7 @@ int add_tcp_connection(char *ipv4, int port) { uloop_fd_add(&tcp_entry->fd, ULOOP_WRITE | ULOOP_EDGE_TRIGGER); dawnlog_debug("New TCP connection to %s:%d\n", ipv4, port); + tcp_entry->time_alive = time(0); list_add(&tcp_entry->list, &tcp_sock_list); return 0; @@ -429,18 +474,59 @@ void send_tcp(char *msg) { } } -void check_client_timeout(int timeout) { +void server_to_clients_ping(void) +{ struct client *cl, *tmp; - time_t now = time(0); + const char *msg = PING_STR; + size_t msglen = PING_SIZE; + uint32_t final_len = msglen + HEADER_SIZE; + char final_str[HEADER_SIZE + PING_SIZE]; + uint32_t *msg_header = (uint32_t *)final_str; + *msg_header = htonl(final_len); + memcpy(final_str + HEADER_SIZE, msg, msglen); + list_for_each_entry_safe(cl, tmp, &cli_list, list) { - if (now - cl->time_alive > timeout || now - cl->time_alive < -timeout) { - dawnlog_debug("Ustream client_close: timeout=%d\n", (int)(now - cl->time_alive)); - client_close(&cl->s.stream); + int len_ustream = ustream_write(&cl->s.stream, final_str, final_len, 0); + if (len_ustream <= 0) { + dawnlog_error("Ustream error(" STR_QUOTE(__LINE__) ")!\n"); + if (cl->s.stream.write_error) { + client_close(&cl->s.stream); + } } } } +void check_timeout(int timeout) { + do { + struct client *cl, *tmp; + time_t now = time(0); + list_for_each_entry_safe(cl, tmp, &cli_list, list) + { + if (now - cl->time_alive > timeout || now - cl->time_alive < -timeout) { + dawnlog_info("server: close client connection! timeout=%d\n", (int)(now - cl->time_alive)); + client_close(&cl->s.stream); + } + } + } while (0); + + do { + struct network_con_s *con, *tmp; + time_t now = time(0); + list_for_each_entry_safe(con, tmp, &cli_list, list) + { + if (now - con->time_alive > timeout || now - con->time_alive < -timeout) { + dawnlog_info("client: close client_to_server connection! timeout=%d\n", (int)(now - con->time_alive)); + ustream_free(&con->stream.stream); + dawn_unregmem(&con->stream.stream); + close(con->fd.fd); + list_del(&con->list); + dawn_free(con); + } + } + } while (0); +} + struct network_con_s* tcp_list_contains_address(struct sockaddr_in entry) { struct network_con_s *con; diff --git a/src/utils/dawn_uci.c b/src/utils/dawn_uci.c index 82873b3..d9750d6 100644 --- a/src/utils/dawn_uci.c +++ b/src/utils/dawn_uci.c @@ -79,7 +79,7 @@ struct time_config_s uci_get_time_config() { .update_tcp_con = 10, .update_chan_util = 5, .update_beacon_reports = 20, - .client_timeout = 60, + .con_timeout = 60, }; dawnlog_debug_func("Entering..."); @@ -106,8 +106,8 @@ struct time_config_s uci_get_time_config() { DAWN_SET_CONFIG_TIME(ret, s, update_chan_util); //CONFIG-T: update_beacon_reports|Timer to ask all connected clients for a new BEACON REPORT|[20] DAWN_SET_CONFIG_TIME(ret, s, update_beacon_reports); - //CONFIG-T: client_timeout|Timespan to check if a client timed out|[60] - DAWN_SET_CONFIG_TIME(ret, s, client_timeout); + //CONFIG-T: con_timeout|Timespan to check if a connection timed out|[60] + DAWN_SET_CONFIG_TIME(ret, s, con_timeout); return ret; } } diff --git a/src/utils/msghandler.c b/src/utils/msghandler.c index 3827662..e074800 100644 --- a/src/utils/msghandler.c +++ b/src/utils/msghandler.c @@ -656,7 +656,7 @@ enum { UCI_UPDATE_TCP_CON, UCI_UPDATE_CHAN_UTIL, UCI_UPDATE_BEACON_REPORTS, - UCI_CLIENT_TIMEOUT, + UCI_CON_TIMEOUT, __UCI_TIMES_MAX, }; @@ -714,7 +714,7 @@ static const struct blobmsg_policy uci_times_policy[__UCI_TIMES_MAX] = { [UCI_UPDATE_TCP_CON] = {.name = "update_tcp_con", .type = BLOBMSG_TYPE_INT32}, [UCI_UPDATE_CHAN_UTIL] = {.name = "update_chan_util", .type = BLOBMSG_TYPE_INT32}, [UCI_UPDATE_BEACON_REPORTS] = {.name = "update_beacon_reports", .type = BLOBMSG_TYPE_INT32}, - [UCI_CLIENT_TIMEOUT] = {.name = "client_timeout", .type = BLOBMSG_TYPE_INT32}, + [UCI_CON_TIMEOUT] = {.name = "con_timeout", .type = BLOBMSG_TYPE_INT32}, }; static void set_uci_item(char* m, struct blob_attr* a) @@ -897,7 +897,7 @@ static int handle_uci_config(struct blob_attr* msg) { set_uci_item("dawn.@times[0].update_beacon_reports=%d", tb_times[UCI_UPDATE_BEACON_REPORTS]); - set_uci_item("dawn.@times[0].client_timeout=%d", tb_times[UCI_CLIENT_TIMEOUT]); + set_uci_item("dawn.@times[0].con_timeout=%d", tb_times[UCI_CON_TIMEOUT]); uci_reset(); dawn_metric = uci_get_dawn_metric(); diff --git a/src/utils/ubus.c b/src/utils/ubus.c index 4280410..7bf06af 100644 --- a/src/utils/ubus.c +++ b/src/utils/ubus.c @@ -23,7 +23,9 @@ void update_clients(struct uloop_timeout *t); void update_tcp_connections(struct uloop_timeout *t); -void check_client_timeouts(struct uloop_timeout *t); +void server_ping_clients(struct uloop_timeout *t); + +void check_timeouts(struct uloop_timeout *t); void update_channel_utilization(struct uloop_timeout *t); @@ -40,8 +42,11 @@ struct uloop_timeout hostapd_timer = { struct uloop_timeout tcp_con_timer = { .cb = update_tcp_connections }; -struct uloop_timeout client_timeout_timer = { - .cb = check_client_timeouts +struct uloop_timeout server_ping_timer = { + .cb = server_ping_clients +}; +struct uloop_timeout check_timeout_timer = { + .cb = check_timeouts }; struct uloop_timeout channel_utilization_timer = { .cb = update_channel_utilization @@ -1153,20 +1158,29 @@ void update_tcp_connections(struct uloop_timeout *t) { uloop_timeout_set(&tcp_con_timer, timeout_config.update_tcp_con * 1000); } -void check_client_timeouts(struct uloop_timeout *t) { +void server_ping_clients(struct uloop_timeout *t) { dawnlog_debug_func("Entering..."); - check_client_timeout(timeout_config.client_timeout); + server_to_clients_ping(); - uloop_timeout_set(&client_timeout_timer, CHECK_CLIENT_TIMEOUT * 1000); + uloop_timeout_set(&server_ping_timer, 20 * 1000); +} + +void check_timeouts(struct uloop_timeout *t) { + dawnlog_debug_func("Entering..."); + + check_timeout(timeout_config.con_timeout); + + uloop_timeout_set(&check_timeout_timer, CHECK_TIMEOUT * 1000); } void start_tcp_con_update() { dawnlog_debug_func("Entering..."); + uloop_timeout_add(&server_ping_timer); // update connections uloop_timeout_add(&tcp_con_timer); // callback = update_tcp_connections - uloop_timeout_add(&client_timeout_timer); // callback = client_timeout + uloop_timeout_add(&check_timeout_timer); // callback = con_timeout } void update_hostapd_sockets(struct uloop_timeout *t) { @@ -1902,7 +1916,7 @@ int uci_send_via_network() blobmsg_add_u32(&b, "update_tcp_con", timeout_config.update_tcp_con); blobmsg_add_u32(&b, "update_chan_util", timeout_config.update_chan_util); blobmsg_add_u32(&b, "update_beacon_reports", timeout_config.update_beacon_reports); - blobmsg_add_u32(&b, "client_timeout", timeout_config.client_timeout); + blobmsg_add_u32(&b, "con_timeout", timeout_config.con_timeout); blobmsg_close_table(&b, times); send_blob_attr_via_network(b.head, "uci");