datastorage: refactor to support scalability testing

src/CMakeLists.txt

@@ -13,6 +13,9 @@ SET(SOURCES
         storage/datastorage.c
         include/datastorage.h
 
+        storage/uface.c
+        include/uface.h
+
         network/networksocket.c
         include/networksocket.h
 
@@ -43,10 +46,25 @@ SET(SOURCES
         utils/ieee80211_utils.c
         include/ieee80211_utils.h)
 
+SET(SOURCES_TEST_STORAGE
+        test/test_storage.c
+
+        include/uface.h
+
+        utils/utils.c
+        include/utils.h
+
+        storage/datastorage.c
+        include/datastorage.h
+
+        utils/ieee80211_utils.c
+        include/ieee80211_utils.h)
+
 SET(LIBS
         ubox ubus json-c blobmsg_json uci gcrypt iwinfo)
 
 ADD_EXECUTABLE(dawn ${SOURCES})
+ADD_EXECUTABLE(test_storage ${SOURCES_TEST_STORAGE})
 
 TARGET_LINK_LIBRARIES(dawn ${LIBS})
 

src/include/datastorage.h

@@ -8,7 +8,6 @@
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
-#include <libubox/blobmsg_json.h>
 
 #ifndef ETH_ALEN
 #define ETH_ALEN 6
@@ -19,7 +18,7 @@
 // ---------------- Defines -------------------
 #define MAC_LIST_LENGTH 100
 
-// ---------------- Structs ----------------
+// ---------------- Global variables ----------------
 uint8_t mac_list[MAC_LIST_LENGTH][ETH_ALEN];
 
 // ---------------- Functions ----------
@@ -29,11 +28,12 @@ int insert_to_maclist(uint8_t mac[]);
 
 int mac_in_maclist(uint8_t mac[]);
 
+int mac_is_equal(uint8_t addr1[], uint8_t addr2[]);
+
+int mac_is_greater(uint8_t addr1[], uint8_t addr2[]);
 
 /* Metric */
 
-struct probe_metric_s dawn_metric;
-
 // ---------------- Structs ----------------
 struct probe_metric_s {
     int ap_weight;
@@ -96,12 +96,12 @@ struct network_config_s {
     int bandwidth;
 };
 
+// ---------------- Global variables ----------------
 struct network_config_s network_config;
 struct time_config_s timeout_config;
 
-// ---------------- Global variables ----------------
 struct probe_metric_s dawn_metric;
-
+extern int probe_entry_last;
 
 /* Probe, Auth, Assoc */
 
@@ -140,20 +140,20 @@ typedef struct hostapd_notify_entry_s {
 
 typedef struct auth_entry_s assoc_entry;
 
-#define DENY_REQ_ARRAY_LEN 100
-struct auth_entry_s denied_req_array[DENY_REQ_ARRAY_LEN];
-pthread_mutex_t denied_array_mutex;
-
-auth_entry insert_to_denied_req_array(auth_entry entry, int inc_counter);
-
 // ---------------- Defines ----------------
+#define DENY_REQ_ARRAY_LEN 100
 #define PROBE_ARRAY_LEN 1000
 
 #define SSID_MAX_LEN 32
 #define NEIGHBOR_REPORT_LEN 200
 
 // ---------------- Global variables ----------------
+struct auth_entry_s denied_req_array[DENY_REQ_ARRAY_LEN];
+extern int denied_req_last;
+pthread_mutex_t denied_array_mutex;
+
 struct probe_entry_s probe_array[PROBE_ARRAY_LEN];
+extern int probe_entry_last;
 pthread_mutex_t probe_array_mutex;
 
 // ---------------- Functions ----------------
@@ -165,17 +165,22 @@ probe_entry probe_array_delete(probe_entry entry);
 
 probe_entry probe_array_get_entry(uint8_t bssid_addr[], uint8_t client_addr[]);
 
+void remove_old_probe_entries(time_t current_time, long long int threshold);
+
 void print_probe_array();
 
 void print_probe_entry(probe_entry entry);
 
-void print_auth_entry(auth_entry entry);
+int eval_probe_metric(struct probe_entry_s probe_entry);
 
-void uloop_add_data_cbs();
+auth_entry denied_req_array_delete(auth_entry entry);
+
+auth_entry insert_to_denied_req_array(auth_entry entry, int inc_counter);
+
+void print_auth_entry(auth_entry entry);
 
 /* AP, Client */
 
-// blobmsg_alloc_string_buffer(&b, "signature", 1024);
 #define SIGNATURE_LEN 1024
 
 // ---------------- Structs ----------------
@@ -225,14 +230,13 @@ typedef struct ap_s {
 #define TIME_THRESHOLD_CLIENT_KICK 60
 
 // ---------------- Global variables ----------------
-struct client_s client_array[ARRAY_CLIENT_LEN];
-pthread_mutex_t client_array_mutex;
 struct ap_s ap_array[ARRAY_AP_LEN];
+extern int ap_entry_last;
 pthread_mutex_t ap_array_mutex;
 
-int mac_is_equal(uint8_t addr1[], uint8_t addr2[]);
+struct client_s client_array[ARRAY_CLIENT_LEN];
-
+extern int client_entry_last;
-int mac_is_greater(uint8_t addr1[], uint8_t addr2[]);
+pthread_mutex_t client_array_mutex;
 
 // ---------------- Functions ----------------
 
@@ -240,6 +244,8 @@ int probe_array_update_rssi(uint8_t bssid_addr[], uint8_t client_addr[], uint32_
 
 int probe_array_update_rcpi_rsni(uint8_t bssid_addr[], uint8_t client_addr[], uint32_t rcpi, uint32_t rsni, int send_network);
 
+void remove_old_client_entries(time_t current_time, long long int threshold);
+
 void insert_client_to_array(client entry);
 
 void kick_clients(uint8_t bssid[], uint32_t id);
@@ -252,24 +258,22 @@ void print_client_array();
 
 void print_client_entry(client entry);
 
+int is_connected_somehwere(uint8_t client_addr[]);
+
 ap insert_to_ap_array(ap entry);
 
+void remove_old_ap_entries(time_t current_time, long long int threshold);
+
 void print_ap_array();
 
 ap ap_array_get_ap(uint8_t bssid_addr[]);
 
-int build_hearing_map_sort_client(struct blob_buf *b);
-
-int build_network_overview(struct blob_buf *b);
-
 int probe_array_set_all_probe_count(uint8_t client_addr[], uint32_t probe_count);
 
 int ap_get_collision_count(int col_domain);
 
 void send_beacon_reports(uint8_t bssid[], int id);
 
-int ap_get_nr(struct blob_buf *b, uint8_t own_bssid_addr[]);
-
 /* Utils */
 
 // ---------------- Defines -------------------

src/include/ubus.h

@@ -4,6 +4,7 @@
 #include <libubox/blobmsg_json.h>
 #include <libubox/uloop.h>
 
+
 #include "datastorage.h"
 
 // 802.11 Status codes
@@ -84,16 +85,6 @@ int parse_to_clients(struct blob_attr *msg, int do_kick, uint32_t id);
  */
 int parse_to_hostapd_notify(struct blob_attr *msg, hostapd_notify_entry *notify_req);
 
-/**
- * Kick client from hostapd interface.
- * @param id - the ubus id.
- * @param client_addr - the client adress of the client to kick.
- * @param reason - the reason to kick the client.
- * @param deauth - if the client should be deauthenticated.
- * @param ban_time - the ban time the client is not allowed to connect again.
- */
-void del_client_interface(uint32_t id, const uint8_t *client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
-
 /**
  * Kick client from all hostapd interfaces.
  * @param client_addr - the client adress of the client to kick.
@@ -103,27 +94,12 @@ void del_client_interface(uint32_t id, const uint8_t *client_addr, uint32_t reas
  */
 void del_client_all_interfaces(const uint8_t *client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
 
-void wnm_disassoc_imminent(uint32_t id, const uint8_t *client_addr, char* dest_ap, uint32_t duration);
-
-/**
- * Send probe message via the network.
- * @param probe_entry
- * @return
- */
-int ubus_send_probe_via_network(struct probe_entry_s probe_entry);
-
 /**
  * Update the hostapd sockets.
  * @param t
  */
 void update_hostapd_sockets(struct uloop_timeout *t);
 
-/**
- * Set client timer for updating the clients.
- * @param time
- */
-void add_client_update_timer(time_t time);
-
 /**
  * Handle network messages.
  * @param msg
@@ -147,14 +123,6 @@ int send_blob_attr_via_network(struct blob_attr *msg, char *method);
  */
 void blobmsg_add_macaddr(struct blob_buf *buf, const char *name, const uint8_t *addr);
 
-/**
- * Function to set the probe counter to the min probe request.
- * This allows that the client is able to connect directly without sending multiple probe requests to the Access Point.
- * @param client_addr
- * @return
- */
-int send_set_probe(uint8_t client_addr[]);
-
 /**
  * Send control message to all hosts to add the mac to a don't control list.
  * @param client_addr
@@ -164,6 +132,10 @@ int send_add_mac(uint8_t *client_addr);
 
 int uci_send_via_network();
 
-void ubus_send_beacon_report(uint8_t client[], int id);
+int build_hearing_map_sort_client(struct blob_buf *b);
+
+int build_network_overview(struct blob_buf *b);
+
+int ap_get_nr(struct blob_buf *b, uint8_t own_bssid_addr[]);
 
 #endif

src/include/uface.h

@@ -0,0 +1,41 @@
+#ifndef __DAWN_UFACE_H
+#define __DAWN_UFACE_H
+
+/**
+ * Set client timer for updating the clients.
+ * @param time
+ */
+void add_client_update_timer(time_t time);
+
+/**
+ * Kick client from hostapd interface.
+ * @param id - the ubus id.
+ * @param client_addr - the client adress of the client to kick.
+ * @param reason - the reason to kick the client.
+ * @param deauth - if the client should be deauthenticated.
+ * @param ban_time - the ban time the client is not allowed to connect again.
+ */
+void del_client_interface(uint32_t id, const uint8_t *client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
+
+/**
+ * Function to set the probe counter to the min probe request.
+ * This allows that the client is able to connect directly without sending multiple probe requests to the Access Point.
+ * @param client_addr
+ * @return
+ */
+int send_set_probe(uint8_t client_addr[]);
+
+void ubus_send_beacon_report(uint8_t client[], int id);
+
+/**
+ * Send probe message via the network.
+ * @param probe_entry
+ * @return
+ */
+int ubus_send_probe_via_network(struct probe_entry_s probe_entry);
+
+void uloop_add_data_cbs();
+
+void wnm_disassoc_imminent(uint32_t id, const uint8_t *client_addr, char* dest_ap, uint32_t duration);
+
+#endif // __DAWN_UFACE_H

src/network/tcpsocket.c

@@ -162,11 +162,10 @@ static void client_not_be_used_read_cb(struct ustream *s, int bytes) {
 
 static void connect_cb(struct uloop_fd *f, unsigned int events) {
 
-    struct network_con_s *entry = container_of(f,
+    struct network_con_s *entry = container_of(f, struct network_con_s, fd);
-    struct network_con_s, fd);
 
     if (f->eof || f->error) {
-        fprintf(stderr, "Connection failed\n");
+        fprintf(stderr, "Connection failed (%s)\n", f->eof ? "EOF" : "ERROR");
         close(entry->fd.fd);
         list_del(&entry->list);
         free(entry);

src/storage/datastorage.c

@@ -1,13 +1,13 @@
-#include "datastorage.h"
-
 #include <limits.h>
-#include <libubox/uloop.h>
+#include <stdbool.h>
 
-#include "ubus.h"
 #include "dawn_iwinfo.h"
 #include "utils.h"
 #include "ieee80211_utils.h"
 
+#include "datastorage.h"
+#include "uface.h"
+
 #define MAC2STR(a) (a)[0], (a)[1], (a)[2], (a)[3], (a)[4], (a)[5]
 
 int go_next_help(char sort_order[], int i, probe_entry entry,
@@ -16,32 +16,22 @@ int go_next_help(char sort_order[], int i, probe_entry entry,
 int go_next(char sort_order[], int i, probe_entry entry,
             probe_entry next_entry);
 
-void remove_old_probe_entries(time_t current_time, long long int threshold);
-
 int client_array_go_next(char sort_order[], int i, client entry,
                          client next_entry);
 
 int client_array_go_next_help(char sort_order[], int i, client entry,
                               client next_entry);
 
-void remove_old_client_entries(time_t current_time, long long int threshold);
-
-int eval_probe_metric(struct probe_entry_s probe_entry);
-
 int kick_client(struct client_s client_entry, char* neighbor_report);
 
 void ap_array_insert(ap entry);
 
 ap ap_array_delete(ap entry);
 
-void remove_old_ap_entries(time_t current_time, long long int threshold);
-
 void print_ap_entry(ap entry);
 
 int is_connected(uint8_t bssid_addr[], uint8_t client_addr[]);
 
-int is_connected_somehwere(uint8_t client_addr[]);
-
 int compare_station_count(uint8_t *bssid_addr_own, uint8_t *bssid_addr_to_compare, uint8_t *client_addr,
                           int automatic_kick);
 
@@ -49,8 +39,6 @@ int compare_ssid(uint8_t *bssid_addr_own, uint8_t *bssid_addr_to_compare);
 
 void denied_req_array_insert(auth_entry entry);
 
-auth_entry denied_req_array_delete(auth_entry entry);
-
 int denied_req_array_go_next(char sort_order[], int i, auth_entry entry,
                              auth_entry next_entry);
 
@@ -63,30 +51,6 @@ int ap_entry_last = -1;
 int mac_list_entry_last = -1;
 int denied_req_last = -1;
 
-void remove_probe_array_cb(struct uloop_timeout *t);
-
-struct uloop_timeout probe_timeout = {
-        .cb = remove_probe_array_cb
-};
-
-void remove_client_array_cb(struct uloop_timeout *t);
-
-struct uloop_timeout client_timeout = {
-        .cb = remove_client_array_cb
-};
-
-void remove_ap_array_cb(struct uloop_timeout *t);
-
-struct uloop_timeout ap_timeout = {
-        .cb = remove_ap_array_cb
-};
-
-void denied_req_array_cb(struct uloop_timeout *t);
-
-struct uloop_timeout denied_req_timeout = {
-        .cb = denied_req_array_cb
-};
-
 void send_beacon_reports(uint8_t bssid[], int id) {
     pthread_mutex_lock(&client_array_mutex);
 
@@ -109,170 +73,8 @@ void send_beacon_reports(uint8_t bssid[], int id) {
     pthread_mutex_unlock(&client_array_mutex);
 }
 
-int build_hearing_map_sort_client(struct blob_buf *b) {
+// TODO: Can metric be cached once calculated? Add score_fresh indicator and reset when signal changes
-    print_probe_array();
+// TODO: as rest of values look to be static fr any given entry.
-    pthread_mutex_lock(&probe_array_mutex);
-
-    void *client_list, *ap_list, *ssid_list;
-    char ap_mac_buf[20];
-    char client_mac_buf[20];
-
-    blob_buf_init(b, 0);
-    int m;
-    for (m = 0; m <= ap_entry_last; m++) {
-        if (m > 0) {
-            if (strcmp((char *) ap_array[m].ssid, (char *) ap_array[m - 1].ssid) == 0) {
-                continue;
-            }
-        }
-        ssid_list = blobmsg_open_table(b, (char *) ap_array[m].ssid);
-
-        int i;
-        for (i = 0; i <= probe_entry_last; i++) {
-            /*if(!mac_is_equal(ap_array[m].bssid_addr, probe_array[i].bssid_addr))
-            {
-                continue;
-            }*/
-
-            ap ap_entry_i = ap_array_get_ap(probe_array[i].bssid_addr);
-
-            if (!mac_is_equal(ap_entry_i.bssid_addr, probe_array[i].bssid_addr)) {
-                continue;
-            }
-
-            if (strcmp((char *) ap_entry_i.ssid, (char *) ap_array[m].ssid) != 0) {
-                continue;
-            }
-
-            int k;
-            sprintf(client_mac_buf, MACSTR, MAC2STR(probe_array[i].client_addr));
-            client_list = blobmsg_open_table(b, client_mac_buf);
-            for (k = i; k <= probe_entry_last; k++) {
-                ap ap_entry = ap_array_get_ap(probe_array[k].bssid_addr);
-
-                if (!mac_is_equal(ap_entry.bssid_addr, probe_array[k].bssid_addr)) {
-                    continue;
-                }
-
-                if (strcmp((char *) ap_entry.ssid, (char *) ap_array[m].ssid) != 0) {
-                    continue;
-                }
-
-                if (!mac_is_equal(probe_array[k].client_addr, probe_array[i].client_addr)) {
-                    i = k - 1;
-                    break;
-                } else if (k == probe_entry_last) {
-                    i = k;
-                }
-
-                sprintf(ap_mac_buf, MACSTR, MAC2STR(probe_array[k].bssid_addr));
-                ap_list = blobmsg_open_table(b, ap_mac_buf);
-                blobmsg_add_u32(b, "signal", probe_array[k].signal);
-                blobmsg_add_u32(b, "rcpi", probe_array[k].rcpi);
-                blobmsg_add_u32(b, "rsni", probe_array[k].rsni);
-                blobmsg_add_u32(b, "freq", probe_array[k].freq);
-                blobmsg_add_u8(b, "ht_capabilities", probe_array[k].ht_capabilities);
-                blobmsg_add_u8(b, "vht_capabilities", probe_array[k].vht_capabilities);
-
-
-                // check if ap entry is available
-                blobmsg_add_u32(b, "channel_utilization", ap_entry.channel_utilization);
-                blobmsg_add_u32(b, "num_sta", ap_entry.station_count);
-                blobmsg_add_u8(b, "ht_support", ap_entry.ht_support);
-                blobmsg_add_u8(b, "vht_support", ap_entry.vht_support);
-
-                blobmsg_add_u32(b, "score", eval_probe_metric(probe_array[k]));
-                blobmsg_close_table(b, ap_list);
-            }
-            blobmsg_close_table(b, client_list);
-        }
-        blobmsg_close_table(b, ssid_list);
-    }
-    pthread_mutex_unlock(&probe_array_mutex);
-    return 0;
-}
-
-int build_network_overview(struct blob_buf *b) {
-    void *client_list, *ap_list, *ssid_list;
-    char ap_mac_buf[20];
-    char client_mac_buf[20];
-
-    blob_buf_init(b, 0);
-    int m;
-    for (m = 0; m <= ap_entry_last; m++) {
-        bool add_ssid = false;
-        bool close_ssid = false;
-
-        if (m == 0 || strcmp((char *) ap_array[m].ssid, (char *) ap_array[m - 1].ssid) != 0) {
-            add_ssid = true;
-        }
-
-        if (m >= ap_entry_last || strcmp((char *) ap_array[m].ssid, (char *) ap_array[m + 1].ssid) != 0) {
-            close_ssid = true;
-        }
-
-        if(add_ssid)
-        {
-            ssid_list = blobmsg_open_table(b, (char *) ap_array[m].ssid);
-        }
-        sprintf(ap_mac_buf, MACSTR, MAC2STR(ap_array[m].bssid_addr));
-        ap_list = blobmsg_open_table(b, ap_mac_buf);
-
-        blobmsg_add_u32(b, "freq", ap_array[m].freq);
-        blobmsg_add_u32(b, "channel_utilization", ap_array[m].channel_utilization);
-        blobmsg_add_u32(b, "num_sta", ap_array[m].station_count);
-        blobmsg_add_u8(b, "ht_support", ap_array[m].ht_support);
-        blobmsg_add_u8(b, "vht_support", ap_array[m].vht_support);
-
-        char *nr;
-        nr = blobmsg_alloc_string_buffer(b, "neighbor_report", NEIGHBOR_REPORT_LEN);
-        sprintf(nr, "%s", ap_array[m].neighbor_report);
-        blobmsg_add_string_buffer(b);
-
-        int k;
-        for (k = 0; k <= client_entry_last; k++) {
-
-            if (mac_is_greater(ap_array[m].bssid_addr, client_array[k].bssid_addr))
-            {
-                break;
-            }
-
-            if (mac_is_equal(ap_array[m].bssid_addr, client_array[k].bssid_addr)) {
-                sprintf(client_mac_buf, MACSTR, MAC2STR(client_array[k].client_addr));
-                client_list = blobmsg_open_table(b, client_mac_buf);
-
-                if(strlen(client_array[k].signature) != 0)
-                {
-                    char *s;
-                    s = blobmsg_alloc_string_buffer(b, "signature", 1024);
-                    sprintf(s, "%s", client_array[k].signature);
-                    blobmsg_add_string_buffer(b);
-                }
-                blobmsg_add_u8(b, "ht", client_array[k].ht);
-                blobmsg_add_u8(b, "vht", client_array[k].vht);
-                blobmsg_add_u32(b, "collision_count", ap_get_collision_count(ap_array[m].collision_domain));
-
-                int n;
-                for(n = 0; n <= probe_entry_last; n++)
-                {
-                    if (mac_is_equal(client_array[k].client_addr, probe_array[n].client_addr) &&
-                            mac_is_equal(client_array[k].bssid_addr, probe_array[n].bssid_addr)) {
-                        blobmsg_add_u32(b, "signal", probe_array[n].signal);
-                        break;
-                    }
-                }
-                blobmsg_close_table(b, client_list);
-            }
-        }
-        blobmsg_close_table(b, ap_list);
-        if(close_ssid)
-        {
-            blobmsg_close_table(b, ssid_list);
-        }
-    }
-    return 0;
-}
-
 int eval_probe_metric(struct probe_entry_s probe_entry) {
 
     int score = 0;
@@ -899,36 +701,6 @@ ap insert_to_ap_array(ap entry) {
 }
 
 
-int ap_get_nr(struct blob_buf *b_local, uint8_t own_bssid_addr[]) {
-
-    pthread_mutex_lock(&ap_array_mutex);
-    int i;
-
-    void* nbs = blobmsg_open_array(b_local, "list");
-
-    for (i = 0; i <= ap_entry_last; i++) {
-        if (mac_is_equal(own_bssid_addr, ap_array[i].bssid_addr)) {
-            continue; //TODO: Skip own entry?!
-        }
-
-        void* nr_entry = blobmsg_open_array(b_local, NULL);
-
-        char mac_buf[20];
-        sprintf(mac_buf, MACSTRLOWER, MAC2STR(ap_array[i].bssid_addr));
-        blobmsg_add_string(b_local, NULL, mac_buf);
-
-        blobmsg_add_string(b_local, NULL, (char *) ap_array[i].ssid);
-        blobmsg_add_string(b_local, NULL, ap_array[i].neighbor_report);
-        blobmsg_close_array(b_local, nr_entry);
-
-    }
-    blobmsg_close_array(b_local, nbs);
-
-    pthread_mutex_unlock(&ap_array_mutex);
-
-    return 0;
-}
-
 int ap_get_collision_count(int col_domain) {
 
     int ret_sta_count = 0;
@@ -1050,71 +822,6 @@ void remove_old_ap_entries(time_t current_time, long long int threshold) {
     }
 }
 
-void uloop_add_data_cbs() {
-    uloop_timeout_add(&probe_timeout);
-    uloop_timeout_add(&client_timeout);
-    uloop_timeout_add(&ap_timeout);
-
-    if (dawn_metric.use_driver_recog) {
-        uloop_timeout_add(&denied_req_timeout);
-    }
-}
-
-void remove_probe_array_cb(struct uloop_timeout *t) {
-    pthread_mutex_lock(&probe_array_mutex);
-    printf("[Thread] : Removing old probe entries!\n");
-    remove_old_probe_entries(time(0), timeout_config.remove_probe);
-    printf("[Thread] : Removing old entries finished!\n");
-    pthread_mutex_unlock(&probe_array_mutex);
-    uloop_timeout_set(&probe_timeout, timeout_config.remove_probe * 1000);
-}
-
-void remove_client_array_cb(struct uloop_timeout *t) {
-    pthread_mutex_lock(&client_array_mutex);
-    printf("[Thread] : Removing old client entries!\n");
-    remove_old_client_entries(time(0), timeout_config.update_client);
-    pthread_mutex_unlock(&client_array_mutex);
-    uloop_timeout_set(&client_timeout, timeout_config.update_client * 1000);
-}
-
-void remove_ap_array_cb(struct uloop_timeout *t) {
-    pthread_mutex_lock(&ap_array_mutex);
-    printf("[ULOOP] : Removing old ap entries!\n");
-    remove_old_ap_entries(time(0), timeout_config.remove_ap);
-    pthread_mutex_unlock(&ap_array_mutex);
-    uloop_timeout_set(&ap_timeout, timeout_config.remove_ap * 1000);
-}
-
-void denied_req_array_cb(struct uloop_timeout *t) {
-    pthread_mutex_lock(&denied_array_mutex);
-    printf("[ULOOP] : Processing denied authentication!\n");
-
-    time_t current_time = time(0);
-
-    for (int i = 0; i <= denied_req_last; i++) {
-        // check counter
-
-        //check timer
-        if (denied_req_array[i].time < current_time - timeout_config.denied_req_threshold) {
-
-            // client is not connected for a given time threshold!
-            if (!is_connected_somehwere(denied_req_array[i].client_addr)) {
-                printf("Client has probably a bad driver!\n");
-
-                // problem that somehow station will land into this list
-                // maybe delete again?
-                if (insert_to_maclist(denied_req_array[i].client_addr) == 0) {
-                    send_add_mac(denied_req_array[i].client_addr);
-                    write_mac_to_file("/tmp/dawn_mac_list", denied_req_array[i].client_addr);
-                }
-            }
-            denied_req_array_delete(denied_req_array[i]);
-        }
-    }
-    pthread_mutex_unlock(&denied_array_mutex);
-    uloop_timeout_set(&denied_req_timeout, timeout_config.denied_req_threshold * 1000);
-}
-
 void insert_client_to_array(client entry) {
     pthread_mutex_lock(&client_array_mutex);
     entry.time = time(0);
@@ -1137,6 +844,7 @@ void insert_macs_from_file() {
     size_t len = 0;
     ssize_t read;
 
+// TODO: Loading to array is not constrained by array checks.  Buffer overrun can occur.
     fp = fopen("/tmp/dawn_mac_list", "r");
     if (fp == NULL)
         exit(EXIT_FAILURE);

src/storage/uface.c

@@ -0,0 +1,96 @@
+#include <libubox/uloop.h>
+#include "datastorage.h"
+#include "utils.h"
+#include "ubus.h"
+#include "uface.h"
+
+void remove_probe_array_cb(struct uloop_timeout *t);
+
+struct uloop_timeout probe_timeout = {
+        .cb = remove_probe_array_cb
+};
+
+void remove_client_array_cb(struct uloop_timeout *t);
+
+struct uloop_timeout client_timeout = {
+        .cb = remove_client_array_cb
+};
+
+void remove_ap_array_cb(struct uloop_timeout *t);
+
+struct uloop_timeout ap_timeout = {
+        .cb = remove_ap_array_cb
+};
+
+void denied_req_array_cb(struct uloop_timeout *t);
+
+struct uloop_timeout denied_req_timeout = {
+        .cb = denied_req_array_cb
+};
+
+void uloop_add_data_cbs() {
+    uloop_timeout_add(&probe_timeout);
+    uloop_timeout_add(&client_timeout);
+    uloop_timeout_add(&ap_timeout);
+
+    if (dawn_metric.use_driver_recog) {
+        uloop_timeout_add(&denied_req_timeout);
+    }
+}
+
+void remove_probe_array_cb(struct uloop_timeout *t) {
+    pthread_mutex_lock(&probe_array_mutex);
+    printf("[Thread] : Removing old probe entries!\n");
+    remove_old_probe_entries(time(0), timeout_config.remove_probe);
+    printf("[Thread] : Removing old entries finished!\n");
+    pthread_mutex_unlock(&probe_array_mutex);
+    uloop_timeout_set(&probe_timeout, timeout_config.remove_probe * 1000);
+}
+
+void remove_client_array_cb(struct uloop_timeout *t) {
+    pthread_mutex_lock(&client_array_mutex);
+    printf("[Thread] : Removing old client entries!\n");
+    remove_old_client_entries(time(0), timeout_config.update_client);
+    pthread_mutex_unlock(&client_array_mutex);
+    uloop_timeout_set(&client_timeout, timeout_config.update_client * 1000);
+}
+
+void remove_ap_array_cb(struct uloop_timeout *t) {
+    pthread_mutex_lock(&ap_array_mutex);
+    printf("[ULOOP] : Removing old ap entries!\n");
+    remove_old_ap_entries(time(0), timeout_config.remove_ap);
+    pthread_mutex_unlock(&ap_array_mutex);
+    uloop_timeout_set(&ap_timeout, timeout_config.remove_ap * 1000);
+}
+
+void denied_req_array_cb(struct uloop_timeout *t) {
+    pthread_mutex_lock(&denied_array_mutex);
+    printf("[ULOOP] : Processing denied authentication!\n");
+
+    time_t current_time = time(0);
+
+    for (int i = 0; i <= denied_req_last; i++) {
+        // check counter
+
+        //check timer
+        if (denied_req_array[i].time < current_time - timeout_config.denied_req_threshold) {
+
+            // client is not connected for a given time threshold!
+            if (!is_connected_somehwere(denied_req_array[i].client_addr)) {
+                printf("Client has probably a bad driver!\n");
+
+                // problem that somehow station will land into this list
+                // maybe delete again?
+                if (insert_to_maclist(denied_req_array[i].client_addr) == 0) {
+                    send_add_mac(denied_req_array[i].client_addr);
+// TODO: File can grow arbitarily large.  Resource consumption risk.
+// TODO: Consolidate use of file across source: shared resource for name, single point of access?
+                    write_mac_to_file("/tmp/dawn_mac_list", denied_req_array[i].client_addr);
+                }
+            }
+            denied_req_array_delete(denied_req_array[i]);
+        }
+    }
+    pthread_mutex_unlock(&denied_array_mutex);
+    uloop_timeout_set(&denied_req_timeout, timeout_config.denied_req_threshold * 1000);
+}

src/test/test_storage.c

@@ -0,0 +1,181 @@
+#include <limits.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+#include "dawn_iwinfo.h"
+#include "utils.h"
+#include "ieee80211_utils.h"
+
+#include "datastorage.h"
+#include "uface.h"
+
+/*** External functions ***/
+void ap_array_insert(ap entry);
+ap ap_array_delete(ap entry);
+
+/*** Testing structures, etc ***/
+union __attribute__((__packed__)) mac_mangler
+{
+    struct {
+        uint8_t b[6];
+        uint8_t packing[2];
+    } u8;
+    uint64_t u64;
+};
+
+/*** Test code */
+int ap_array_helper_auto(int action, int i0, int i1);
+int ap_array_helper_auto(int action, int i0, int i1)
+{
+int m;
+int step = (i0 > i1) ? -1 : 1;
+int ret = 0;
+
+    switch (action)
+    {
+    case 0:
+    case 1:
+        m = i0;
+        int cont = 1;
+        while (cont) {
+        union mac_mangler this_mac;
+        ap ap0;
+
+            this_mac.u64 = m;
+            memcpy(ap0.bssid_addr, this_mac.u8.b, sizeof(ap0.bssid_addr));
+	    if (action == 0)
+                ap_array_insert(ap0);
+	    else
+                ap_array_delete(ap0);
+
+            if (m == i1)
+                cont = 0;
+            else
+                m += step;
+        }
+        break;
+    default:
+        ret = 1;
+        break;
+    }
+
+    return ret;
+}
+
+int main(int argc, char** argv)
+{
+int ret = 0;
+int args_ok = 1;
+int arg_consumed = 0;
+
+    printf("DAWN datastorage.c test harness.  Ready for commands...\n");
+
+    int this_arg = 1;
+    argv++;
+
+    while (args_ok)
+    {
+        if (strcmp(*argv, "help") == 0)
+        {
+            arg_consumed = 1;
+	    if (this_arg + arg_consumed > argc) goto next_command;
+
+	    printf("Help is on its way...\n");
+        }
+	else if (strcmp(*argv, "ap_show") == 0)
+	{
+            arg_consumed = 1;
+	    if (this_arg + arg_consumed > argc) goto next_command;
+
+            print_ap_array();
+	}
+        else if (strcmp(*argv, "ap_add_auto") == 0)
+	{
+            arg_consumed = 3;
+	    if (this_arg + arg_consumed > argc) goto next_command;
+
+	    ap_array_helper_auto(0, atoi(*(argv + 1)), atoi(*(argv + 2)));
+	}
+	else if (strcmp(*argv, "ap_del_auto") == 0)
+	{
+            arg_consumed = 3;
+	    if (this_arg + arg_consumed > argc) goto next_command;
+
+	    ap_array_helper_auto(1, atoi(*(argv + 1)), atoi(*(argv + 2)));
+	}
+	else
+	{
+            arg_consumed = 1;
+	    if (this_arg + arg_consumed > argc) goto next_command;
+
+	    printf("COMMAND \"%s\": Unknown - skipping!\n", *argv);
+	}
+
+next_command:
+	this_arg += arg_consumed;
+        if (this_arg > argc)
+	{
+            printf("Commands are mangled at: \"%s\"!\n", *argv);
+            args_ok = 0;
+	}
+        else if (this_arg == argc)
+            args_ok = 0;
+        else
+	    argv += arg_consumed;
+    }
+
+    printf("\n\nDAWN datastorage.c test harness - finshed.  \n");
+
+    return ret;
+}
+
+void ubus_send_beacon_report(uint8_t client[], int id)
+{
+    printf("send_beacon_report() was called...\n");
+}
+
+int send_set_probe(uint8_t client_addr[])
+{
+    printf("send_set_probe() was called...\n");
+    return 0;
+}
+
+void wnm_disassoc_imminent(uint32_t id, const uint8_t *client_addr, char* dest_ap, uint32_t duration)
+{
+    printf("wnm_disassoc_imminent() was called...\n");
+}
+
+void add_client_update_timer(time_t time)
+{
+    printf("add_client_update_timer() was called...\n");
+}
+
+void del_client_interface(uint32_t id, const uint8_t *client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time)
+{
+    printf("del_client_interface() was called...\n");
+}
+
+int ubus_send_probe_via_network(struct probe_entry_s probe_entry)
+{
+    printf("send_probe_via_network() was called...\n");
+    return 0;
+}
+
+int get_rssi_iwinfo(uint8_t *client_addr)
+{
+    printf("get_rssi_iwinfo() was called...\n");
+    return 0;
+}
+
+int get_expected_throughput_iwinfo(uint8_t *client_addr)
+{
+    printf("get_expected_throughput_iwinfo() was called...\n");
+    return 0;
+}
+
+int get_bandwidth_iwinfo(uint8_t *client_addr, float *rx_rate, float *tx_rate)
+{
+    printf("get_bandwidth_iwinfo() was called...\n");
+    return 0;
+}
+

src/utils/ubus.c

@@ -1,3 +1,6 @@
+#include <limits.h>
+
+
 #include <ctype.h>
 #include <dirent.h>
 #include <libubox/blobmsg_json.h>
@@ -14,14 +17,16 @@
 #define REQ_TYPE_AUTH 1
 #define REQ_TYPE_ASSOC 2
 
-#include "ubus.h"
-
 #include "networksocket.h"
 #include "utils.h"
 #include "dawn_uci.h"
 #include "dawn_iwinfo.h"
-#include "datastorage.h"
 #include "tcpsocket.h"
+#include "ieee80211_utils.h"
+
+#include "datastorage.h"
+#include "uface.h"
+#include "ubus.h"
 
 static struct ubus_context *ctx = NULL;
 
@@ -703,6 +708,7 @@ int handle_network_msg(char *msg) {
 
     // add inactive death...
 
+// TODO: strncmp() look wrong - should all tests be for n = 5 characters? Shorthand checks?
     if (strncmp(method, "probe", 5) == 0) {
         probe_entry entry;
         if (parse_to_probe_req(data_buf.head, &entry) == 0) {
@@ -1442,6 +1448,8 @@ static int parse_add_mac_to_file(struct blob_attr *msg) {
         hwaddr_aton(blobmsg_data(attr), addr);
 
         if (insert_to_maclist(addr) == 0) {
+// TODO: File can grow arbitarily large.  Resource consumption risk.
+// TODO: Consolidate use of file across source: shared resource for name, single point of access?
             write_mac_to_file("/tmp/dawn_mac_list", addr);
         }
     }
@@ -1992,3 +2000,197 @@ int handle_uci_config(struct blob_attr *msg) {
 
     return 0;
 }
+
+int build_hearing_map_sort_client(struct blob_buf *b) {
+    print_probe_array();
+    pthread_mutex_lock(&probe_array_mutex);
+
+    void *client_list, *ap_list, *ssid_list;
+    char ap_mac_buf[20];
+    char client_mac_buf[20];
+
+    blob_buf_init(b, 0);
+    int m;
+    for (m = 0; m <= ap_entry_last; m++) {
+        if (m > 0) {
+            if (strcmp((char *) ap_array[m].ssid, (char *) ap_array[m - 1].ssid) == 0) {
+                continue;
+            }
+        }
+        ssid_list = blobmsg_open_table(b, (char *) ap_array[m].ssid);
+
+        int i;
+        for (i = 0; i <= probe_entry_last; i++) {
+            /*if(!mac_is_equal(ap_array[m].bssid_addr, probe_array[i].bssid_addr))
+            {
+                continue;
+            }*/
+
+            ap ap_entry_i = ap_array_get_ap(probe_array[i].bssid_addr);
+
+            if (!mac_is_equal(ap_entry_i.bssid_addr, probe_array[i].bssid_addr)) {
+                continue;
+            }
+
+            if (strcmp((char *) ap_entry_i.ssid, (char *) ap_array[m].ssid) != 0) {
+                continue;
+            }
+
+            int k;
+            sprintf(client_mac_buf, MACSTR, MAC2STR(probe_array[i].client_addr));
+            client_list = blobmsg_open_table(b, client_mac_buf);
+            for (k = i; k <= probe_entry_last; k++) {
+                ap ap_entry = ap_array_get_ap(probe_array[k].bssid_addr);
+
+                if (!mac_is_equal(ap_entry.bssid_addr, probe_array[k].bssid_addr)) {
+                    continue;
+                }
+
+                if (strcmp((char *) ap_entry.ssid, (char *) ap_array[m].ssid) != 0) {
+                    continue;
+                }
+
+                if (!mac_is_equal(probe_array[k].client_addr, probe_array[i].client_addr)) {
+                    i = k - 1;
+                    break;
+                } else if (k == probe_entry_last) {
+                    i = k;
+                }
+
+                sprintf(ap_mac_buf, MACSTR, MAC2STR(probe_array[k].bssid_addr));
+                ap_list = blobmsg_open_table(b, ap_mac_buf);
+                blobmsg_add_u32(b, "signal", probe_array[k].signal);
+                blobmsg_add_u32(b, "rcpi", probe_array[k].rcpi);
+                blobmsg_add_u32(b, "rsni", probe_array[k].rsni);
+                blobmsg_add_u32(b, "freq", probe_array[k].freq);
+                blobmsg_add_u8(b, "ht_capabilities", probe_array[k].ht_capabilities);
+                blobmsg_add_u8(b, "vht_capabilities", probe_array[k].vht_capabilities);
+
+
+                // check if ap entry is available
+                blobmsg_add_u32(b, "channel_utilization", ap_entry.channel_utilization);
+                blobmsg_add_u32(b, "num_sta", ap_entry.station_count);
+                blobmsg_add_u8(b, "ht_support", ap_entry.ht_support);
+                blobmsg_add_u8(b, "vht_support", ap_entry.vht_support);
+
+                blobmsg_add_u32(b, "score", eval_probe_metric(probe_array[k]));
+                blobmsg_close_table(b, ap_list);
+            }
+            blobmsg_close_table(b, client_list);
+        }
+        blobmsg_close_table(b, ssid_list);
+    }
+    pthread_mutex_unlock(&probe_array_mutex);
+    return 0;
+}
+
+int build_network_overview(struct blob_buf *b) {
+    void *client_list, *ap_list, *ssid_list;
+    char ap_mac_buf[20];
+    char client_mac_buf[20];
+
+    blob_buf_init(b, 0);
+    int m;
+    for (m = 0; m <= ap_entry_last; m++) {
+        bool add_ssid = false;
+        bool close_ssid = false;
+
+        if (m == 0 || strcmp((char *) ap_array[m].ssid, (char *) ap_array[m - 1].ssid) != 0) {
+            add_ssid = true;
+        }
+
+        if (m >= ap_entry_last || strcmp((char *) ap_array[m].ssid, (char *) ap_array[m + 1].ssid) != 0) {
+            close_ssid = true;
+        }
+
+        if(add_ssid)
+        {
+            ssid_list = blobmsg_open_table(b, (char *) ap_array[m].ssid);
+        }
+        sprintf(ap_mac_buf, MACSTR, MAC2STR(ap_array[m].bssid_addr));
+        ap_list = blobmsg_open_table(b, ap_mac_buf);
+
+        blobmsg_add_u32(b, "freq", ap_array[m].freq);
+        blobmsg_add_u32(b, "channel_utilization", ap_array[m].channel_utilization);
+        blobmsg_add_u32(b, "num_sta", ap_array[m].station_count);
+        blobmsg_add_u8(b, "ht_support", ap_array[m].ht_support);
+        blobmsg_add_u8(b, "vht_support", ap_array[m].vht_support);
+
+        char *nr;
+        nr = blobmsg_alloc_string_buffer(b, "neighbor_report", NEIGHBOR_REPORT_LEN);
+        sprintf(nr, "%s", ap_array[m].neighbor_report);
+        blobmsg_add_string_buffer(b);
+
+        int k;
+        for (k = 0; k <= client_entry_last; k++) {
+
+            if (mac_is_greater(ap_array[m].bssid_addr, client_array[k].bssid_addr))
+            {
+                break;
+            }
+
+            if (mac_is_equal(ap_array[m].bssid_addr, client_array[k].bssid_addr)) {
+                sprintf(client_mac_buf, MACSTR, MAC2STR(client_array[k].client_addr));
+                client_list = blobmsg_open_table(b, client_mac_buf);
+
+                if(strlen(client_array[k].signature) != 0)
+                {
+                    char *s;
+                    s = blobmsg_alloc_string_buffer(b, "signature", 1024);
+                    sprintf(s, "%s", client_array[k].signature);
+                    blobmsg_add_string_buffer(b);
+                }
+                blobmsg_add_u8(b, "ht", client_array[k].ht);
+                blobmsg_add_u8(b, "vht", client_array[k].vht);
+                blobmsg_add_u32(b, "collision_count", ap_get_collision_count(ap_array[m].collision_domain));
+
+                int n;
+                for(n = 0; n <= probe_entry_last; n++)
+                {
+                    if (mac_is_equal(client_array[k].client_addr, probe_array[n].client_addr) &&
+                            mac_is_equal(client_array[k].bssid_addr, probe_array[n].bssid_addr)) {
+                        blobmsg_add_u32(b, "signal", probe_array[n].signal);
+                        break;
+                    }
+                }
+                blobmsg_close_table(b, client_list);
+            }
+        }
+        blobmsg_close_table(b, ap_list);
+        if(close_ssid)
+        {
+            blobmsg_close_table(b, ssid_list);
+        }
+    }
+    return 0;
+}
+
+int ap_get_nr(struct blob_buf *b_local, uint8_t own_bssid_addr[]) {
+
+    pthread_mutex_lock(&ap_array_mutex);
+    int i;
+
+    void* nbs = blobmsg_open_array(b_local, "list");
+
+    for (i = 0; i <= ap_entry_last; i++) {
+        if (mac_is_equal(own_bssid_addr, ap_array[i].bssid_addr)) {
+            continue; //TODO: Skip own entry?!
+        }
+
+        void* nr_entry = blobmsg_open_array(b_local, NULL);
+
+        char mac_buf[20];
+        sprintf(mac_buf, MACSTRLOWER, MAC2STR(ap_array[i].bssid_addr));
+        blobmsg_add_string(b_local, NULL, mac_buf);
+
+        blobmsg_add_string(b_local, NULL, (char *) ap_array[i].ssid);
+        blobmsg_add_string(b_local, NULL, ap_array[i].neighbor_report);
+        blobmsg_close_array(b_local, nr_entry);
+
+    }
+    blobmsg_close_array(b_local, nbs);
+
+    pthread_mutex_unlock(&ap_array_mutex);
+
+    return 0;
+}

src/utils/utils.c

@@ -1,5 +1,8 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include "datastorage.h"
 #include "utils.h"
-#include "ubus.h"
 
 int string_is_greater(uint8_t *str, uint8_t *str_2) {
 
@@ -39,30 +42,37 @@ int hwaddr_aton(const char *txt, uint8_t *addr) {
         b = hex_to_bin(*txt++);
         if (b < 0) return -1;
         *addr++ = (a << 4) | b;
-        if (i < 5 && *txt++ != ':') return -1;
+// TODO: Should NUL terminator be checked for? Is aa:bb:cc:dd:ee:ff00 valid input?
+        if (i < (ETH_ALEN - 1) && *txt++ != ':') return -1;
     }
 
     return 0;
 }
 
+// TODO: Never called in DAWN code.  Remove?
+#if 0
+/* Convert badly formed MAC addresses with single digit element to double digit
+** eg 11:2:33:4:55:6 to 11:02:33:04:55:06 */`
 int convert_mac(char *in, char *out) {
     int i, j = 0;
 
-    for (i = 0; i < 6; i++) {
+    for (i = 0; i < ETH_ALEN; i++) {
-        if (in[j + 1] != ':' && in[j + 1] != '\0') {
+        /* Do we have a single-digit element? */
-            out[3 * i] = toupper(in[j]);
+        if (in[j + 1] == ':' || in[j + 1] == '\0') {
-            out[(3 * i) + 1] = toupper(in[j + 1]);
-            out[(3 * i) + 2] = in[j + 2];
-            j += 3;
-        } else {
             out[3 * i] = '0';
             out[(3 * i) + 1] = toupper(in[j]);
             out[(3 * i) + 2] = toupper(in[j + 1]);
             j += 2;
+        } else {
+            out[3 * i] = toupper(in[j]);
+            out[(3 * i) + 1] = toupper(in[j + 1]);
+            out[(3 * i) + 2] = in[j + 2];
+            j += 3;
         }
     }
     return 0;
 }
+#endif
 
 void write_mac_to_file(char *path, uint8_t addr[]) {
     FILE *f = fopen(path, "a");