datastorage/test: improve scalability and performance

datastorage: convert to linked lists and optimise use of pointers
datastorage: AP, client, probe, auth entry and MAC list converted to
             linked list
datastorage: functions adjusted to take pointers as parameters
datastorage: optimised sort and search functions added
mac_utils: struct dawn_mac added and comparisons adjusted
general: adjust code to  call new datastorage functions
test_storage: large scale 100 AP, 3000 client, 70k probe added

[fix commit]
Signed-off-by: Nick Hainke <vincent@systemli.org>
This commit is contained in:
Ian Clowes 2020-08-03 18:49:35 +02:00 committed by Polynomialdivision
parent 2f585043c3
commit 7262cf02d0
20 changed files with 77420 additions and 1437 deletions

View file

@ -9,20 +9,35 @@
#include "mac_utils.h"
#include "utils.h"
// Core data storage array sizes
#define ARRAY_AP_LEN 100
#define ARRAY_CLIENT_LEN 300
#define PROBE_ARRAY_LEN 1000
#define DENY_REQ_ARRAY_LEN 100
/* Mac */
// ---------------- Defines -------------------
#define MAC_LIST_LENGTH 100
// ---------------- Global variables ----------------
extern uint8_t mac_list[][ETH_ALEN];
extern struct mac_entry_s *mac_set;
struct mac_entry_s {
struct mac_entry_s* next_mac;
struct dawn_mac mac;
};
// ---------------- Functions ----------
void insert_macs_from_file();
int insert_to_maclist(uint8_t mac[]);
int insert_to_maclist(struct dawn_mac mac);
int mac_in_maclist(uint8_t mac[]);
int mac_in_maclist(struct dawn_mac mac);
struct mac_entry_s* insert_to_mac_array(struct mac_entry_s* entry, struct mac_entry_s** insert_pos);
void mac_array_delete(struct mac_entry_s* entry);
// ---------------- Global variables ----------------
@ -97,7 +112,7 @@ extern struct probe_metric_s dawn_metric;
/*** Core DAWN data structures for tracking network devices and status ***/
// Define this to remove printing / reporing of fields, and hence observe
// which fields are evaluated in use.
// which fields are evaluated in use at compile time.
// #define DAWN_NO_OUTPUT
// TODO notes:
@ -108,9 +123,11 @@ extern struct probe_metric_s dawn_metric;
// ---------------- Structs ----------------
typedef struct probe_entry_s {
uint8_t bssid_addr[ETH_ALEN];
uint8_t client_addr[ETH_ALEN];
uint8_t target_addr[ETH_ALEN]; // TODO: Never evaluated?
struct probe_entry_s* next_probe;
struct probe_entry_s* next_probe_skip;
struct dawn_mac client_addr;
struct dawn_mac bssid_addr;
struct dawn_mac target_addr; // TODO: Never evaluated?
uint32_t signal; // eval_probe_metric()
uint32_t freq; // eval_probe_metric()
uint8_t ht_capabilities; // eval_probe_metric()
@ -126,10 +143,17 @@ typedef struct probe_entry_s {
uint32_t rsni;
} probe_entry;
//struct probe_entry_s {
// struct dawn_mac client_addr;
// struct dawn_mac bssid_addr;
// struct probe_entry_s* entry;
//};
typedef struct auth_entry_s {
uint8_t bssid_addr[ETH_ALEN];
uint8_t client_addr[ETH_ALEN];
uint8_t target_addr[ETH_ALEN]; // TODO: Never evaluated?
struct auth_entry_s* next_auth;
struct dawn_mac bssid_addr;
struct dawn_mac client_addr;
struct dawn_mac target_addr; // TODO: Never evaluated?
uint32_t signal; // TODO: Never evaluated?
uint32_t freq; // TODO: Never evaluated?
time_t time; // Never used for removal?
@ -137,62 +161,42 @@ typedef struct auth_entry_s {
} auth_entry;
typedef struct hostapd_notify_entry_s {
uint8_t bssid_addr[ETH_ALEN];
uint8_t client_addr[ETH_ALEN];
struct dawn_mac bssid_addr;
struct dawn_mac client_addr;
} hostapd_notify_entry;
typedef struct auth_entry_s assoc_entry;
// ---------------- Defines ----------------
#define DENY_REQ_ARRAY_LEN 100
#define PROBE_ARRAY_LEN 1000
#define SSID_MAX_LEN 32
#define NEIGHBOR_REPORT_LEN 200
// ---------------- Global variables ----------------
extern struct auth_entry_s denied_req_array[];
extern int denied_req_last;
extern struct auth_entry_s *denied_req_set;
extern pthread_mutex_t denied_array_mutex;
extern struct probe_entry_s probe_array[];
extern int probe_entry_last;
extern struct probe_entry_s *probe_set;
extern pthread_mutex_t probe_array_mutex;
// ---------------- Functions ----------------
probe_entry insert_to_array(probe_entry entry, int inc_counter, int save_80211k, int is_beacon);
void probe_array_insert(probe_entry entry);
probe_entry probe_array_delete(probe_entry entry);
probe_entry probe_array_get_entry(uint8_t bssid_addr[], uint8_t client_addr[]);
void remove_old_probe_entries(time_t current_time, long long int threshold);
void print_probe_array();
void print_probe_entry(probe_entry entry);
int eval_probe_metric(struct probe_entry_s probe_entry);
void denied_req_array_insert(auth_entry entry);
auth_entry denied_req_array_delete(auth_entry entry);
auth_entry insert_to_denied_req_array(auth_entry entry, int inc_counter);
void print_auth_entry(auth_entry entry);
/* AP, Client */
#define SIGNATURE_LEN 1024
#define MAX_INTERFACE_NAME 64
// ---------------- Structs ----------------
// Testing only: Removes the ability to find clients via secondary search, hence replicates
// the pre-optimisation behaviour of only scanning the BSSID+MAC orderd list
//#define DAWN_CLIENT_SCAN_BC_ONLY
typedef struct client_s {
uint8_t bssid_addr[ETH_ALEN];
uint8_t client_addr[ETH_ALEN];
struct client_s* next_entry_bc;
struct client_s* next_skip_entry_bc;
#ifndef DAWN_CLIENT_SCAN_BC_ONLY
struct client_s* next_entry_c;
#endif
struct dawn_mac bssid_addr;
struct dawn_mac client_addr;
char signature[SIGNATURE_LEN]; // TODO: Never evaluated?
uint8_t ht_supported; // TODO: Never evaluated?
uint8_t vht_supported; // TODO: Never evaluated?
@ -214,7 +218,8 @@ typedef struct client_s {
} client;
typedef struct ap_s {
uint8_t bssid_addr[ETH_ALEN];
struct ap_s* next_ap;
struct dawn_mac bssid_addr;
uint32_t freq; // TODO: Never evaluated?
uint8_t ht_support; // eval_probe_metric()
uint8_t vht_support; // eval_probe_metric()
@ -231,71 +236,90 @@ typedef struct ap_s {
} ap;
// ---------------- Defines ----------------
#define ARRAY_AP_LEN 50
#define TIME_THRESHOLD_AP 30
#define ARRAY_CLIENT_LEN 1000
#define TIME_THRESHOLD_CLIENT 30
#define TIME_THRESHOLD_CLIENT_UPDATE 10
#define TIME_THRESHOLD_CLIENT_KICK 60
// ---------------- Global variables ----------------
extern struct ap_s ap_array[];
extern int ap_entry_last;
extern struct ap_s* ap_set;
extern pthread_mutex_t ap_array_mutex;
extern struct client_s client_array[];
extern int client_entry_last;
extern struct client_s *client_set_bc;
extern pthread_mutex_t client_array_mutex;
// ---------------- Functions ----------------
probe_entry *insert_to_array(probe_entry *entry, int inc_counter, int save_80211k, int is_beacon);
int probe_array_update_rssi(uint8_t bssid_addr[], uint8_t client_addr[], uint32_t rssi, int send_network);
int probe_array_delete(probe_entry *entry);
int probe_array_update_rcpi_rsni(uint8_t bssid_addr[], uint8_t client_addr[], uint32_t rcpi, uint32_t rsni, int send_network);
probe_entry *probe_array_get_entry(struct dawn_mac bssid_addr, struct dawn_mac client_addr);
void remove_old_probe_entries(time_t current_time, long long int threshold);
void print_probe_array();
void print_probe_entry(probe_entry *entry);
int eval_probe_metric(struct probe_entry_s * probe_entry, ap *ap_entry);
void denied_req_array_delete(auth_entry *entry);
auth_entry *insert_to_denied_req_array(auth_entry*entry, int inc_counter);
void print_auth_entry(auth_entry *entry);
// ---------------- Functions ----------------
int probe_array_update_rssi(struct dawn_mac bssid_addr, struct dawn_mac client_addr, uint32_t rssi, int send_network);
int probe_array_update_rcpi_rsni(struct dawn_mac bssid_addr, struct dawn_mac client_addr, uint32_t rcpi, uint32_t rsni, int send_network);
void remove_old_client_entries(time_t current_time, long long int threshold);
void insert_client_to_array(client entry);
void insert_client_to_array(client *entry);
int kick_clients(uint8_t bssid[], uint32_t id);
int kick_clients(ap* kicking_ap, uint32_t id);
void update_iw_info(uint8_t bssid[]);
void update_iw_info(struct dawn_mac bssid);
void client_array_insert(client entry);
void client_array_insert(client *entry, client ** insert_pos);
client client_array_get_client(const uint8_t* client_addr);
client *client_array_get_client(const struct dawn_mac client_addr);
client client_array_delete(client entry);
client *client_array_delete(client *entry, int unlink_only);
void print_client_array();
void print_client_entry(client entry);
void print_client_entry(client *entry);
int is_connected_somehwere(uint8_t client_addr[]);
int is_connected_somehwere(struct dawn_mac client_addr);
ap insert_to_ap_array(ap entry);
ap *insert_to_ap_array(ap *entry);
void remove_old_ap_entries(time_t current_time, long long int threshold);
void print_ap_array();
ap ap_array_get_ap(uint8_t bssid_addr[]);
ap *ap_array_get_ap(struct dawn_mac bssid_mac);
int probe_array_set_all_probe_count(uint8_t client_addr[], uint32_t probe_count);
int probe_array_set_all_probe_count(struct dawn_mac client_addr, uint32_t probe_count);
#ifndef DAWN_NO_OUTPUT
int ap_get_collision_count(int col_domain);
#endif
void send_beacon_reports(uint8_t bssid[], int id);
void send_beacon_reports(struct dawn_mac bssid, int id);
/* Utils */
// deprecate use of this - it makes things slow
#define SORT_LENGTH 5
extern char sort_string[];
// ---------------- Functions -------------------
int better_ap_available(uint8_t bssid_addr[], uint8_t client_addr[], char* neighbor_report, int automatic_kick);
int better_ap_available(ap *kicking_ap, struct dawn_mac client_addr, char* neighbor_report);
// All users of datastorage should call init_ / destroy_mutex at initialisation and termination respectively
int init_mutex();

View file

@ -4,6 +4,8 @@
#include <stddef.h>
#include <stdint.h>
#include "mac_utils.h"
// ---------------- Global variables ----------------
#define HOSTAPD_DIR_LEN 200
extern char hostapd_dir_glob[];
@ -14,7 +16,7 @@ extern char hostapd_dir_glob[];
* @param client_addr - mac adress of the client
* @return The RSSI of the client if successful. INT_MIN if client was not found.
*/
int get_rssi_iwinfo(uint8_t *client_addr);
int get_rssi_iwinfo(struct dawn_mac client_addr);
/**
* Get expected throughut using the mac adress of the client.
@ -25,7 +27,7 @@ int get_rssi_iwinfo(uint8_t *client_addr);
* + INT_MIN if client was not found.
* + 0 if the client is not supporting this feature.
*/
int get_expected_throughput_iwinfo(uint8_t *client_addr);
int get_expected_throughput_iwinfo(struct dawn_mac client_addr);
/**
* Get rx and tx bandwidth using the mac of the client.
@ -35,7 +37,7 @@ int get_expected_throughput_iwinfo(uint8_t *client_addr);
* @param tx_rate - float pointer for returning the tx rate
* @return 0 if successful 1 otherwise.
*/
int get_bandwidth_iwinfo(uint8_t *client_addr, float *rx_rate, float *tx_rate);
int get_bandwidth_iwinfo(struct dawn_mac client_addr, float *rx_rate, float *tx_rate);
/**
* Function checks if two bssid adresses have the same essid.
@ -44,7 +46,7 @@ int get_bandwidth_iwinfo(uint8_t *client_addr, float *rx_rate, float *tx_rate);
* @param bssid_addr_to_compares
* @return 1 if the bssid adresses have the same essid.
*/
int compare_essid_iwinfo(uint8_t *bssid_addr, uint8_t *bssid_addr_to_compare);
int compare_essid_iwinfo(struct dawn_mac bssid_addr, struct dawn_mac bssid_addr_to_compare);
/**
* Function returns the expected throughput using the interface and the client address.
@ -55,9 +57,9 @@ int compare_essid_iwinfo(uint8_t *bssid_addr, uint8_t *bssid_addr_to_compare);
* + INT_MIN if client was not found.
* + 0 if the client is not supporting this feature.
*/
int get_expected_throughput(const char *ifname, uint8_t *client_addr);
int get_expected_throughput(const char *ifname, struct dawn_mac client_addr);
int get_bssid(const char *ifname, uint8_t *bssid_addr);
int get_bssid(const char *ifname, uint8_t bssid_addr[]);
int get_ssid(const char *ifname, char *ssid, size_t ssidmax);

View file

@ -2,6 +2,7 @@
#define __DAWN_MAC_UTILS_H
#include <stdint.h>
#include <string.h>
#define MAC2STR(a) (a)[0], (a)[1], (a)[2], (a)[3], (a)[4], (a)[5]
#define STR2MAC(a) &(a)[0], &(a)[1], &(a)[2], &(a)[3], &(a)[4], &(a)[5]
@ -13,6 +14,22 @@
#define ETH_ALEN 6
#endif
// Simplify some handling of MAC addresses
struct __attribute__((__packed__)) dawn_mac
{
uint8_t u8[ETH_ALEN];
};
// Compare a raw MAC address to 00:00:00:00:00:00
#define mac_is_null(a1) ((a1)[0] == 0) && ((a1)[1] == 0) && ((a1)[2] == 0) && ((a1)[3] == 0) && ((a1)[4] == 0) && ((a1)[5] == 0)
// For byte arrays outside MAC structure
#define mac_is_equal(addr1, addr2) (memcmp(addr1, addr2, ETH_ALEN) == 0)
// For byte arrays inside MAC structure
#define mac_compare_bb(addr1, addr2) memcmp((addr1).u8, (addr2).u8, ETH_ALEN)
#define mac_is_equal_bb(addr1, addr2) (memcmp((addr1).u8, (addr2).u8, ETH_ALEN) == 0)
/**
* Convert mac adress string to mac adress.
* @param txt
@ -21,23 +38,11 @@
*/
int hwaddr_aton(const char* txt, uint8_t* addr);
/**
* Convert mac to use big characters.
* @param in
* @param out
* @return
*/
int convert_mac(char* in, char* out);
/**
* Write mac to a file.
* @param path
* @param addr
*/
void write_mac_to_file(char* path, uint8_t addr[]);
int mac_is_equal(const uint8_t addr1[], const uint8_t addr2[]);
int mac_is_greater(const uint8_t addr1[], const uint8_t addr2[]);
void write_mac_to_file(char* path, struct dawn_mac addr);
#endif

View file

@ -11,7 +11,7 @@
* @param prob_req
* @return
*/
int parse_to_probe_req(struct blob_attr* msg, probe_entry* prob_req);
probe_entry *parse_to_probe_req(struct blob_attr* msg);
/**
* Dump a client array into the database.

View file

@ -7,8 +7,10 @@
** Contains declerations, etc needed across datastorage and its test harness,
** but not more widely.
*/
void ap_array_insert(ap entry);
void ap_array_insert(ap *entry);
ap ap_array_delete(ap entry);
int ap_array_delete(ap *entry);
auth_entry** auth_entry_find_first_entry(struct dawn_mac bssid_mac, struct dawn_mac client_mac);
#endif

View file

@ -60,7 +60,7 @@ int parse_to_assoc_req(struct blob_attr *msg, assoc_entry *assoc_req);
* @param deauth - if the client should be deauthenticated.
* @param ban_time - the ban time the client is not allowed to connect again.
*/
void del_client_all_interfaces(const uint8_t *client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
void del_client_all_interfaces(const struct dawn_mac client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
/**
* Update the hostapd sockets.
@ -73,9 +73,9 @@ void update_hostapd_sockets(struct uloop_timeout *t);
* @param client_addr
* @return
*/
int send_add_mac(uint8_t *client_addr);
int send_add_mac(struct dawn_mac client_addr);
void ubus_send_beacon_report(uint8_t client[], int id);
void ubus_send_beacon_report(struct dawn_mac client, int id);
void uloop_add_data_cbs();
@ -85,7 +85,7 @@ int build_hearing_map_sort_client(struct blob_buf* b);
int build_network_overview(struct blob_buf* b);
int ap_get_nr(struct blob_buf* b, uint8_t own_bssid_addr[]);
int ap_get_nr(struct blob_buf* b, struct dawn_mac own_bssid_addr);
int parse_add_mac_to_file(struct blob_attr* msg);
@ -96,7 +96,7 @@ int handle_auth_req(struct blob_attr* msg);
* @param probe_entry
* @return
*/
int ubus_send_probe_via_network(struct probe_entry_s probe_entry);
int ubus_send_probe_via_network(struct probe_entry_s *probe_entry);
/**
* Add mac to a list that contains addresses of clients that can not be controlled.
@ -104,7 +104,7 @@ int ubus_send_probe_via_network(struct probe_entry_s probe_entry);
* @param name
* @param addr
*/
void blobmsg_add_macaddr(struct blob_buf *buf, const char *name, const uint8_t *addr);
void blobmsg_add_macaddr(struct blob_buf *buf, const char *name, const struct dawn_mac addr);
/**
* Send message via network.
@ -128,7 +128,7 @@ void add_client_update_timer(time_t time);
* @param deauth - if the client should be deauthenticated.
* @param ban_time - the ban time the client is not allowed to connect again.
*/
void del_client_interface(uint32_t id, const uint8_t* client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
void del_client_interface(uint32_t id, const struct dawn_mac client_addr, uint32_t reason, uint8_t deauth, uint32_t ban_time);
/**
* Function to set the probe counter to the min probe request.
@ -136,7 +136,7 @@ void del_client_interface(uint32_t id, const uint8_t* client_addr, uint32_t reas
* @param client_addr
* @return
*/
int send_set_probe(uint8_t client_addr[]);
int send_set_probe(struct dawn_mac client_addr);
/**
* Function to tell a client that it is about to be disconnected.
@ -146,6 +146,6 @@ int send_set_probe(uint8_t client_addr[]);
* @param duration
* @return - 0 = asynchronous (client has been told to remove itself, and caller should manage arrays); 1 = synchronous (caller should assume arrays are updated)
*/
int wnm_disassoc_imminent(uint32_t id, const uint8_t* client_addr, char* dest_ap, uint32_t duration);
int wnm_disassoc_imminent(uint32_t id, const struct dawn_mac client_addr, char* dest_ap, uint32_t duration);
#endif

View file

@ -3,19 +3,12 @@
#include <stdint.h>
/**
* Convert char to binary.
* @param ch
* @return
*/
int hex_to_dec(char ch);
/**
* Check if a string is greater than another one.
* @param str
* @param str_2
* @return
*/
int string_is_greater(uint8_t *str, uint8_t *str_2);
int string_is_greater(char *str, char *str_2);
#endif