Add server tracker for server failover

luci-app-omr-tracker/luasrc/model/cbi/omr-tracker.lua

@@ -41,6 +41,37 @@ o.default     = { "bing.com", "google.com" }
 o.rmempty     = false
 
 
+s = m:section(TypedSection, "server", translate("Server tracker Settings"), translate("Detect if Server is down and use defined backup server in this case."))
+s.anonymous   = true
+s.addremove = false
+
+local sdata = m:get('server')
+if not sdata then
+	m:set('server', nil, 'server')
+	m:set('server', 'enabled', "1")
+end
+
+o = s:option(Flag, "enabled", translate("Enable"), translate("When tracker is disabled, server failover is also disabled"))
+o.rmempty     = false
+
+o = s:option(Value, "timeout", translate("Timeout (s)"))
+o.placeholder = "1"
+o.default     = "1"
+o.datatype    = "range(1, 100)"
+o.rmempty     = false
+
+o = s:option(Value, "tries", translate("Tries"))
+o.placeholder = "4"
+o.default     = "4"
+o.datatype    = "range(1, 10)"
+o.rmempty     = false
+
+o = s:option(Value, "interval", translate("Retry interval (s)"))
+o.placeholder = "2"
+o.default     = "2"
+o.datatype    = "range(1, 100)"
+o.rmempty     = false
+
 s = m:section(TypedSection, "defaults", translate("Defaults Settings"), translate("OMR-Tracker detect when a connection is down and execute needed scripts"))
 s.anonymous   = true
 

luci-app-openmptcprouter/luasrc/controller/openmptcprouter.lua

@@ -167,6 +167,8 @@ function wizard_add()
 		ucic:set("sqm","wan" .. i,"verbosity","5")
 		ucic:set("sqm","wan" .. i,"download","0")
 		ucic:set("sqm","wan" .. i,"upload","0")
+		ucic:set("sqm","wan" .. i,"iqdisc_opts","autorate-ingress nat dual-dsthost")
+		ucic:set("sqm","wan" .. i,"eqdisc_opts","nat dual-srchost")
 		ucic:save("sqm")
 		ucic:commit("sqm")
 		
@@ -263,6 +265,8 @@ function wizard_add()
 			ucic:set("sqm",intf,"verbosity","5")
 			ucic:set("sqm",intf,"download","0")
 			ucic:set("sqm",intf,"upload","0")
+			ucic:set("sqm",intf,"iqdisc_opts","autorate-ingress nat dual-dsthost")
+			ucic:set("sqm",intf,"eqdisc_opts","nat dual-srchost")
 		end
 
 		if downloadspeed ~= "0" and uploadspeed ~= "0" and downloadspeed ~= "" and uploadspeed ~= "" then
@@ -281,6 +285,8 @@ function wizard_add()
 			ucic:set("qos",intf,"enabled","0")
 		end
 		if sqmenabled == "1" then
+			ucic:set("sqm",intf,"iqdisc_opts","autorate-ingress nat dual-dsthost")
+			ucic:set("sqm",intf,"eqdisc_opts","nat dual-srchost")
 			ucic:set("sqm",intf,"enabled","1")
 			ucic:set("qos",intf,"enabled","1")
 		else
@@ -395,45 +401,35 @@ function wizard_add()
 			if serversnb > 1 then
 				if master == server then
 					ss_ip=server_ip
-					table.insert(ss_servers_nginx,server_ip .. ":65101 max_fails=2 fail_timeout=20s")
+					ucic:set("shadowsocks-libev","sss0","server",server_ip)
-					table.insert(ss_servers_ha,server_ip .. ":65101 check")
+					ucic:set("glorytun","vpn","host",server_ip)
-					if vpn_port ~= "" then
+					ucic:set("dsvpn","vpn","host",server_ip)
-						table.insert(vpn_servers,server_ip .. ":" .. vpn_port .. " max_fails=2 fail_timeout=20s")
+					ucic:set("mlvpn","general","host",server_ip)
-					end
+					ucic:set("ubond","general","host",server_ip)
-				else
+					luci.sys.call("uci -q del openvpn.omr.remote")
-					table.insert(ss_servers_nginx,server_ip .. ":65101 backup")
+					luci.sys.call("uci -q add_list openvpn.omr.remote=" .. server_ip)
-					table.insert(ss_servers_ha,server_ip .. ":65101 backup")
+					ucic:set("qos","serverin","srchost",server_ip)
-					if vpn_port ~= "" then
+					ucic:set("qos","serverout","dsthost",server_ip)
-						table.insert(vpn_servers,server_ip .. ":" .. vpn_port .. " backup")
-					end
 				end
 				k = k + 1
-				ucic:set("nginx-ha","ShadowSocks","enable","1")
+				ucic:set("nginx-ha","ShadowSocks","enable","0")
-				ucic:set("nginx-ha","VPN","enable","1")
+				ucic:set("nginx-ha","VPN","enable","0")
-				ucic:set("nginx-ha","ShadowSocks","upstreams",ss_servers_nginx)
-				ucic:set("nginx-ha","VPN","upstreams",vpn_servers)
 				ucic:set("haproxy-tcp","general","enable","0")
-				ucic:set("haproxy-tcp","general","upstreams",ss_servers_ha)
 				ucic:set("openmptcprouter","settings","ha","1")
-				server_ip = "127.0.0.1"
-				--ucic:set("shadowsocks-libev","sss0","server",ss_ip)
 			else
 				ucic:set("openmptcprouter","settings","ha","0")
 				ucic:set("nginx-ha","ShadowSocks","enable","0")
 				ucic:set("nginx-ha","VPN","enable","0")
-				--ucic:set("shadowsocks-libev","sss0","server",server_ip)
+				ucic:set("shadowsocks-libev","sss0","server",server_ip)
-				--ucic:set("openmptcprouter","vps","ip",server_ip)
+				ucic:set("glorytun","vpn","host",server_ip)
-				--ucic:save("openmptcprouter")
+				ucic:set("dsvpn","vpn","host",server_ip)
+				ucic:set("mlvpn","general","host",server_ip)
+				ucic:set("ubond","general","host",server_ip)
+				luci.sys.call("uci -q del openvpn.omr.remote")
+				luci.sys.call("uci -q add_list openvpn.omr.remote=" .. server_ip)
+				ucic:set("qos","serverin","srchost",server_ip)
+				ucic:set("qos","serverout","dsthost",server_ip)
 			end
-			ucic:set("shadowsocks-libev","sss0","server",server_ip)
-			ucic:set("glorytun","vpn","host",server_ip)
-			ucic:set("dsvpn","vpn","host",server_ip)
-			ucic:set("mlvpn","general","host",server_ip)
-			ucic:set("ubond","general","host",server_ip)
-			luci.sys.call("uci -q del openvpn.omr.remote")
-			luci.sys.call("uci -q add_list openvpn.omr.remote=" .. server_ip)
-			ucic:set("qos","serverin","srchost",server_ip)
-			ucic:set("qos","serverout","dsthost",server_ip)
 		end
 	end
 

omr-tracker/files/bin/omr-tracker-server

@@ -0,0 +1,125 @@
+#!/bin/sh
+# vim: set noexpandtab tabstop=4 shiftwidth=4 softtabstop=4 :
+
+name=$0
+basename="$(basename $0)"
+
+_log() {
+	logger -p daemon.info -t "${basename}" "$@"
+}
+
+_ping_server() {
+	local host=$1
+	ret=$(ping \
+	    -w "$OMR_TRACKER_TIMEOUT" \
+	    -c 1 \
+	    -q \
+	    "${host}"
+	) && echo "$ret" | grep -sq " 0% packet loss" && {
+		server_ping=true
+	}
+}
+
+_check_server() {
+	local host=$1
+	local port=$2
+	local k=0
+	while [ "$server_ping" = false ] && [ "$k" -le "$retry" ]; do
+		ret=$(curl -4 \
+		    --max-time "$OMR_TRACKER_TIMEOUT" \
+		    -s \
+		    -k \
+		    "https://${host}:${port}/"
+		)
+		[ -n "$ret" ] && server_ping=true
+		k=$((k+1))
+		sleep "${intervaltries}"
+	done
+}
+
+_check_master() {
+	local name=$1
+	config_get master $1 master
+	config_get ip $1 ip
+	config_get port $1 port "65500"
+	[ "$master" = "1" ] && [ -n "$ip" ] && {
+		#_ping_server $ip
+		_check_server $ip $port
+		[ "$server_ping" = true ] && [ "$(uci -q get shadowsocks-libev.sss0.server | tr -d '\n')" != "$ip" ] && {
+			logger -t "OMR-Tracker-Server" "Master server up, set it back"
+			logger -t "OMR-Tracker-Server" "$(uci -q get shadowsocks-libev.sss0.server | tr -d '\n') - $ip"
+			uci -q batch <<-EOF >/dev/null
+				set shadowsocks-libev.sss0.server=$ip
+				commit shadowsocks-libev
+				set glorytun.vpn.host=$ip
+				commit glorytun
+				set dsvpn.vpn.host=$ip
+				commit dsvpn
+				set mlvpn.general.host=$ip
+				commit mlvpn
+				del openvpn.omr.remote
+				add_list openvpn.omr.remote=$ip
+				commit openvpn
+			EOF
+			/etc/init.d/shadowsocks-libev restart >/dev/null 2>/dev/null
+			/etc/init.d/glorytun restart >/dev/null 2>/dev/null
+			/etc/init.d/glorytun-udp restart >/dev/null 2>/dev/null
+			/etc/init.d/mlvpn restart >/dev/null 2>/dev/null
+			/etc/init.d/openvpn restart >/dev/null 2>/dev/null
+			/etc/init.d/dsvpn restart >/dev/null 2>/dev/null
+		}
+		break
+	}
+}
+
+_check_backup() {
+	local name=$1
+	config_get backup $1 backup
+	config_get ip $1 ip
+	config_get port $1 port
+	[ "$backup" = "1" ] && [ -n "$ip" ] && {
+		#_ping_server $ip
+		_check_server $ip $port
+	}
+	[ "$server_ping" = true ] && [ "$(uci -q get shadowsocks-libev.sss0.server | tr -d '\n')" = "$ip" ] && break
+	[ "$server_ping" = true ] && [ "$(uci -q get shadowsocks-libev.sss0.server | tr -d '\n')" != "$ip" ] && {
+		logger -t "OMR-Tracker-Server" "User backup server $1 ($ip)"
+		uci -q batch <<-EOF >/dev/null
+			set shadowsocks-libev.sss0.server=$ip
+			commit shadowsocks-libev
+			set glorytun.vpn.host=$ip
+			commit glorytun
+			set dsvpn.vpn.host=$ip
+			commit dsvpn
+			set mlvpn.general.host=$ip
+			commit mlvpn
+			del openvpn.omr.remote
+			add_list openvpn.omr.remote=$ip
+			commit openvpn
+		EOF
+		/etc/init.d/shadowsocks-libev restart >/dev/null 2>/dev/null
+		/etc/init.d/glorytun restart >/dev/null 2>/dev/null
+		/etc/init.d/glorytun-udp restart >/dev/null 2>/dev/null
+		/etc/init.d/mlvpn restart >/dev/null 2>/dev/null
+		/etc/init.d/openvpn restart >/dev/null 2>/dev/null
+		/etc/init.d/dsvpn restart >/dev/null 2>/dev/null
+		break
+	}
+}
+
+. /lib/functions.sh
+
+timeout=${OMR_TRACKER_TIMEOUT:-5}
+interval=${OMR_TRACKER_INTERVAL:-10}
+intervaltries=${OMR_TRACKER_INTERVAL_TRIES:-2}
+retry=${OMR_TRACKER_TRIES:-4}
+
+while true; do
+	server_ping=false
+	config_load openmptcprouter
+	config_foreach _check_master server
+	[ "$server_ping" = false ] && {
+		config_foreach _check_backup server
+	}
+	sleep "${interval}"
+done

omr-tracker/files/etc/init.d/omr-tracker

@@ -77,6 +77,26 @@ _launch_tracker() {
 	procd_close_instance
 }
 
+_launch_server_tracker() {
+	local hosts timeout tries interval interval_tries options type enabled
+	_validate_section "server" "server"
+
+	[ "${enabled}" = "0" ] && return
+	[ -z "${interval_tries}" ] && interval_tries=1
+
+	procd_open_instance
+	# shellcheck disable=SC2086
+	procd_set_param command /bin/omr-tracker-server "$1" $options
+	procd_append_param env "OMR_TRACKER_TIMEOUT=$timeout"
+	procd_append_param env "OMR_TRACKER_TRIES=$tries"
+	procd_append_param env "OMR_TRACKER_INTERVAL=$interval"
+	procd_append_param env "OMR_TRACKER_INTERVAL_TRIES=$interval_tries"
+	procd_set_param limits nofile="51200 51200"
+	procd_set_param respawn 0 10 0
+	procd_set_param stderr 1
+	procd_close_instance
+}
+
 _initialize_shadowsocks_tracker() {
 	local redir_tcp server tracker_server
 	config_get redir_tcp ss_rules redir_tcp
@@ -146,6 +166,11 @@ _launch_shadowsocks_tracker() {
 	procd_close_instance
 }
 
+_multi_server() {
+	config_get backup $1 backup
+	[ "$backup" = "1" ] && multiserver=true
+}
+
 start_service() {
 	local ss_disabled
 	logger -t "omr-tracker" "Launching..."
@@ -161,6 +186,11 @@ start_service() {
 	else
 		/etc/init.d/shadowsocks-libev rules_down
 	fi
+	
+	multiserver=false
+	config_load openmptcprouter
+	config_foreach _multi_server server
+	[ "$multiserver" = true ] && _launch_server_tracker
 	logger -t "omr-tracker" "Launched"
 }