omr-bypass: New way to bypass services without nDPI

luci-app-omr-bypass/Makefile

@@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk
 
 LUCI_TITLE:=LuCI Interface to bypass domains
 #LUCI_DEPENDS:=+dnsmasq-full +shadowsocks-libev-ss-rules +(LINUX_5_4||LINUX_5_15||TARGET_x86_64):iptables-mod-ndpi +iptables-mod-extra +(LINUX_5_4||LINUX_5_15||TARGET_x86_64):kmod-ipt-ndpi +iptables
-LUCI_DEPENDS:=+dnsmasq-full +shadowsocks-libev-ss-rules +(LINUX_5_4||LINUX_5_15):iptables-mod-ndpi +iptables-mod-extra +(LINUX_5_4||LINUX_5_15):kmod-ipt-ndpi +iptables
+LUCI_DEPENDS:=+dnsmasq-full +shadowsocks-libev-ss-rules +(LINUX_5_4||LINUX_5_15):iptables-mod-ndpi +iptables-mod-extra +(LINUX_5_4||LINUX_5_15):kmod-ipt-ndpi +iptables +sqlite3-cli
 
 PKG_LICENSE:=GPLv3
 

luci-app-omr-bypass/htdocs/luci-static/resources/view/services/omr-bypass.js

@@ -15,11 +15,15 @@ return L.view.extend({
 	}),
 
 	load: function() {
-		return  this.callHostHints();
+		return Promise.all([
+			L.resolveDefault(fs.stat('/proc/net/xt_ndpi/proto'), null),
+			this.callHostHints()
+		]);
 	},
 
-	render: function(hosts) {
+	render: function(testhosts) {
-		var m, s, o;
+		var m, s, o, hosts;
+		hosts = testhosts[1];
 
 		m = new form.Map('omr-bypass', _('OMR-Bypass'),_('OpenMPTCProuter IP must be used as DNS.'));
 
@@ -215,22 +219,34 @@ return L.view.extend({
 		o.rmempty = false;
 		o.load = function(section_id) {
 			return Promise.all([
-				fs.read_direct('/proc/net/xt_ndpi/proto'),
+				L.resolveDefault(fs.read_direct('/proc/net/xt_ndpi/proto'), ''),
-				fs.read_direct('/proc/net/xt_ndpi/host_proto')
+				L.resolveDefault(fs.read_direct('/proc/net/xt_ndpi/host_proto'), ''),
+				fs.read_direct('/usr/share/omr-bypass/omr-bypass-proto.lst')
 			]).then(L.bind(function(filesi) {
 				var proto = filesi[0].split(/\n/),
 				    host = filesi[1].split(/\n/),
+				    protofile = filesi[2].split(/\n/),
 				    name = [];
+				if (proto.length > 2) {
 					for (var i = 0; i < proto.length; i++) {
 						var m = proto[i].split(/\s+/);
 						if (m && m[0] != "#id" && m[1] != "disabled")
 						    name.push(m[2]);
 					}
+				}
+				if (host.length > 2) {
 					for (var i = 0; i < host.length; i++) {
 						var m = host[i].split(/:/);
 						if (m && m[0] != "#Proto")
 						  name.push(m[0].toLowerCase());
 					}
+				}
+				if (proto.length == 1 && host.length == 1) {
+					for (var i = 0; i < protofile.length; i++) {
+						var m = protofile[i];
+						name.push(m);
+					}
+				}
 				name = Array.from(new Set(name)).sort(function (a, b) { return a.toLowerCase().localeCompare(b.toLowerCase())}).reduce(function(a, b){ if (a.slice(-1)[0] !== b) a.push(b);return a;},[]);
 				for (var i = 0; i < name.length; i++) {
 					this.value(name[i]);
@@ -258,9 +274,11 @@ return L.view.extend({
 		o.default = o.enabled;
 		o.modalonly = true
 
+		if (testhosts[0]) {
 			o = s.option(form.Flag, 'ndpi', _('Enable ndpi'));
 			o.default = o.enabled;
 			o.modalonly = true
+		}
 
 		return m.render();
 	}

luci-app-omr-bypass/root/etc/init.d/omr-bypass

@@ -45,6 +45,12 @@ _add_proto() {
 	done
 }
 
+_add_proto_without_ndpi() {
+	protoname=$1
+	[ -z "$protoname" ] && return
+	echo "$protoname" >> /usr/share/omr-bypass/omr-bypass-proto.lst
+}
+
 _bypass_ip() {
 	local ip=$1
 	local type=$2
@@ -469,6 +475,111 @@ _bypass_proto() {
 	fi
 }
 
+_bypass_proto_without_ndpi() {
+	local proto
+	local intf
+	local enabled
+	config_get proto $1 proto
+	config_get intf $1 interface
+	config_get enabled $1 enabled
+	config_get ndpi $1 ndpi
+	config_get noipv6 $1 noipv6
+	config_get family $1 family
+	[ "$enabled" = "0" ] && return
+	[ -z "$noipv6" ] && noipv6="0"
+	[ -z "$family" ] && family="ipv4ipv6"
+	intf=$(echo $intf | sed -e 's/\./_/')
+	[ -n "$intf" ] && [ -z "$(ipset --list | grep omr_dst_bypass_$intf)" ] && return
+	local intfid="$(uci -q get omr-bypass.$intf.id)"
+
+	[ -z "$intf" ] && intf="all"
+	[ -z "$proto" ] && return
+	if [ "$(uci -q get openmptcprouter.settings.ndpi)" == "0" ] || [ "$ndpi" == "0" ]; then
+		ipset -q flush bypass_$proto > /dev/null 2>&1
+		ipset -q --exist restore <<-EOF
+		create bypass_$proto hash:net hashsize 64
+		EOF
+		ALLIPS=$(sqlite3 omr-bypass.db "select ip from ipproto where proto='"$proto"';" ".exit")
+		for ip in $ALLIPS; do
+			ipset -q add bypass_$proto $ip
+		done
+		if [ "$intf" = "all" ]; then
+			if [ "$family" = "ipv4" ] || [ "$family" = "ipv4ipv6" ]; then
+				$IPTABLESRESTORE -w --wait=60  --noflush <<-EOF
+				*mangle
+				-A omr-bypass-dpi -m set --match-set $proto -j MARK --set-mark 0x539
+				-A omr-bypass-dpi -m mark --mark 0x539 -j RETURN
+				COMMIT
+				EOF
+			fi
+			if [ "$disableipv6" = "0" ] && ([ "$family" = "ipv6" ] || [ "$family" = "ipv4ipv6" ]); then
+				$IP6TABLESRESTORE -w --wait=60  --noflush <<-EOF
+				*mangle
+				-A omr-bypass6-dpi -m set --match-set $proto -j MARK --set-mark 0x6539
+				-A omr-bypass6-dpi -m mark --mark 0x6539 -j RETURN
+				COMMIT
+				EOF
+			fi
+		else
+			if [ "$family" = "ipv4" ] || [ "$family" = "ipv4ipv6" ]; then
+				$IPTABLESRESTORE -w --wait=60  --noflush <<-EOF
+				*mangle
+				-A omr-bypass-dpi -m set --match-set $proto -j MARK --set-mark 0x539$intfid
+				-A omr-bypass-dpi -m mark --mark 0x539$intfid -j RETURN
+				COMMIT
+				EOF
+			fi
+			if [ "$disableipv6" = "0" ] && ([ "$family" = "ipv6" ] || [ "$family" = "ipv4ipv6" ]); then
+				$IP6TABLESRESTORE -w --wait=60  --noflush <<-EOF
+				*mangle
+				-A omr-bypass6-dpi -m set --match-set $proto -j MARK --set-mark 0x6539$intfid
+				-A omr-bypass6-dpi -m mark --mark 0x6539$intfid -j RETURN
+				COMMIT
+				EOF
+			fi
+		fi
+	fi
+	# Use dnsmasq ipset to bypass domains of the proto
+	local domains
+	#domains="$(cat /proc/net/xt_ndpi/host_proto | grep -i $proto: | sed -e "s/$proto://i" -e 's/*//' -e 's/,/ /g')"
+	domains=$(sqlite3 omr-bypass.db "select host from hostproto where proto='"$proto"';" ".exit")
+	if [ -n "$domains" ]; then
+		tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt`
+		for domain in $domains; do
+			if [ -n "$domain" ]; then
+				domain="$(echo $domain | sed 's/^\.//')"
+				if [ "$(echo $domain | grep '\.$')" != "" ]; then
+					domainlist=""
+					# construct list of domains to query
+					i=0
+					for tld in $tlds; do
+						i=$((i+1))
+						# trim off header
+						if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then
+							continue
+						fi 
+						# add to command
+						domainlist="${domainlist} ${domain}${tld}"
+					done
+					domainlist="$(echo $domainlist  `# Get the list of valid domains, pass it to awk` \
+						| awk '{print tolower($0)}'   `# awk lowercases the whole string and passes it to ` \
+						| xargs -n8 -P12              `# xargs sends 8 arguments at a time to` \
+							dig a +timeout=1 +tries=1 +retry=1 +nocmd +noall +answer `# dig, which passes results (if any) to` \
+						| awk '{print $1}'            `# awk, which outputs queried domain to` \
+						| sed -e 's/.$//'             `# sed, which trims off the trailing dot (google.com. -> google.com)` to \
+						| grep $domain                `# grep, only keep wanted domain` \
+						| awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate
+					for validdomain in $domainlist; do
+						_bypass_domain $validdomain $intf $family $noipv6
+					done
+				else
+					_bypass_domain $domain $intf $family $noipv6
+				fi
+			fi
+		done
+	fi
+}
+
 _intf_rule_ss_rules() {
 	rule_name=$1
 	[ "$rule_name" = "ss_rules" ] && rule_name="def"
@@ -760,8 +871,10 @@ start_service() {
 	#local count
 	logger -t "omr-bypass" "Starting OMR-ByPass..."
 	add_domains="false"
-	[ -d /proc/net/xt_ndpi ] && config_load omr-bypass
+	[ -d /proc/net/xt_ndpi ] && {
+		config_load omr-bypass
 		config_foreach _add_proto proto
+	}
 	disableipv6="$(uci -q get openmptcprouter.settings.disable_ipv6)"
 	#noipv6="$(uci -q get omr-bypass.global.noipv6)"
 
@@ -877,6 +990,7 @@ start_service() {
 	fi
 	config_load omr-bypass
 	[ -d /proc/net/xt_ndpi/proto ] && config_foreach _bypass_proto dpis
+	config_foreach _bypass_proto_without_ndpi dpis
 	uci -q commit omr-bypass
 
 	[ -z "$RELOAD" ] && [ "$add_domains" = "true" ] && {
@@ -887,6 +1001,13 @@ start_service() {
 		logger -t "omr-bypass" "Reload dnsmasq..."
 		/etc/init.d/dnsmasq reload
 	}
+
+	# Create a protocol list for UI from a sqlite DB when NDPI is not available
+	sqlite3 /usr/share/omr-bypass/omr-bypass.db "select distinct(proto) from (select proto from hostproto union all select proto from ipproto) a order by proto;" ".exit" > /usr/share/omr-bypass/omr-bypass-proto.lst
+	config_load omr-bypass
+	config_foreach _add_proto_without_ndpi proto
+	sort -o /usr/share/omr-bypass/omr-bypass-proto.lst /usr/share/omr-bypass/omr-bypass-proto.lst
+	
 	logger -t "omr-bypass" "OMR-ByPass is running"
 }
 

luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass-proto.lst

@@ -0,0 +1,150 @@
+1kxun
+accuweather
+activision
+ads_analytic_track
+adultcontent
+alibaba
+amazon
+amazonalexa
+amazonaws
+amazonvideo
+anydesk
+apple
+appleicloud
+appleitunes
+applepush
+applesiri
+applestore
+appletvplus
+azure
+badoo
+bittorrent
+bloomberg
+cachefly
+cloudflare
+cloudflarewarp
+cnn
+crashlytics
+cybersec
+dailymotion
+datasaver
+dazn
+deezer
+directv
+discord
+disneyplus
+dnscrypt
+doh_dot
+dropbox
+eaq
+ebay
+facebook
+fbookreelstory
+fuze
+github
+gitlab
+gmail
+google
+googleclassroom
+googlecloud
+googledocs
+googledrive
+googlehangout
+googlemaps
+googleplus
+googleservices
+goto
+hbo
+hotspotshield
+hulu
+icloudprivaterelay
+iflix
+iheartradio
+instagram
+kakaotalk
+lastfm
+likee
+line
+linkedin
+livestream
+messenger
+microsoft
+microsoft365
+ms_onedrive
+netflix
+nintendo
+ntop
+ocs
+ocsp
+ookla
+opendns
+outlook
+pandora
+pastebin
+pinterest
+playstation
+playstore
+pluralsight
+ppstream
+psiphon
+qq
+reddit
+riotgames
+salesforce
+showtime
+signal
+sina
+sina(weibo)
+siriusxmradio
+skype_teams
+slack
+snapchat
+softether
+soundcloud
+spotify
+steam
+syncthing
+tailscale
+teams
+teamviewer
+telegram
+tencentvideo
+tidal
+tiktok
+tor
+tuenti
+tumblr
+tunein
+tunnelbear
+twitch
+twitter
+ubuntuone
+vevo
+viber
+vidto
+vimeo
+vk
+vudu
+waze
+webex
+wechat
+whatsapp
+whatsappfiles
+wikipedia
+windowsupdate
+worldofwarcraft
+xbox
+xiaomi
+yahoo
+yandex
+yandexcloud
+yandexdirect
+yandexdisk
+yandexmail
+yandexmarket
+yandexmetrika
+yandexmusic
+youtube
+youtubeupload
+zattoo
+zoom

luci-app-omr-bypass/root/usr/share/rpcd/acl.d/luci-app-omr-bypass.json

@@ -4,7 +4,8 @@
 	"read": {
 	    "file": {
 		"/proc/net/xt_ndpi/proto": [ "read" ],
-		"/proc/net/xt_ndpi/host_proto": [ "read" ]
+		"/proc/net/xt_ndpi/host_proto": [ "read" ],
+		"/usr/share/omr-bypass/omr-bypass-proto.lst": [ "read" ]
 	    },
 	    "ubus": {
 		"luci-rpc": [ "getHostHints" ]