diff --git a/luci-app-omr-bypass/Makefile b/luci-app-omr-bypass/Makefile index 0c92a8bf8..bca6d24e6 100644 --- a/luci-app-omr-bypass/Makefile +++ b/luci-app-omr-bypass/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk LUCI_TITLE:=LuCI Interface to bypass domains #LUCI_DEPENDS:=+dnsmasq-full +shadowsocks-libev-ss-rules +(LINUX_5_4||LINUX_5_15||TARGET_x86_64):iptables-mod-ndpi +iptables-mod-extra +(LINUX_5_4||LINUX_5_15||TARGET_x86_64):kmod-ipt-ndpi +iptables -LUCI_DEPENDS:=+dnsmasq-full +shadowsocks-libev-ss-rules +(LINUX_5_4||LINUX_5_15):iptables-mod-ndpi +iptables-mod-extra +(LINUX_5_4||LINUX_5_15):kmod-ipt-ndpi +iptables +LUCI_DEPENDS:=+dnsmasq-full +shadowsocks-libev-ss-rules +(LINUX_5_4||LINUX_5_15):iptables-mod-ndpi +iptables-mod-extra +(LINUX_5_4||LINUX_5_15):kmod-ipt-ndpi +iptables +sqlite3-cli PKG_LICENSE:=GPLv3 diff --git a/luci-app-omr-bypass/htdocs/luci-static/resources/view/services/omr-bypass.js b/luci-app-omr-bypass/htdocs/luci-static/resources/view/services/omr-bypass.js index 59efc4eec..5f6485dde 100644 --- a/luci-app-omr-bypass/htdocs/luci-static/resources/view/services/omr-bypass.js +++ b/luci-app-omr-bypass/htdocs/luci-static/resources/view/services/omr-bypass.js @@ -15,11 +15,15 @@ return L.view.extend({ }), load: function() { - return this.callHostHints(); + return Promise.all([ + L.resolveDefault(fs.stat('/proc/net/xt_ndpi/proto'), null), + this.callHostHints() + ]); }, - render: function(hosts) { - var m, s, o; + render: function(testhosts) { + var m, s, o, hosts; + hosts = testhosts[1]; m = new form.Map('omr-bypass', _('OMR-Bypass'),_('OpenMPTCProuter IP must be used as DNS.')); @@ -215,21 +219,33 @@ return L.view.extend({ o.rmempty = false; o.load = function(section_id) { return Promise.all([ - fs.read_direct('/proc/net/xt_ndpi/proto'), - fs.read_direct('/proc/net/xt_ndpi/host_proto') + L.resolveDefault(fs.read_direct('/proc/net/xt_ndpi/proto'), ''), + L.resolveDefault(fs.read_direct('/proc/net/xt_ndpi/host_proto'), ''), + fs.read_direct('/usr/share/omr-bypass/omr-bypass-proto.lst') ]).then(L.bind(function(filesi) { var proto = filesi[0].split(/\n/), host = filesi[1].split(/\n/), + protofile = filesi[2].split(/\n/), name = []; - for (var i = 0; i < proto.length; i++) { - var m = proto[i].split(/\s+/); - if (m && m[0] != "#id" && m[1] != "disabled") - name.push(m[2]); + if (proto.length > 2) { + for (var i = 0; i < proto.length; i++) { + var m = proto[i].split(/\s+/); + if (m && m[0] != "#id" && m[1] != "disabled") + name.push(m[2]); + } } - for (var i = 0; i < host.length; i++) { - var m = host[i].split(/:/); - if (m && m[0] != "#Proto") - name.push(m[0].toLowerCase()); + if (host.length > 2) { + for (var i = 0; i < host.length; i++) { + var m = host[i].split(/:/); + if (m && m[0] != "#Proto") + name.push(m[0].toLowerCase()); + } + } + if (proto.length == 1 && host.length == 1) { + for (var i = 0; i < protofile.length; i++) { + var m = protofile[i]; + name.push(m); + } } name = Array.from(new Set(name)).sort(function (a, b) { return a.toLowerCase().localeCompare(b.toLowerCase())}).reduce(function(a, b){ if (a.slice(-1)[0] !== b) a.push(b);return a;},[]); for (var i = 0; i < name.length; i++) { @@ -258,9 +274,11 @@ return L.view.extend({ o.default = o.enabled; o.modalonly = true - o = s.option(form.Flag, 'ndpi', _('Enable ndpi')); - o.default = o.enabled; - o.modalonly = true + if (testhosts[0]) { + o = s.option(form.Flag, 'ndpi', _('Enable ndpi')); + o.default = o.enabled; + o.modalonly = true + } return m.render(); } diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 65f308e27..5304b1a06 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -45,6 +45,12 @@ _add_proto() { done } +_add_proto_without_ndpi() { + protoname=$1 + [ -z "$protoname" ] && return + echo "$protoname" >> /usr/share/omr-bypass/omr-bypass-proto.lst +} + _bypass_ip() { local ip=$1 local type=$2 @@ -469,6 +475,111 @@ _bypass_proto() { fi } +_bypass_proto_without_ndpi() { + local proto + local intf + local enabled + config_get proto $1 proto + config_get intf $1 interface + config_get enabled $1 enabled + config_get ndpi $1 ndpi + config_get noipv6 $1 noipv6 + config_get family $1 family + [ "$enabled" = "0" ] && return + [ -z "$noipv6" ] && noipv6="0" + [ -z "$family" ] && family="ipv4ipv6" + intf=$(echo $intf | sed -e 's/\./_/') + [ -n "$intf" ] && [ -z "$(ipset --list | grep omr_dst_bypass_$intf)" ] && return + local intfid="$(uci -q get omr-bypass.$intf.id)" + + [ -z "$intf" ] && intf="all" + [ -z "$proto" ] && return + if [ "$(uci -q get openmptcprouter.settings.ndpi)" == "0" ] || [ "$ndpi" == "0" ]; then + ipset -q flush bypass_$proto > /dev/null 2>&1 + ipset -q --exist restore <<-EOF + create bypass_$proto hash:net hashsize 64 + EOF + ALLIPS=$(sqlite3 omr-bypass.db "select ip from ipproto where proto='"$proto"';" ".exit") + for ip in $ALLIPS; do + ipset -q add bypass_$proto $ip + done + if [ "$intf" = "all" ]; then + if [ "$family" = "ipv4" ] || [ "$family" = "ipv4ipv6" ]; then + $IPTABLESRESTORE -w --wait=60 --noflush <<-EOF + *mangle + -A omr-bypass-dpi -m set --match-set $proto -j MARK --set-mark 0x539 + -A omr-bypass-dpi -m mark --mark 0x539 -j RETURN + COMMIT + EOF + fi + if [ "$disableipv6" = "0" ] && ([ "$family" = "ipv6" ] || [ "$family" = "ipv4ipv6" ]); then + $IP6TABLESRESTORE -w --wait=60 --noflush <<-EOF + *mangle + -A omr-bypass6-dpi -m set --match-set $proto -j MARK --set-mark 0x6539 + -A omr-bypass6-dpi -m mark --mark 0x6539 -j RETURN + COMMIT + EOF + fi + else + if [ "$family" = "ipv4" ] || [ "$family" = "ipv4ipv6" ]; then + $IPTABLESRESTORE -w --wait=60 --noflush <<-EOF + *mangle + -A omr-bypass-dpi -m set --match-set $proto -j MARK --set-mark 0x539$intfid + -A omr-bypass-dpi -m mark --mark 0x539$intfid -j RETURN + COMMIT + EOF + fi + if [ "$disableipv6" = "0" ] && ([ "$family" = "ipv6" ] || [ "$family" = "ipv4ipv6" ]); then + $IP6TABLESRESTORE -w --wait=60 --noflush <<-EOF + *mangle + -A omr-bypass6-dpi -m set --match-set $proto -j MARK --set-mark 0x6539$intfid + -A omr-bypass6-dpi -m mark --mark 0x6539$intfid -j RETURN + COMMIT + EOF + fi + fi + fi + # Use dnsmasq ipset to bypass domains of the proto + local domains + #domains="$(cat /proc/net/xt_ndpi/host_proto | grep -i $proto: | sed -e "s/$proto://i" -e 's/*//' -e 's/,/ /g')" + domains=$(sqlite3 omr-bypass.db "select host from hostproto where proto='"$proto"';" ".exit") + if [ -n "$domains" ]; then + tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` + for domain in $domains; do + if [ -n "$domain" ]; then + domain="$(echo $domain | sed 's/^\.//')" + if [ "$(echo $domain | grep '\.$')" != "" ]; then + domainlist="" + # construct list of domains to query + i=0 + for tld in $tlds; do + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" + done + domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time to` \ + dig a +timeout=1 +tries=1 +retry=1 +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` to \ + | grep $domain `# grep, only keep wanted domain` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + for validdomain in $domainlist; do + _bypass_domain $validdomain $intf $family $noipv6 + done + else + _bypass_domain $domain $intf $family $noipv6 + fi + fi + done + fi +} + _intf_rule_ss_rules() { rule_name=$1 [ "$rule_name" = "ss_rules" ] && rule_name="def" @@ -760,8 +871,10 @@ start_service() { #local count logger -t "omr-bypass" "Starting OMR-ByPass..." add_domains="false" - [ -d /proc/net/xt_ndpi ] && config_load omr-bypass - config_foreach _add_proto proto + [ -d /proc/net/xt_ndpi ] && { + config_load omr-bypass + config_foreach _add_proto proto + } disableipv6="$(uci -q get openmptcprouter.settings.disable_ipv6)" #noipv6="$(uci -q get omr-bypass.global.noipv6)" @@ -877,6 +990,7 @@ start_service() { fi config_load omr-bypass [ -d /proc/net/xt_ndpi/proto ] && config_foreach _bypass_proto dpis + config_foreach _bypass_proto_without_ndpi dpis uci -q commit omr-bypass [ -z "$RELOAD" ] && [ "$add_domains" = "true" ] && { @@ -887,6 +1001,13 @@ start_service() { logger -t "omr-bypass" "Reload dnsmasq..." /etc/init.d/dnsmasq reload } + + # Create a protocol list for UI from a sqlite DB when NDPI is not available + sqlite3 /usr/share/omr-bypass/omr-bypass.db "select distinct(proto) from (select proto from hostproto union all select proto from ipproto) a order by proto;" ".exit" > /usr/share/omr-bypass/omr-bypass-proto.lst + config_load omr-bypass + config_foreach _add_proto_without_ndpi proto + sort -o /usr/share/omr-bypass/omr-bypass-proto.lst /usr/share/omr-bypass/omr-bypass-proto.lst + logger -t "omr-bypass" "OMR-ByPass is running" } diff --git a/luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass-proto.lst b/luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass-proto.lst new file mode 100644 index 000000000..8e24f9282 --- /dev/null +++ b/luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass-proto.lst @@ -0,0 +1,150 @@ +1kxun +accuweather +activision +ads_analytic_track +adultcontent +alibaba +amazon +amazonalexa +amazonaws +amazonvideo +anydesk +apple +appleicloud +appleitunes +applepush +applesiri +applestore +appletvplus +azure +badoo +bittorrent +bloomberg +cachefly +cloudflare +cloudflarewarp +cnn +crashlytics +cybersec +dailymotion +datasaver +dazn +deezer +directv +discord +disneyplus +dnscrypt +doh_dot +dropbox +eaq +ebay +facebook +fbookreelstory +fuze +github +gitlab +gmail +google +googleclassroom +googlecloud +googledocs +googledrive +googlehangout +googlemaps +googleplus +googleservices +goto +hbo +hotspotshield +hulu +icloudprivaterelay +iflix +iheartradio +instagram +kakaotalk +lastfm +likee +line +linkedin +livestream +messenger +microsoft +microsoft365 +ms_onedrive +netflix +nintendo +ntop +ocs +ocsp +ookla +opendns +outlook +pandora +pastebin +pinterest +playstation +playstore +pluralsight +ppstream +psiphon +qq +reddit +riotgames +salesforce +showtime +signal +sina +sina(weibo) +siriusxmradio +skype_teams +slack +snapchat +softether +soundcloud +spotify +steam +syncthing +tailscale +teams +teamviewer +telegram +tencentvideo +tidal +tiktok +tor +tuenti +tumblr +tunein +tunnelbear +twitch +twitter +ubuntuone +vevo +viber +vidto +vimeo +vk +vudu +waze +webex +wechat +whatsapp +whatsappfiles +wikipedia +windowsupdate +worldofwarcraft +xbox +xiaomi +yahoo +yandex +yandexcloud +yandexdirect +yandexdisk +yandexmail +yandexmarket +yandexmetrika +yandexmusic +youtube +youtubeupload +zattoo +zoom \ No newline at end of file diff --git a/luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass.db b/luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass.db new file mode 100644 index 000000000..2c0740b08 Binary files /dev/null and b/luci-app-omr-bypass/root/usr/share/omr-bypass/omr-bypass.db differ diff --git a/luci-app-omr-bypass/root/usr/share/rpcd/acl.d/luci-app-omr-bypass.json b/luci-app-omr-bypass/root/usr/share/rpcd/acl.d/luci-app-omr-bypass.json index 3281f4fca..0c8727a42 100644 --- a/luci-app-omr-bypass/root/usr/share/rpcd/acl.d/luci-app-omr-bypass.json +++ b/luci-app-omr-bypass/root/usr/share/rpcd/acl.d/luci-app-omr-bypass.json @@ -4,7 +4,8 @@ "read": { "file": { "/proc/net/xt_ndpi/proto": [ "read" ], - "/proc/net/xt_ndpi/host_proto": [ "read" ] + "/proc/net/xt_ndpi/host_proto": [ "read" ], + "/usr/share/omr-bypass/omr-bypass-proto.lst": [ "read" ] }, "ubus": { "luci-rpc": [ "getHostHints" ]