From fcddc353579554d31937b951321eb51907794380 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:54:52 +0000 Subject: [PATCH 1/6] parallel digs --- .../root/etc/init.d/omr-bypass | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index b5bd902a0..1682c565a 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -331,21 +331,32 @@ _bypass_proto() { local domains domains="$(cat /proc/net/xt_ndpi/host_proto | grep -i $proto: | sed -e "s/$proto://i" -e 's/*//' -e 's/,/ /g')" if [ -n "$domains" ]; then + tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` for domain in $domains; do if [ -n "$domain" ]; then domain="$(echo $domain | sed 's/^\.//')" if [ "$(echo $domain | grep '\.$')" != "" ]; then - tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` - if [ -n "tlds" ]; then - i=0 - for tld in $tlds; do - i=$((i+1)) - tld="$(echo $tld | awk '{print tolower($0)}')" - if [ "$i" -gt "11" ] && [ "$(dig a +timeout=1 +tries=1 +retry=1 +nocmd +noall +answer ${domain}${tld})" != "" ]; then - _bypass_domain ${domain}${tld} $intf - fi - done - fi + domainlist="" + # construct list of domains to query + for tld in $tlds; do + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" + done + domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + for validdomain in $domainlist; do + _bypass_domain $validdomain $intf + done else _bypass_domain $domain $intf fi From eb10bc4ecb99e1f52a298a93eef8f3b302a58d18 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:56:59 +0000 Subject: [PATCH 2/6] add CLA --- contributors/cr3ative.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 contributors/cr3ative.md diff --git a/contributors/cr3ative.md b/contributors/cr3ative.md new file mode 100644 index 000000000..b0c58c54e --- /dev/null +++ b/contributors/cr3ative.md @@ -0,0 +1,9 @@ +2020-11-10 + +I hereby agree to the terms of the "OpenMPTCProuter Individual Contributor License Agreement", with MD5 checksum bc827a07eb93611d793ddb7c75083c00. + +I furthermore declare that I am authorized and able to make this agreement and sign this declaration. + +Signed, + +Paul Curry https://github.com/cr3ative From 402d51ae142ee63692ae17244cd41d5161126955 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:59:15 +0000 Subject: [PATCH 3/6] fix indentation --- .../root/etc/init.d/omr-bypass | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 1682c565a..2dec8e79f 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -331,32 +331,32 @@ _bypass_proto() { local domains domains="$(cat /proc/net/xt_ndpi/host_proto | grep -i $proto: | sed -e "s/$proto://i" -e 's/*//' -e 's/,/ /g')" if [ -n "$domains" ]; then - tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` + tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` for domain in $domains; do if [ -n "$domain" ]; then domain="$(echo $domain | sed 's/^\.//')" if [ "$(echo $domain | grep '\.$')" != "" ]; then domainlist="" - # construct list of domains to query - for tld in $tlds; do - i=$((i+1)) - # trim off header - if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then - continue - fi - # add to command - domainlist="${domainlist} ${domain}${tld}" - done - domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ - | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ - dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ - | awk '{print $1}' `# awk, which outputs queried domain to` \ - | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ - | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate - for validdomain in $domainlist; do - _bypass_domain $validdomain $intf - done + # construct list of domains to query + for tld in $tlds; do + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" + done + domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + for validdomain in $domainlist; do + _bypass_domain $validdomain $intf + done else _bypass_domain $domain $intf fi From 4abe0ab8af79c73ac0fac47fba72da2789055c84 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:59:55 +0000 Subject: [PATCH 4/6] ugh --- .../root/etc/init.d/omr-bypass | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 2dec8e79f..61e99753c 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -339,21 +339,21 @@ _bypass_proto() { domainlist="" # construct list of domains to query for tld in $tlds; do - i=$((i+1)) - # trim off header - if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then - continue - fi - # add to command - domainlist="${domainlist} ${domain}${tld}" + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" done domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ - | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ - dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ - | awk '{print $1}' `# awk, which outputs queried domain to` \ - | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ - | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate for validdomain in $domainlist; do _bypass_domain $validdomain $intf done From 6f4685b003c4c66a5b5830dc7a813d1dd64df1a0 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 14:13:39 +0000 Subject: [PATCH 5/6] Remove threading --- luci-app-omr-bypass/root/etc/init.d/omr-bypass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 61e99753c..40007e410 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -349,7 +349,7 @@ _bypass_proto() { done domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + | xargs -n8 `# xargs sends 8 arguments at a time to` \ dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ | awk '{print $1}' `# awk, which outputs queried domain to` \ | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ From ec3cbfa8ad50e41029e5c8a372397b3a2c7ead6a Mon Sep 17 00:00:00 2001 From: "Ycarus (Yannick Chabanois)" Date: Tue, 10 Nov 2020 17:01:41 +0100 Subject: [PATCH 6/6] Use threading and only use wanted domain --- luci-app-omr-bypass/root/etc/init.d/omr-bypass | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 40007e410..8e5a0c5bd 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -349,10 +349,11 @@ _bypass_proto() { done domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 `# xargs sends 8 arguments at a time to` \ - dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time to` \ + dig a +timeout=1 +tries=1 +retry=1 +nocmd +noall +answer `# dig, which passes results (if any) to` \ | awk '{print $1}' `# awk, which outputs queried domain to` \ - | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` to \ + | grep $domain `# grep, only keep wanted domain` \ | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate for validdomain in $domainlist; do _bypass_domain $validdomain $intf