From 62a29f8ff0805844b5631cb4337d5b0d8e1a5d91 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:54:52 +0000 Subject: [PATCH 1/5] parallel digs --- .../root/etc/init.d/omr-bypass | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index b5bd902a0..1682c565a 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -331,21 +331,32 @@ _bypass_proto() { local domains domains="$(cat /proc/net/xt_ndpi/host_proto | grep -i $proto: | sed -e "s/$proto://i" -e 's/*//' -e 's/,/ /g')" if [ -n "$domains" ]; then + tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` for domain in $domains; do if [ -n "$domain" ]; then domain="$(echo $domain | sed 's/^\.//')" if [ "$(echo $domain | grep '\.$')" != "" ]; then - tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` - if [ -n "tlds" ]; then - i=0 - for tld in $tlds; do - i=$((i+1)) - tld="$(echo $tld | awk '{print tolower($0)}')" - if [ "$i" -gt "11" ] && [ "$(dig a +timeout=1 +tries=1 +retry=1 +nocmd +noall +answer ${domain}${tld})" != "" ]; then - _bypass_domain ${domain}${tld} $intf - fi - done - fi + domainlist="" + # construct list of domains to query + for tld in $tlds; do + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" + done + domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + for validdomain in $domainlist; do + _bypass_domain $validdomain $intf + done else _bypass_domain $domain $intf fi From dee7b381cc317d0d64a9a6e2434209519f90600d Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:56:59 +0000 Subject: [PATCH 2/5] add CLA --- contributors/cr3ative.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 contributors/cr3ative.md diff --git a/contributors/cr3ative.md b/contributors/cr3ative.md new file mode 100644 index 000000000..b0c58c54e --- /dev/null +++ b/contributors/cr3ative.md @@ -0,0 +1,9 @@ +2020-11-10 + +I hereby agree to the terms of the "OpenMPTCProuter Individual Contributor License Agreement", with MD5 checksum bc827a07eb93611d793ddb7c75083c00. + +I furthermore declare that I am authorized and able to make this agreement and sign this declaration. + +Signed, + +Paul Curry https://github.com/cr3ative From 08174a265099604d975b0e2ea31b87b655bb9e1f Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:59:15 +0000 Subject: [PATCH 3/5] fix indentation --- .../root/etc/init.d/omr-bypass | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 1682c565a..2dec8e79f 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -331,32 +331,32 @@ _bypass_proto() { local domains domains="$(cat /proc/net/xt_ndpi/host_proto | grep -i $proto: | sed -e "s/$proto://i" -e 's/*//' -e 's/,/ /g')" if [ -n "$domains" ]; then - tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` + tlds=`curl --max-time 4 -s -k https://data.iana.org/TLD/tlds-alpha-by-domain.txt` for domain in $domains; do if [ -n "$domain" ]; then domain="$(echo $domain | sed 's/^\.//')" if [ "$(echo $domain | grep '\.$')" != "" ]; then domainlist="" - # construct list of domains to query - for tld in $tlds; do - i=$((i+1)) - # trim off header - if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then - continue - fi - # add to command - domainlist="${domainlist} ${domain}${tld}" - done - domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ - | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ - dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ - | awk '{print $1}' `# awk, which outputs queried domain to` \ - | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ - | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate - for validdomain in $domainlist; do - _bypass_domain $validdomain $intf - done + # construct list of domains to query + for tld in $tlds; do + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" + done + domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + for validdomain in $domainlist; do + _bypass_domain $validdomain $intf + done else _bypass_domain $domain $intf fi From 43da6c6e1e26f21abac1aaeeb3ea68a8569eb4d2 Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 13:59:55 +0000 Subject: [PATCH 4/5] ugh --- .../root/etc/init.d/omr-bypass | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 2dec8e79f..61e99753c 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -339,21 +339,21 @@ _bypass_proto() { domainlist="" # construct list of domains to query for tld in $tlds; do - i=$((i+1)) - # trim off header - if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then - continue - fi - # add to command - domainlist="${domainlist} ${domain}${tld}" + i=$((i+1)) + # trim off header + if [ "$i" -lt "12" ] || [ "$i" -gt "50" ]; then + continue + fi + # add to command + domainlist="${domainlist} ${domain}${tld}" done domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ - | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ - dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ - | awk '{print $1}' `# awk, which outputs queried domain to` \ - | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ - | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate + | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ + | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ + | awk '{print $1}' `# awk, which outputs queried domain to` \ + | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \ + | awk '{for (i=1;i<=NF;i++) if (!a[$i]++) printf("%s%s",$i,FS)}{printf("\n")}')" # deduplicate for validdomain in $domainlist; do _bypass_domain $validdomain $intf done From d74fb4941856063f23fa5b66268d848d5d601f6b Mon Sep 17 00:00:00 2001 From: Paul Curry Date: Tue, 10 Nov 2020 14:13:39 +0000 Subject: [PATCH 5/5] Remove threading --- luci-app-omr-bypass/root/etc/init.d/omr-bypass | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luci-app-omr-bypass/root/etc/init.d/omr-bypass b/luci-app-omr-bypass/root/etc/init.d/omr-bypass index 61e99753c..40007e410 100755 --- a/luci-app-omr-bypass/root/etc/init.d/omr-bypass +++ b/luci-app-omr-bypass/root/etc/init.d/omr-bypass @@ -349,7 +349,7 @@ _bypass_proto() { done domainlist="$(echo $domainlist `# Get the list of valid domains, pass it to awk` \ | awk '{print tolower($0)}' `# awk lowercases the whole string and passes it to ` \ - | xargs -n8 -P12 `# xargs sends 8 arguments at a time across 12 threads to` \ + | xargs -n8 `# xargs sends 8 arguments at a time to` \ dig a +nocmd +noall +answer `# dig, which passes results (if any) to` \ | awk '{print $1}' `# awk, which outputs queried domain to` \ | sed -e 's/.$//' `# sed, which trims off the trailing dot (google.com. -> google.com)` \