Merge pull request #2400 from dibdot/adblock

adblock: 0.80.0

net/adblock/Makefile

@@ -7,7 +7,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=0.70.2
+PKG_VERSION:=0.80.0
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <openwrt@brenken.org>

net/adblock/files/README.md

@@ -25,7 +25,7 @@ When the dns server on your router receives dns requests, you will sort out quer
     * [palevotracker](https://palevotracker.abuse.ch)
     * => daily updates, approx. 15 entries
     * [ruadlist+easylist](https://code.google.com/p/ruadlist)
-    * => weekly updates, approx. 2.000 entries (experimental support, may include false positives!)
+    * => weekly updates, approx. 2.000 entries
     * [shallalist](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
     * => daily updates, approx. 32.000 entries (a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html))
     * [spam404](http://www.spam404.com)
@@ -68,6 +68,11 @@ When the dns server on your router receives dns requests, you will sort out quer
 * optional: enable/disable your required adblock list sources in */etc/config/adblock* - 'adaway', 'disconnect' and 'yoyo' are enabled by default
 * optional: maintain the adblock service in luci under 'System => Startup'
 
+## LuCI adblock companion package
+For easy management of the various blocklist sources and and the adblock options there is also a nice & efficient LuCI frontend available.  
+Please install the package 'luci-app-adblock'. Then you will find the application in LuCI located under 'Services' menu.  
+Thanks to Hannu Nyman for this great adblock LuCI frontend!  
+
 ## Tweaks
 * there is no need to enable all blacklist sites at once, for normal use one to three adblock list sources should be sufficient
 * if you really need to handle all blacklists at once add an usb stick or any other storage device to supersize your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details
@@ -84,7 +89,9 @@ When the dns server on your router receives dns requests, you will sort out quer
     * adb\_lanif => name of the logical lan interface (default: 'lan')
     * adb\_port => port of the adblock uhttpd instance (default: '65535')
     * adb\_nullipv4 => IPv4 blackhole ip address (default: '192.0.2.1')
-    * adb\_nullipv6 => IPv6 blackhole ip address (default '::ffff:c000:0201')
+    * adb\_nullipv6 => IPv6 blackhole ip address (default: '::ffff:c000:0201')
+    * adb\_probeipv4 => IPv4 address used for uplink online check (default: '8.8.8.8')
+    * adb\_probeipv6 => IPv6 address used for uplink online check (default: '2001:4860:4860::8888')
     * adb\_maxtime => download timeout limit in seconds (default: '60')
     * adb\_maxloop => startup timeout limit in seconds to wait for an active wan interface (default: '20')
 
@@ -93,8 +100,7 @@ This adblock package is a dns/dnsmasq based adblock solution for openwrt.
 Queries to ad/abuse domains are never forwarded and always replied with a local IP address which may be IPv4 or IPv6.  
 For that purpose adblock uses an ip address from the private 'TEST-NET-1' subnet (192.0.2.1 / ::ffff:c000:0201) by default.  
 Furthermore all ad/abuse queries will be filtered by ip(6)tables and redirected to internal adblock pixel server (in PREROUTING chain) or rejected (in FORWARD or OUTPUT chain).  
-All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files.  
-There is *no* adblock background daemon running, the (scheduled) start of the adblock service keeps only the adblock lists up-to-date.
+All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files. There is *no* adblock background daemon running, the (scheduled) start of the adblock service keeps only the adblock lists up-to-date.
 
 ## Support
 Please join the adblock discussion in this [openwrt forum thread](https://forum.openwrt.org/viewtopic.php?id=59803) or contact me by mail <openwrt@brenken.org>

net/adblock/files/adblock-helper.sh

@@ -86,13 +86,14 @@ f_envparse()
     adb_port="65535"
     adb_nullipv4="192.0.2.1"
     adb_nullipv6="::ffff:c000:0201"
+    adb_probeipv4="8.8.8.8"
+    adb_probeipv6="2001:4860:4860::8888"
     adb_maxtime="60"
     adb_maxloop="20"
     adb_blacklist="/etc/adblock/adblock.blacklist"
     adb_whitelist="/etc/adblock/adblock.whitelist"
 
-    # function to read/set global options by callback,
-    # prepare list items and build option list for all others
+    # function to read global options by callback
     #
     config_cb()
     {
@@ -107,45 +108,39 @@ f_envparse()
                 eval "${option}=\"${value}\""
             }
         else
-            option_cb()
-            {
-                local option="${1}"
-                local value="${2}"
-                local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p' 2>/dev/null)"
-                if [ -z "${opt_out}" ]
-                then
-                    all_options="${all_options} ${option}"
-                fi
-            }
-            list_cb()
-            {
-                local list="${1}"
-                local value="${2}"
-                if [ "${list}" = "adb_catlist" ]
-                then
-                    adb_cat_shalla="${adb_cat_shalla} ${value}"
-                fi
-            }
+            reset_cb
         fi
     }
 
-    # function to iterate through option list, read/set all options in "enabled" sections
+    # function to iterate through config list, read only options in "enabled" sections
     #
+    adb_cfglist="adb_backupdir adb_logfile adb_src"
+    unset adb_sources
     parse_config()
     {
         local config="${1}"
         config_get switch "${config}" "enabled"
         if [ "${switch}" = "1" ]
         then
-            for option in ${all_options}
+            for option in ${adb_cfglist}
             do
                 config_get value "${config}" "${option}"
                 if [ -n "${value}" ]
                 then
-                    local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p' 2>/dev/null)"
-                    if [ -n "${opt_src}" ]
+                    if [ "${option}" = "adb_src" ]
                     then
-                        adb_sources="${adb_sources} ${value}"
+                        if [ "${config}" = "shalla" ]
+                        then
+                            categories()
+                            {
+                                local cat="${1}"
+                                adb_cat_shalla="${adb_cat_shalla} ${cat}"
+                            }
+                            eval "adb_arc_shalla=\"${value}\""
+                            config_list_foreach "shalla" "adb_catlist" "categories"
+                        else
+                            adb_sources="${adb_sources} ${value}"
+                        fi
                     else
                         eval "${option}=\"${value}\""
                     fi
@@ -218,7 +213,23 @@ f_envparse()
         else
             network_get_device adb_wandev4 "${adb_wanif4}" 2>/dev/null
             network_get_device adb_wandev6 "${adb_wanif6}" 2>/dev/null
-            break
+            if [ -n "${adb_wandev4}" ]
+            then
+                rc="$(ping -q -4 -c1 -W1 -I${adb_wandev4} "${adb_probeipv4}" >/dev/null 2>&1; printf ${?})"
+                if [ $((rc)) -eq 0 ]
+                then
+                    f_log "get active IPv4 wan update interface/device (${adb_wanif4}/${adb_wandev4})"
+                    break
+                fi
+            elif [ -n "${adb_wandev6}" ]
+            then
+                rc="$(ping -q -6 -c1 -W1 -I${adb_wandev6} "${adb_probeipv6}" >/dev/null 2>&1; printf ${?})"
+                if [ $((rc)) -eq 0 ]
+                then
+                    f_log "get active IPv6 wan update interface/device (${adb_wanif6}/${adb_wandev6})"
+                    break
+                fi
+            fi
         fi
         if [ $((adb_cnt)) -ge $((adb_maxloop)) ]
         then
@@ -227,7 +238,6 @@ f_envparse()
             f_exit
         fi
         adb_cnt=$((adb_cnt + 1))
-        sleep 1
     done
 
     # get lan ip addresses
@@ -240,10 +250,6 @@ f_envparse()
         f_log "no valid IPv4/IPv6 configuration for given logical LAN interface found (${adb_lanif}), please set 'adb_lanif' manually" "${rc}"
         f_exit
     fi
-
-    # read system ntp server names
-    #
-    adb_ntpsrv="$(uci get system.ntp.server 2>/dev/null)"
 }
 
 #################################################
@@ -450,69 +456,6 @@ f_envcheck()
         fi
     fi
 
-    # wait for active wan update interface
-    #
-    while [ $((adb_cnt)) -le $((adb_maxloop)) ]
-    do
-        for interface in ${adb_wanif}
-        do
-            network_get_device adb_wandev "${interface}" 2>/dev/null
-            if [ -z "${adb_wandev}" ] || [ ! -d "/sys/class/net/${adb_wandev}" ]
-            then
-                if [ -n "${adb_wandev4}" ]
-                then
-                    adb_wandev="${adb_wandev4}"
-                else
-                    adb_wandev="${adb_wandev6}"
-                fi
-                if [ -z "${adb_wandev}" ] || [ ! -d "/sys/class/net/${adb_wandev}" ]
-                then
-                    rc=145
-                    f_log "no valid network device for given logical WAN interface found, please set 'adb_wanif' manually" "${rc}"
-                    f_restore
-                fi
-            fi
-            if [ -n "${adb_wandev4}" ]
-            then
-                rc="$(/bin/ping -c1 -W1 8.8.8.8 -I ${adb_wandev} >/dev/null 2>&1; printf $?)"
-            else
-                rc="$(/bin/ping -6 -c1 -W1 2001:4860:4860::8888 -I ${adb_wandev} >/dev/null 2>&1; printf $?)"
-            fi
-            if [ $((rc)) -eq 0  ]
-            then
-                f_log "get active wan update interface/device (${adb_wanif}/${adb_wandev})"
-                break 2
-            elif [ $((adb_cnt)) -eq $((adb_maxloop)) ]
-            then
-                rc=145
-                f_log "wan update interface/device not running (${adb_wanif}/${adb_wandev})" "${rc}"
-                f_restore
-            fi
-            adb_cnt=$((adb_cnt + 1))
-            sleep 1
-        done
-    done
-
-    # ntp time sync
-    #
-    if [ -n "${adb_ntpsrv}" ]
-    then
-        unset ntp_pool
-        for srv in ${adb_ntpsrv}
-        do
-            ntp_pool="${ntp_pool} -p ${srv}"
-        done
-        /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
-        rc=${?}
-        if [ $((rc)) -eq 0 ]
-        then
-            f_log "get ntp time sync"
-        else
-            rc=0
-            f_log "ntp time sync failed"
-        fi
-    fi
-
     # set dnsmasq defaults
     #
     if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
@@ -536,6 +479,7 @@ f_envcheck()
 #
 f_depend()
 {
+    local check
     local package="${1}"
 
     check="$(printf "${pkg_list}" | grep "^${package} -" 2>/dev/null)"

net/adblock/files/adblock-update.sh

@@ -45,7 +45,7 @@ fi
 # get current directory, script- and openwrt version
 #
 adb_scriptdir="${0%/*}"
-adb_scriptver="0.70.2"
+adb_scriptver="0.80.0"
 openwrt_version="$(cat /etc/openwrt_version 2>/dev/null)"
 
 # source in adblock function library

net/adblock/files/adblock.conf

@@ -3,7 +3,7 @@
 
 config adblock 'global'
 	option adb_enabled '1'
-	option adb_cfgver '0.70'
+	option adb_cfgver '0.80'
 	option adb_blacklist '/etc/adblock/adblock.blacklist'
 	option adb_whitelist '/etc/adblock/adblock.whitelist'
 
@@ -11,49 +11,59 @@ config service 'backup'
 	option enabled '0'
 	option adb_backupdir '/tmp'
 
-config service 'debuglog'
+config service 'log'
 	option enabled '0'
 	option adb_logfile '/tmp/adb_debug.log'
 
 config source 'adaway'
 	option enabled '1'
-	option adb_src_adaway 'https://adaway.org/hosts.txt&ruleset=rset_adaway'
+	option adb_src 'https://adaway.org/hosts.txt&ruleset=rset_adaway'
+	option adb_srcdesc 'focus on mobile ads, infrequent updates, approx. 400 entries'
 
 config source 'disconnect'
 	option enabled '1'
-	option adb_src_disconnect 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect'
+	option adb_src 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect'
+	option adb_srcdesc 'mozilla driven content blocklist, numerous updates on the same day, approx. 6.500 entries'
 
 config source 'dshield'
 	option enabled '0'
-	option adb_src_dshield 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield'
+	option adb_src 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield'
+	option adb_srcdesc 'broad blocklist for suspicious domains, daily updates, approx. 4.500 entries'
 
 config source 'feodo'
 	option enabled '0'
-	option adb_src_feodo 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo'
+	option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo'
+	option adb_srcdesc 'focus on feodo botnet domains, daily updates, approx. 0-10 entries'
 
 config source 'malware'
 	option enabled '0'
-	option adb_src_malware 'https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware'
+	option adb_src 'https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware'
+	option adb_srcdesc 'broad blocklist for malware domains, daily updates, approx. 16.000 entries'
 
 config source 'malwarelist'
 	option enabled '0'
-	option adb_src_malwarelist 'http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist'
+	option adb_src 'http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist'
+	option adb_srcdesc 'generic blocklist for malware domains, daily updates, approx. 1.500 entries'
 
 config source 'openphish'
 	option enabled '0'
-	option adb_src_openphish 'https://openphish.com/feed.txt&ruleset=rset_openphish'
+	option adb_src 'https://openphish.com/feed.txt&ruleset=rset_openphish'
+	option adb_srcdesc 'focus on phishing domains, numerous updates on the same day, approx. 1.800 entries'
 
 config source 'palevo'
 	option enabled '0'
-	option adb_src_palevo 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo'
+	option adb_src 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo'
+	option adb_srcdesc 'focus on palevo worm domains, daily updates, approx. 15 entries'
 
 config source 'ruadlist'
 	option enabled '0'
-	option adb_src_ruadlist 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt&ruleset=rset_ruadlist'
+	option adb_src 'https://easylist-downloads.adblockplus.org/ruadlist+easylist.txt&ruleset=rset_ruadlist'
+	option adb_srcdesc 'focus on russian ad related domains plus generic easylist additions, weekly updates, approx. 2.000 entries'
 
 config source 'shalla'
 	option enabled '0'
-	option adb_arc_shalla 'http://www.shallalist.de/Downloads/shallalist.tar.gz'
+	option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz'
+	option adb_srcdesc 'broad blocklist subdivided in different categories (adv, costtraps, spyware, tracker and warez enabled by default), daily updates, approx. 32.000 entries'
 	list adb_catlist 'adv'
 	list adb_catlist 'costtraps'
 	list adb_catlist 'spyware'
@@ -62,20 +72,25 @@ config source 'shalla'
 
 config source 'spam404'
 	option enabled '0'
-	option adb_src_spam404 'http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404'
+	option adb_src 'http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404'
+	option adb_srcdesc 'generic blocklist for suspicious domains, infrequent updates, approx. 5.000 entries'
 
 config source 'whocares'
 	option enabled '0'
-	option adb_src_whocares 'http://someonewhocares.org/hosts/hosts&ruleset=rset_whocares'
+	option adb_src 'http://someonewhocares.org/hosts/hosts&ruleset=rset_whocares'
+	option adb_srcdesc 'broad blocklist for suspicious domains, weekly updates, approx. 12.000 entries'
 
 config source 'winhelp'
 	option enabled '0'
-	option adb_src_winhelp 'http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp'
+	option adb_src 'http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp'
+	option adb_srcdesc 'broad blocklist for suspicious domains, infrequent updates, approx. 15.000 entries'
 
 config source 'yoyo'
 	option enabled '1'
-	option adb_src_yoyo 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo'
+	option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo'
+	option adb_srcdesc 'focus on ad related domains, weekly updates, approx. 2.500 entries'
 
 config source 'zeus'
 	option enabled '0'
-	option adb_src_zeus 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus'
+	option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus'
+	option adb_srcdesc 'focus on zeus botnet domains, daily updates, approx. 440 entries'