adblock: update 3.5.2

backend:
* add generic blocklist archive support
* add support for blacklist archive from Toulouse 1 University Capitole
* add support for urlhaus RPZ domains by abuse.ch
* small bugfixes & enhancements

luci:
* archive sub-categories (shalla & ut_capitole) are now configurable
  via LuCI CBI template
* small bugfixes & enhancements

Signed-off-by: Dirk Brenken <dev@brenken.org>

net/adblock/Makefile

@@ -6,7 +6,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=3.5.1
+PKG_VERSION:=3.5.2
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>

net/adblock/files/README.md

@@ -46,11 +46,15 @@ A lot of people already use adblocker plugins within their desktop browsers, but
     * [reg_ru](https://code.google.com/p/ruadlist)
     * => regional blocklist for Russia, weekly updates, approx. 2.000 entries
     * [shallalist](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
-    * => daily updates, approx. 32.000 entries (a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html))
+    * => daily updates, approx. 32.000 entries (a short description of all categories can be found [online](http://www.shallalist.de/categories.html))
     * [spam404](http://www.spam404.com)
     * => infrequent updates, approx. 5.000 entries
     * [sysctl/cameleon](http://sysctl.org/cameleon)
     * => weekly updates, approx. 21.000 entries
+    * [ut_capitole](https://dsi.ut-capitole.fr/blacklists) (categories "cryptojacking" "ddos" "malware" "phishing" "warez" enabled by default)
+    * => daily updates, approx. 64.000 entries (a short description of all categories can be found [online](https://dsi.ut-capitole.fr/blacklists/index_en.php))
+    * [urlhaus](https://urlhaus.abuse.ch)
+    * => numerous updates on the same day, approx. 3.500 entries
     * [whocares](http://someonewhocares.org)
     * => weekly updates, approx. 12.000 entries
     * [winhelp](http://winhelp2002.mvps.org)

net/adblock/files/adblock.conf

@@ -83,7 +83,7 @@ config source 'openphish'
 config source 'ransomware'
 	option adb_src 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt'
 	option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}'
-	option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 1900 entries'
+	option adb_src_desc 'focus on ransomware by abuse.ch, numerous updates on the same day, approx. 1900 entries'
 	option enabled '0'
 
 config source 'reg_cn'
@@ -137,7 +137,7 @@ config source 'reg_ru'
 config source 'shalla'
 	option adb_src 'http://www.shallalist.de/Downloads/shallalist.tar.gz'
 	option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}'
-	option adb_src_desc 'broad blocklist subdivided in different categories, daily updates, approx. 31.700 entries'
+	option adb_src_desc 'huge blocklist archive subdivided in different categories, daily updates. Check http://www.shallalist.de/categories.html for more categories'
 	list adb_src_cat 'adv'
 	list adb_src_cat 'costtraps'
 	list adb_src_cat 'spyware'
@@ -157,6 +157,23 @@ config source 'sysctl'
 	option adb_src_desc 'broad blocklist, weekly updates, approx. 16.500 entries'
 	option enabled '0'
 
+config source 'urlhaus'
+	option adb_src 'https://urlhaus.abuse.ch/downloads/rpz'
+	option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)+/{print tolower(\$1)}'
+	option adb_src_desc 'urlhaus RPZ domains by abuse.ch, numerous updates on the same day, approx. 3.500 entries'
+	option enabled '0'
+
+config source 'ut_capitole'
+	option adb_src 'https://dsi.ut-capitole.fr/blacklists/download/blacklists.tar.gz'
+	option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}'
+	option adb_src_desc 'huge blocklist archive subdivided in different categories, daily updates. Check https://dsi.ut-capitole.fr/blacklists/index_en.php for more categories'
+	list adb_src_cat 'cryptojacking'
+	list adb_src_cat 'ddos'
+	list adb_src_cat 'malware'
+	list adb_src_cat 'phishing'
+	list adb_src_cat 'warez'
+	option enabled '0'
+
 config source 'whocares'
 	option adb_src 'http://someonewhocares.org/hosts/hosts'
 	option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$2)}'
@@ -184,5 +201,5 @@ config source 'yoyo'
 config source 'zeus'
 	option adb_src 'https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist'
 	option adb_src_rset '/^([^([:space:]|#|\*|\/).]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}'
-	option adb_src_desc 'focus on zeus botnet, daily updates, approx. 400 entries'
+	option adb_src_desc 'focus on zeus botnet by abuse.ch, daily updates, approx. 400 entries'
 	option enabled '0'

net/adblock/files/adblock.notify

@@ -1,8 +1,8 @@
 #!/bin/sh
 #
-# adblock send mail script for mstmp
+# adblock send mail script for msmtp
 # written by Dirk Brenken (dev@brenken.org)
-# Please note: you have to install and configure the package 'mstmp' before using this script.
+# Please note: you have to install and configure the package 'msmtp' before using this script.
 
 # This is free software, licensed under the GNU General Public License v3.
 # You should have received a copy of the GNU General Public License
@@ -10,12 +10,25 @@
 
 LC_ALL=C
 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
-mail_ver="1.0.2"
+mail_ver="1.0.3"
 mail_daemon="$(command -v msmtp)"
 mail_profile="adb_notify"
 #mail_debug="--debug"
 mail_rc=1
 
+# mail header & mail receiver check
+#
+mail_sender="no-reply@adblock"
+mail_receiver="!!!ChangeMe!!!"
+mail_topic="${HOSTNAME}: adblock notification"
+mail_head="From: ${mail_sender}\nTo: ${mail_receiver}\nSubject: ${mail_topic}\nReply-to: ${mail_sender}\nMime-Version: 1.0\nContent-Type: text/html\nContent-Disposition: inline\n\n"
+
+if [ "${mail_receiver}" = "!!!ChangeMe!!!" ]
+then
+    logger -p "err" -t "adblock-notify-[${mail_ver}]" "please change the 'mail_receiver' in '/etc/adblock/adblock.notify'"
+    exit ${mail_rc}
+fi
+
 # mail daemon check
 #
 if [ ! -x "${mail_daemon}" ]
@@ -34,13 +47,6 @@ else
     log_info="$(logread -e "adblock-" | awk '{NR=1;max=79;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max),"↵"} else {print " ",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}')"
 fi
 
-# mail header
-#
-mail_sender="no-reply@adblock"
-mail_receiver="!!!ChangeMe!!!"
-mail_topic="adblock notification"
-mail_head="From: ${mail_sender}\nTo: ${mail_receiver}\nSubject: ${mail_topic}\nReply-to: ${mail_sender}\nMime-Version: 1.0\nContent-Type: text/html\nContent-Disposition: inline\n\n"
-
 # mail body
 #
 mail_text="<html><body><pre style='display:block;font-family:monospace;font-size:1rem;padding:20;background-color:#f3eee5;white-space:pre'>"

net/adblock/files/adblock.sh

@@ -10,7 +10,7 @@
 #
 LC_ALL=C
 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
-adb_ver="3.5.1"
+adb_ver="3.5.2"
 adb_sysver="unknown"
 adb_enabled=0
 adb_debug=0
@@ -354,7 +354,7 @@ f_count()
     then
         if [ "${adb_dns}" = "named" ] || [ "${adb_dns}" = "kresd" ]
         then
-            adb_cnt="$(( ($(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}") - $(printf "%s" "${adb_dnsheader}" | grep -c "^")) / 2 ))"
+            adb_cnt="$(( ($(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}") - $(printf '%s' "${adb_dnsheader}" | grep -c "^")) / 2 ))"
         else
             adb_cnt="$(wc -l 2>/dev/null < "${adb_dnsdir}/${adb_dnsfile}")"
         fi
@@ -406,15 +406,17 @@ f_extconf()
 
     uci_config="firewall"
     if [ ${adb_enabled} -eq 1 ] && [ ${adb_forcedns} -eq 1 ] && \
-       [ -z "$(uci -q get firewall.adblock_dns)" ] && [ $(/etc/init.d/firewall enabled; printf "%u" ${?}) -eq 0 ]
+       [ -z "$(uci -q get firewall.adblock_dns)" ] && [ $(/etc/init.d/firewall enabled; printf '%u' ${?}) -eq 0 ]
     then
-        uci -q set firewall.adblock_dns="redirect"
-        uci -q set firewall.adblock_dns.name="Adblock DNS"
-        uci -q set firewall.adblock_dns.src="lan"
-        uci -q set firewall.adblock_dns.proto="tcp udp"
-        uci -q set firewall.adblock_dns.src_dport="53"
-        uci -q set firewall.adblock_dns.dest_port="53"
-        uci -q set firewall.adblock_dns.target="DNAT"
+		uci -q batch <<-EOF
+			set firewall.adblock_dns="redirect"
+			set firewall.adblock_dns.name="Adblock DNS"
+			set firewall.adblock_dns.src="lan"
+			set firewall.adblock_dns.proto="tcp udp"
+			set firewall.adblock_dns.src_dport="53"
+			set firewall.adblock_dns.dest_port="53"
+			set firewall.adblock_dns.target="DNAT"
+		EOF
     elif [ -n "$(uci -q get firewall.adblock_dns)" ] && ([ ${adb_enabled} -eq 0 ] || [ ${adb_forcedns} -eq 0 ])
     then
         uci -q delete firewall.adblock_dns
@@ -672,7 +674,7 @@ f_query()
 
     if [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]
     then
-        printf "%s\n" "::: invalid domain input, please submit a single domain, e.g. 'doubleclick.net'"
+        printf '%s\n' "::: invalid domain input, please submit a single domain, e.g. 'doubleclick.net'"
     else
         case "${adb_dns}" in
             dnsmasq)
@@ -686,8 +688,8 @@ f_query()
         do
             search="${domain//./\.}"
             result="$(awk -F '/|\"| ' "/^($search|[^\*].*[\/\"\. ]+${search})/{i++;{printf(\"  + %s\n\",\$${field})};if(i>9){printf(\"  + %s\n\",\"[...]\");exit}}" "${adb_dnsdir}/${adb_dnsfile}")"
-            printf "%s\n" "::: results for domain '${domain}'"
-            printf "%s\n" "${result:-"  - no match"}"
+            printf '%s\n' "::: results for domain '${domain}'"
+            printf '%s\n' "${result:-"  - no match"}"
             domain="${tld}"
             tld="${domain#*.}"
         done
@@ -778,7 +780,7 @@ f_log()
 #
 f_main()
 {
-    local tmp_load tmp_file src_name src_rset src_arc src_log mem_total mem_free enabled url cnt=1
+    local tmp_load tmp_file src_name src_rset src_url src_log src_arc src_cat cat list entry suffix mem_total mem_free enabled cnt=1
 
     mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
     mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
@@ -820,16 +822,17 @@ f_main()
     #
     for src_name in ${adb_sources}
     do
-        eval "enabled=\"\${enabled_${src_name}}\""
-        eval "url=\"\${adb_src_${src_name}}\""
-        eval "src_rset=\"\${adb_src_rset_${src_name}}\""
+        enabled="$(eval printf '%s' \"\${enabled_${src_name}\}\")"
+        src_url="$(eval printf '%s' \"\${adb_src_${src_name}\}\")"
+        src_rset="$(eval printf '%s' \"\${adb_src_rset_${src_name}\}\")"
+        src_cat="$(eval printf '%s' \"\${adb_src_cat_${src_name}\}\")"
         adb_tmpload="${tmp_load}.${src_name}"
         adb_tmpfile="${tmp_file}.${src_name}"
 
         # basic pre-checks
         #
         f_log "debug" "f_main ::: name: ${src_name}, enabled: ${enabled}"
-        if [ "${enabled}" != "1" ] || [ -z "${url}" ] || [ -z "${src_rset}" ]
+        if [ "${enabled}" != "1" ] || [ -z "${src_url}" ] || [ -z "${src_rset}" ]
         then
             f_list remove
             continue
@@ -854,10 +857,10 @@ f_main()
         #
         if [ "${src_name}" = "blacklist" ]
         then
-            if [ -s "${url}" ]
+            if [ -s "${src_url}" ]
             then
                 (
-                  src_log="$(cat "${url}" > "${adb_tmpload}" 2>&1)"
+                  src_log="$(cat "${src_url}" > "${adb_tmpload}" 2>&1)"
                   adb_rc=${?}
                   if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
                   then
@@ -874,32 +877,38 @@ f_main()
                       fi
                   else
                       src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
-                      f_log "debug" "f_main ::: name: ${src_name}, url: ${url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
+                      f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
                   fi
                 ) &
             else
                 continue
             fi
-        elif [ "${src_name}" = "shalla" ]
+        elif [ -n "${src_cat}" ]
         then
             (
-              src_arc="${adb_tmpdir}"/shallalist.tar.gz
-              src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${url}" 2>&1)"
+              src_arc="${adb_tmpdir}/${src_url##*/}"
+              src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${src_arc}" "${src_url}" 2>&1)"
               adb_rc=${?}
               if [ ${adb_rc} -eq 0 ] && [ -s "${src_arc}" ]
               then
-                  for category in ${adb_src_cat_shalla}
+                  list="$(tar -tzf "${src_arc}")"
+                  suffix="$(eval printf '%s' \"\${adb_src_suffix_${src_name}:-\"domains\"\}\")"
+                  for cat in ${src_cat}
                   do
-                      tar -xOzf "${src_arc}" "BL/${category}/domains" >> "${adb_tmpload}"
-                      adb_rc=${?}
-                      if [ ${adb_rc} -ne 0 ]
+                      entry="$(printf '%s' "${list}" | grep -E "[\^/]+${cat}/${suffix}")"
+                      if [ -n "${entry}" ]
                       then
-                          break
+                          tar -xOzf "${src_arc}" "${entry}" >> "${adb_tmpload}"
+                          adb_rc=${?}
+                          if [ ${adb_rc} -ne 0 ]
+                          then
+                              break
+                          fi
                       fi
                   done
               else
                   src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
-                  f_log "debug" "f_main ::: name: ${src_name}, url: ${url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
+                  f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
               fi
               if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
               then
@@ -929,7 +938,7 @@ f_main()
             ) &
         else
             (
-              src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>&1)"
+              src_log="$("${adb_fetchutil}" ${adb_fetchparm} "${adb_tmpload}" "${src_url}" 2>&1)"
               adb_rc=${?}
               if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
               then
@@ -953,7 +962,7 @@ f_main()
                   fi
               else
                   src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
-                  f_log "debug" "f_main ::: name: ${src_name}, url: ${url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
+                  f_log "debug" "f_main ::: name: ${src_name}, url: ${src_url}, rc: ${adb_rc}, log: ${src_log:-"-"}"
                   if [ ${adb_backup} -eq 1 ]
                   then
                       f_list restore