adblock: update 3.6.2

* enhance the query function to search in adblock backups as well,
  to get back the set of blocking lists sources for a certain domain
* add "Latest DNS Queries" report to commandline version as well
  (already in LuCI)
* made the tld compression (the error handling) more robust,
  remove the needless 'adb_forcesrt' option
* removed abandoned 'feodo' list source
* updated readme

Signed-off-by: Dirk Brenken <dev@brenken.org>

net/adblock/Makefile

@@ -1,12 +1,12 @@
 #
-# Copyright (c) 2015-2018 Dirk Brenken (dev@brenken.org)
+# Copyright (c) 2015-2019 Dirk Brenken (dev@brenken.org)
 # This is free software, licensed under the GNU General Public License v3.
 #
 
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=3.6.1
+PKG_VERSION:=3.6.2
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>

net/adblock/files/README.md

@@ -17,8 +17,6 @@ A lot of people already use adblocker plugins within their desktop browsers, but
     * => numerous updates on the same day, approx. 6.500 entries (enabled by default)
     * [dshield](http://dshield.org)
     * => daily updates, approx. 4.500 entries
-    * [feodotracker](https://feodotracker.abuse.ch)
-    * => daily updates, approx. 0-10 entries
     * [hphosts](https://hosts-file.net)
     * => monthly updates, approx. 50.000 entries
     * [malwaredomains](http://malwaredomains.com)
@@ -147,7 +145,6 @@ A lot of people already use adblocker plugins within their desktop browsers, but
     * adb\_nice => set the nice level of the adblock process and all sub-processes (int/default: '0', standard priority)
     * adb\_triggerdelay => additional trigger delay in seconds before adblock processing begins (int/default: '2')
     * adb\_forcedns => force dns requests to local resolver (bool/default: '0', disabled)
-    * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (bool/default: '0', disabled)
     * adb\_backup => create compressed blocklist backups, they will be used in case of download errors or during startup in backup mode (bool/default: '0', disabled)
     * adb\_backupdir => target directory for adblock backups (default: not set)
     * adb\_backup_mode => do not automatically update blocklists during startup, use backups instead (bool/default: '0', disabled)
@@ -313,23 +310,32 @@ This entry does not remove:
 
 The query function checks against the submitted (sub-)domain and recurses automatically to the upper top level domain. For every (sub-)domain it returns the first ten relevant results.
 <pre><code>
-/etc/init.d/adblock query www.example.google.com
-::: results for domain 'www.example.google.com'
-  - no match
-::: results for domain 'example.google.com'
-  - no match
+/etc/init.d/adblock query google.com
+:::
 ::: results for domain 'google.com'
-  + ads.google.com
-  + adservices.google.com
-  + adwords.google.com
-  + ampcid.google.com
+:::
   + analytics.google.com
-  + gg.google.com
-  + google.com.analytics.kdgsrltkcun.com
   + googleadapis.l.google.com
-  + id.google.com
-  + pagead-googlehosted.l.google.com
-  + [...]
+  + pagead.l.google.com
+  + partnerad.l.google.com
+  + ssl-google-analytics.l.google.com
+  + www-google-analytics.l.google.com
+  + video-stats.video.google.com
+:::
+::: results for domain 'google.com' in backups
+:::
+  + adb_list.adguard.gz           partnerad.l.google.com
+  + adb_list.adguard.gz           googleadapis.l.google.com
+  + adb_list.adguard.gz           ssl-google-analytics.l.google.com
+  + adb_list.adguard.gz           [...]
+  + adb_list.disconnect.gz        pagead.l.google.com
+  + adb_list.disconnect.gz        partnerad.l.google.com
+  + adb_list.disconnect.gz        video-stats.video.google.com
+  + adb_list.disconnect.gz        [...]
+  + adb_list.yoyo.gz              analytics.google.com
+  + adb_list.yoyo.gz              pagead.l.google.com
+  + adb_list.yoyo.gz              partnerad.l.google.com
+  + adb_list.yoyo.gz              [...]
 </code></pre>
   
 **add a new blocklist source:**

net/adblock/files/adblock.conf

@@ -50,12 +50,6 @@ config source 'dshield'
 	option adb_src_desc 'generic blocklist, daily updates, approx. 3.500 entries'
 	option enabled '0'
 
-config source 'feodo'
-	option adb_src 'https://feodotracker.abuse.ch/blocklist/?download=domainblocklist'
-	option adb_src_rset '/^([[:alnum:]_-]+\.)+[[:alpha:]]+([[:space:]]|$)/{print tolower(\$1)}'
-	option adb_src_desc 'focus on feodo botnet, daily updates, approx. 0-10 entries'
-	option enabled '0'
-
 config source 'hphosts'
 	option adb_src 'https://hosts-file.net/ad_servers.txt'
 	option adb_src_rset '/^127\.0\.0\.1[[:space:]]+([[:alnum:]_-]+\.)+[[:alpha:]]+([[:space:]]|\$)+/{print tolower(\$2)}'

net/adblock/files/adblock.sh

@@ -10,12 +10,10 @@
 #
 LC_ALL=C
 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
-adb_ver="3.6.1"
+adb_ver="3.6.2"
 adb_sysver="unknown"
 adb_enabled=0
 adb_debug=0
-adb_backup_mode=0
-adb_forcesrt=0
 adb_forcedns=0
 adb_jail=0
 adb_maxqueue=8
@@ -23,6 +21,7 @@ adb_notify=0
 adb_notifycnt=0
 adb_triggerdelay=0
 adb_backup=0
+adb_backup_mode=0
 adb_backupdir="/mnt"
 adb_fetchutil="uclient-fetch"
 adb_dns="dnsmasq"
@@ -571,33 +570,45 @@ f_list()
 #
 f_tld()
 {
-	local cnt cnt_srt cnt_tld source="${1}" temp="${1}.tld"
+	local cnt cnt_srt cnt_tld source="${1}" temp_src="${1}.src.gz" temp_tld="${1}.tld" tld_ok="false"
 
-	cnt="$(wc -l 2>/dev/null < "${source}")"
-	awk 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${source}" > "${temp}"
+	gzip -cf "${source}" 2>/dev/null > "${temp_src}"
 	if [ ${?} -eq 0 ]
-	then
-		sort -u "${temp}" > "${source}"
+	then	
+		cnt="$(wc -l 2>/dev/null < "${source}")"
+		awk 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${source}" > "${temp_tld}"
 		if [ ${?} -eq 0 ]
 		then
-			cnt_srt="$(wc -l 2>/dev/null < "${source}")"
-			awk '{if(NR==1){tld=$NF};while(getline){if($NF!~tld"\\."){print tld;tld=$NF}}print tld}' "${source}" > "${temp}"
+			sort -u "${temp_tld}" > "${source}"
 			if [ ${?} -eq 0 ]
 			then
-				awk 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${temp}" > "${source}"
+				cnt_srt="$(wc -l 2>/dev/null < "${source}")"
+				awk '{if(NR==1){tld=$NF};while(getline){if($NF!~tld"\\."){print tld;tld=$NF}}print tld}' "${source}" > "${temp_tld}"
 				if [ ${?} -eq 0 ]
 				then
-					cnt_tld="$(wc -l 2>/dev/null < "${source}")"
-					rm -f "${temp}"
-				else
-					mv -f "${temp}" > "${source}"
+					awk 'BEGIN{FS="."}{for(f=NF;f>1;f--)printf "%s.",$f;print $1}' "${temp_tld}" > "${source}"
+					if [ ${?} -eq 0 ]
+					then
+						rm -f "${temp_src}" "${temp_tld}"
+						cnt_tld="$(wc -l 2>/dev/null < "${source}")"
+						tld_ok="true"
+					fi
 				fi
 			fi
-		else
-			mv -f "${temp}" "${source}"
 		fi
 	fi
-	f_log "debug" "f_tld    ::: source: ${source}, cnt: ${cnt:-"-"}, cnt_srt: ${cnt_srt:-"-"}, cnt_tld: ${cnt_tld:-"-"}"
+
+	if [ "${tld_ok}" = "false" ]
+	then
+		rm -f "${temp_tld}"
+		gunzip -cf "${temp_src}" 2>/dev/null > "${source}"
+		if [ ${?} -ne 0 ]
+		then
+			rm -f "${temp_src}"
+			> "${source}"
+		fi
+	fi
+	f_log "debug" "f_tld    ::: source: ${source}, cnt: ${cnt:-"-"}, cnt_srt: ${cnt_srt:-"-"}, cnt_tld: ${cnt_tld:-"-"}, tld_ok: ${tld_ok}"
 }
 
 # blocklist hash compare
@@ -712,11 +723,21 @@ f_query()
 		do
 			search="${domain//./\.}"
 			result="$(awk -F '/|\"| ' "/^($search|${prefix}+${search}.*${suffix}$)/{i++;{printf(\"  + %s\n\",\$${field})};if(i>9){printf(\"  + %s\n\",\"[...]\");exit}}" "${adb_dnsdir}/${adb_dnsfile}")"
-			printf '%s\n' "::: results for domain '${domain}'"
+			printf '%s\n%s\n%s\n' ":::" "::: results for domain '${domain}'" ":::"
 			printf '%s\n' "${result:-"  - no match"}"
 			domain="${tld}"
 			tld="${domain#*.}"
 		done
+
+		if [ ${adb_backup} -eq 1 ] && [ -d "${adb_backupdir}" ]
+		then
+			search="${1//./\.}"
+			printf '%s\n%s\n%s\n' ":::" "::: results for domain '${1}' in backups" ":::"
+			for file in ${adb_backupdir}/${adb_dnsprefix}.*.gz
+			do
+				zcat "${file}" 2>/dev/null | awk -v f="${file##*/}" "/^($search|.*\.${search})/{i++;{printf(\"  + %-30s%s\n\",f,\$1)};if(i>=3){printf(\"  + %-30s%s\n\",f,\"[...]\");exit}}"
+			done
+		fi
 	fi
 }
 
@@ -824,7 +845,7 @@ f_main()
 	> "${adb_tmpdir}/tmp.raw_whitelist"
 	> "${adb_tmpdir}/tmp.add_whitelist"
 	> "${adb_tmpdir}/tmp.rem_whitelist"
-	f_log "debug" "f_main   ::: dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, dns_jail: ${adb_jail}, force_srt: ${adb_forcesrt}, force_dns: ${adb_forcedns}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${adb_maxqueue}"
+	f_log "debug" "f_main   ::: dns: ${adb_dns}, fetch_util: ${adb_fetchinfo}, backup: ${adb_backup}, backup_mode: ${adb_backup_mode}, dns_jail: ${adb_jail}, force_dns: ${adb_forcedns}, mem_total: ${mem_total:-0}, mem_free: ${mem_free:-0}, max_queue: ${adb_maxqueue}"
 
 	# prepare whitelist entries
 	#
@@ -883,10 +904,6 @@ f_main()
 			f_list restore
 			if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ]
 			then
-				if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ]
-				then
-					f_tld "${adb_tmpfile}"
-				fi
 				continue
 			fi
 		fi
@@ -908,10 +925,6 @@ f_main()
 						then
 							rm -f "${adb_tmpload}"
 							f_list download
-							if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ]
-							then
-								f_tld "${adb_tmpfile}"
-							fi
 						fi
 					else
 						src_log="$(printf '%s' "${src_log}" | awk '{ORS=" ";print $0}')"
@@ -961,10 +974,6 @@ f_main()
 						then
 							f_list backup
 						fi
-						if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ]
-						then
-							f_tld "${adb_tmpfile}"
-						fi
 					elif [ ${adb_backup} -eq 1 ]
 					then
 						f_list restore
@@ -990,10 +999,6 @@ f_main()
 						then
 							f_list backup
 						fi
-						if ([ ${mem_total} -lt 64 ] || [ ${mem_free} -lt 40 ]) && [ ${adb_forcesrt} -eq 0 ]
-						then
-							f_tld "${adb_tmpfile}"
-						fi
 					elif [ ${adb_backup} -eq 1 ]
 					then
 						f_list restore
@@ -1028,10 +1033,7 @@ f_main()
 	f_hash
 	if [ -s "${adb_tmpdir}/${adb_dnsfile}" ]
 	then
-		if ([ ${mem_total} -ge 64 ] && [ ${mem_free} -ge 40 ]) || [ ${adb_forcesrt} -eq 1 ]
-		then
-			f_tld "${adb_tmpdir}/${adb_dnsfile}"
-		fi
+		f_tld "${adb_tmpdir}/${adb_dnsfile}"
 		f_list final
 	else
 		> "${adb_dnsdir}/${adb_dnsfile}"
@@ -1057,7 +1059,7 @@ f_main()
 #
 f_report()
 {
-	local bg_pid total blocked percent rep_clients rep_domains rep_blocked index hold ports cnt=0 print="${1:-"true"}"
+	local bg_pid total blocked percent rep_clients rep_domains rep_blocked rep_latest index hold ports cnt=0 print="${1:-"true"}"
 
 	if [ ! -x "${adb_reputil}" ]
 	then
@@ -1125,6 +1127,7 @@ f_report()
 			rep_clients="$(awk '{print $3}' ${adb_repdir}/adb_report | sort | uniq -c | sort -r | awk '{ORS=" ";if(NR<=10) printf("%s_%s ",$1,$2)}')"
 			rep_domains="$(awk '{if($5!="NX")print $4}' ${adb_repdir}/adb_report | sort | uniq -c | sort -r | awk '{ORS=" ";if(NR<=10)printf("%s_%s ",$1,$2)}')"
 			rep_blocked="$(awk '{if($5=="NX")print $4}' ${adb_repdir}/adb_report | sort | uniq -c | sort -r | awk '{ORS=" ";if(NR<=10)printf("%s_%s ",$1,$2)}')"
+			rep_latest="$(awk 'BEGIN{printf("  + %-15s%-15s%-45s%-50s%s\n","Date","Time","Client","Domain","Answer")}FNR<=50{printf("  + %-15s%-15s%-45s%-50s%s\n",$1,$2,$3,$4,$5)}' ${adb_repdir}/adb_report)"
 
 			> "${adb_repdir}/adb_report.json"
 			json_load_file "${adb_repdir}/adb_report.json" >/dev/null 2>&1
@@ -1182,12 +1185,12 @@ f_report()
 					json_get_var value "${key}"
 					eval "${key}=\"${value}\""
 				done
-				printf "  + %s\n  + %s\n" "Start   ::: ${start_date}, ${start_time}" "End     ::: ${end_date}, ${end_time}"
-				printf "  + %s\n  + %s %s\n" "Total   ::: ${total}" "Blocked ::: ${blocked}" "(${percent})"
+				printf "  + %s\n  + %s\n" "Start    ::: ${start_date}, ${start_time}" "End      ::: ${end_date}, ${end_time}"
+				printf "  + %s\n  + %s %s\n" "Total    ::: ${total}" "Blocked  ::: ${blocked}" "(${percent})"
 				json_select ".."
 				if json_get_type Status "top_clients" && [ "${Status}" = "array" ]
 				then
-					printf "%s\n%s\n" ":::" "::: Top 10 Clients"
+					printf "%s\n%s\n%s\n" ":::" "::: Top 10 Clients" ":::"
 					json_select "top_clients"
 					index=1
 					while json_get_type Status ${index} && [ "${Status}" = "object" ]
@@ -1200,7 +1203,7 @@ f_report()
 				json_select ".."
 				if json_get_type Status "top_domains" && [ "${Status}" = "array" ]
 				then
-					printf "%s\n%s\n" ":::" "::: Top 10 Domains"
+					printf "%s\n%s\n%s\n" ":::" "::: Top 10 Domains" ":::"
 					json_select "top_domains"
 					index=1
 					while json_get_type Status ${index} && [ "${Status}" = "object" ]
@@ -1213,7 +1216,7 @@ f_report()
 				json_select ".."
 				if json_get_type Status "top_blocked" && [ "${Status}" = "array" ]
 				then
-					printf "%s\n%s\n" ":::" "::: Top 10 Blocked Domains"
+					printf "%s\n%s\n%s\n" ":::" "::: Top 10 Blocked Domains" ":::"
 					json_select "top_blocked"
 					index=1
 					while json_get_type Status ${index} && [ "${Status}" = "object" ]
@@ -1223,8 +1226,10 @@ f_report()
 						index=$((index + 1))
 					done
 				fi
+				printf "%s\n%s\n%s\n" ":::" "::: Latest DNS Queries" ":::"
+				printf "%s\n" "${rep_latest}"
 			else
-				printf "%s\n" "::: no reporting data available yet"
+				printf "%s\n%s\n%s\n" ":::" "::: no reporting data available yet" ":::"
 			fi
 		fi
 	fi