From 617448dc9339bd9e90c90cbe1ae8b6aad96814bd Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Wed, 7 Jun 2017 19:31:13 +0200 Subject: [PATCH] adblock: update 2.7.0 backend: * add new 'manual mode' to re-use blocklist backups during startup, get fresh lists only via manual reload or restart action * additional free memory check during dns restart to prevent OOM errors * removed palevo tracker from default config, this tracker has been discontinued * cosmetics LuCI frontend (see luci repo): * add new 'manual mode' under extra options Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 2 +- net/adblock/files/README.md | 6 +-- net/adblock/files/adblock.conf | 6 --- net/adblock/files/adblock.sh | 68 ++++++++++++++++++++++------------ 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index 91f16c1c4..d409bcc10 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=2.6.4 +PKG_VERSION:=2.7.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 6950ec792..156d6f9ab 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -25,8 +25,6 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 1.500 entries * [openphish](https://openphish.com) * => numerous updates on the same day, approx. 1.800 entries - * [palevo tracker](https://palevotracker.abuse.ch) - * => daily updates, approx. 15 entries * [ransomware tracker](https://ransomwaretracker.abuse.ch) * => daily updates, approx. 150 entries * [reg_cn](https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt) @@ -76,7 +74,8 @@ A lot of people already use adblocker plugins within their desktop browsers, but * query function to quickly identify blocked (sub-)domains, e.g. for whitelisting * optional: force dns requests to local resolver * optional: force overall sort / duplicate removal for low memory devices (handle with care!) -* optional: automatic block list backup & restore, backups will be (de-)compressed and restored on the fly in case of any runtime error +* optional: 'manual mode' to re-use blocklist backups during startup, get fresh lists only via manual reload or restart action +* optional: automatic block list backup & restore, they will be used in case of download errors or during startup in manual mode * optional: add new adblock sources on your own via uci config ## Prerequisites @@ -120,6 +119,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * adb\_triggerdelay => additional trigger delay in seconds before adblock processing starts (default: '2') * adb\_forcedns => force dns requests to local resolver (default: '0', disabled) * adb\_forcesrt => force overall sort on low memory devices with less than 64 MB RAM (default: '0', disabled) + * adb\_manmode => do not automatically update blocklists during startup, use blocklist backups instead (default: '0', disabled) ## Examples **change default dns backend to 'unbound':** diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 0e4fd0257..c26fb9e27 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -74,12 +74,6 @@ config source 'openphish' option adb_src_rset '{FS=\"/\"} \$3 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$3)}' option adb_src_desc 'focus on phishing, numerous updates on the same day, approx. 1.800 entries' -config source 'palevo' - option enabled '0' - option adb_src 'https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist' - option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' - option adb_src_desc 'focus on palevo worm, daily updates, approx. 15 entries' - config source 'ransomware' option enabled '0' option adb_src 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt' diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index ade3c73ef..5f52203de 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,10 +10,12 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="2.6.4" +adb_ver="2.7.0" adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" adb_enabled=1 adb_debug=0 +adb_minfree=2 +adb_manmode=0 adb_forcesrt=0 adb_forcedns=0 adb_backup=0 @@ -35,7 +37,7 @@ f_envload() { local services dns_up cnt=0 - # source in system library + # source in system libraries # if [ -r "/lib/functions.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ] then @@ -253,19 +255,24 @@ f_rmdns() # f_dnsrestart() { - local cnt=0 + local dns_up mem_free cnt=0 "/etc/init.d/${adb_dns}" restart >/dev/null 2>&1 while [ ${cnt} -le 10 ] do - adb_dnsup="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@.${adb_dns}.instances.*.running")" - if [ "${adb_dnsup}" = "true" ] + dns_up="$(ubus -S call service list "{\"name\":\"${adb_dns}\"}" | jsonfilter -l1 -e "@.${adb_dns}.instances.*.running")" + if [ "${dns_up}" = "true" ] then - break + mem_free="$(awk '/^MemFree/ {print int($2/1000)}' "/proc/meminfo")" + if [ ${mem_free} -ge ${adb_minfree} ] + then + return 0 + fi fi cnt=$((cnt+1)) sleep 1 done + return 1 } # f_list: backup/restore/remove block lists @@ -302,6 +309,15 @@ f_list() fi adb_rc=${?} ;; + format) + if [ -s "${adb_tmpdir}/tmp.whitelist" ] + then + grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}" + else + eval "${adb_dnsformat}" "${adb_tmpfile}" > "${adb_dnsfile}" + fi + adb_rc=${?} + ;; esac f_log "debug" "name: ${src_name}, mode: ${mode}, count: ${cnt}, in_rc: ${in_rc}, out_rc: ${adb_rc}" } @@ -414,7 +430,7 @@ f_log() f_rmdns f_dnsrestart fi - exit 255 + exit 1 fi fi } @@ -423,12 +439,11 @@ f_log() # f_main() { - local enabled url cnt sum_cnt=0 mem_total=0 - local src_name src_rset shalla_archive - mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)" + local src_name src_rset shalla_archive enabled url cnt sum_cnt=0 + local mem_total="$(awk '/^MemTotal/ {print int($2/1000)}' "/proc/meminfo")" f_log "info " "start adblock processing ..." - f_log "debug" "action: ${adb_action}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, memory: ${mem_total}, force srt/dns: ${adb_forcesrt}/${adb_forcedns}" + f_log "debug" "action: ${adb_action}, manual_mode:${adb_manmode}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, mem_total: ${mem_total}, force_srt/_dns: ${adb_forcesrt}/${adb_forcedns}" > "${adb_rtfile}" for src_name in ${adb_sources} do @@ -449,16 +464,29 @@ f_main() continue fi + # manual mode + # + if [ ${adb_manmode} -eq 1 ] && [ -z "${adb_action}" ] + then + adb_rc=4 + f_list restore + if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] + then + f_list format + continue + fi + fi + # download block list # if [ "${src_name}" = "blacklist" ] then - cat "${url}" 2>/dev/null > "${adb_tmpload}" + cat "${url}" > "${adb_tmpload}" adb_rc=${?} elif [ "${src_name}" = "shalla" ] then shalla_archive="${adb_tmpdir}/shallalist.tar.gz" - "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" 2>/dev/null + "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}" adb_rc=${?} if [ ${adb_rc} -eq 0 ] then @@ -475,7 +503,7 @@ f_main() rm -f "${shalla_archive}" rm -rf "${adb_tmpdir}/BL" else - "${adb_fetch}" ${adb_fetchparm} "${adb_tmpload}" "${url}" 2>/dev/null + "${adb_fetch}" ${adb_fetchparm} "${adb_tmpload}" "${url}" adb_rc=${?} fi @@ -502,13 +530,7 @@ f_main() # if [ ${adb_rc} -eq 0 ] && [ -s "${adb_tmpfile}" ] then - if [ -s "${adb_tmpdir}/tmp.whitelist" ] - then - grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" > "${adb_dnsfile}" - else - eval "${adb_dnsformat}" "${adb_tmpfile}" > "${adb_dnsfile}" - fi - adb_rc=${?} + f_list format if [ ${adb_rc} -ne 0 ] then f_list remove @@ -522,7 +544,7 @@ f_main() # for src_name in $(ls -dASr "${adb_tmpdir}/${adb_dnsprefix}"* 2>/dev/null) do - if [ ${mem_total} -ge 64000 ] || [ ${adb_forcesrt} -eq 1 ] + if [ ${mem_total} -ge 64 ] || [ ${adb_forcesrt} -eq 1 ] then if [ -s "${adb_tmpdir}/blocklist.overall" ] then @@ -541,7 +563,7 @@ f_main() chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null f_rmtemp f_dnsrestart - if [ "${adb_dnsup}" = "true" ] + if [ ${?} -eq 0 ] then json_init json_add_object "data"