From 45af003ba138b55834c44e3ae8638bb2c4725ae7 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Mon, 10 Apr 2017 21:15:34 +0200 Subject: [PATCH] adblock: update 2.6.0 * no longer misuse ubus/procd service object for adblock runtime information, now save all required information directly in a JSON file/format (/tmp/adb_runtime.json) * new 'status' init command to print runtime information * add a configurable interface trigger timeout for nested or slow modem/router setups, set 'adb_triggerdelay' accordingly (default 1 second) * add support for pure http download utilities like wget-nossl or uclient-fetch without libustream-ssl (http donwloads only!) * fix stop action * fix enabled/disabled action * fix country code in regional list for china * LuCI update to reflect all changes Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 2 +- net/adblock/files/README.md | 61 ++++++---------- net/adblock/files/adblock.conf | 6 +- net/adblock/files/adblock.init | 26 ++++--- net/adblock/files/adblock.sh | 123 ++++++++++++++++++++++----------- 5 files changed, 127 insertions(+), 91 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index ce4421b81..5b25930c0 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=2.5.1 +PKG_VERSION:=2.6.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/README.md b/net/adblock/files/README.md index 10199c606..fa43b1161 100644 --- a/net/adblock/files/README.md +++ b/net/adblock/files/README.md @@ -29,7 +29,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * => daily updates, approx. 15 entries * [ransomware tracker](https://ransomwaretracker.abuse.ch) * => daily updates, approx. 150 entries - * [reg_ch](https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt) + * [reg_cn](https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt) * => regional blocklist for China, daily updates, approx. 1.600 entries * [reg_pl](http://adblocklist.org) * => regional blocklist for Poland, daily updates, approx. 50 entries @@ -68,10 +68,10 @@ A lot of people already use adblocker plugins within their desktop browsers, but * additional whitelist for manual overrides, located by default in /etc/adblock/adblock.whitelist * quality checks during block list update to ensure a reliable dns backend service * minimal status & error logging to syslog, enable debug logging to receive more output -* procd based init system support (start/stop/restart/reload/suspend/resume/query) +* procd based init system support (start/stop/restart/reload/suspend/resume/query/status) * procd based hotplug support, the adblock start will be solely triggered by network interface triggers * suspend & resume adblock actions temporarily without block list reloading -* runtime information via ubus service call +* runtime information available via LuCI & via 'status' init command * query function to quickly identify blocked (sub-)domains, e.g. for whitelisting * optional: automatic block list backup & restore, backups will be (de-)compressed and restored on the fly in case of any runtime error * optional: add new adblock sources on your own via uci config @@ -79,23 +79,23 @@ A lot of people already use adblocker plugins within their desktop browsers, but ## Prerequisites * [LEDE project](https://www.lede-project.org), tested with latest stable release (LEDE 17.01) and with current LEDE snapshot * a usual setup with an enabled dns backend at minimum - dump AP modes without a working dns backend are _not_ supported -* a download utility: full versions (with ssl support) of 'wget', 'uclient-fetch', 'aria2c' or 'curl' are supported - * LEDE 17.01 or LEDE snapshot: use built-in 'uclient-fetch' or download & install the external 'wget' package +* a download utility: + * to support all blocklist sources a full version (with ssl support) of 'wget', 'uclient-fetch' with one of the 'libustream-*' ssl libraries, 'aria2c' or 'curl' is required + * for limited devices with real memory constraints, adblock provides also a plain http option and supports wget-nossl and uclient-fetch (wihout libustream-ssl), too * for more configuration options see examples below -## OpenWrt / LEDE trunk Installation & Usage -* install 'adblock' (_opkg install adblock_) and that's it - the adblock start will be automatically triggered by procd interface triggers -* start/stop/restart/reload/suspend/resume the adblock service manually with _/etc/init.d/adblock_ +## LEDE trunk Installation & Usage +* install 'adblock' (_opkg install adblock_) and that's it - the adblock start will be automatically triggered by procd interface trigger +* control the adblock service manually with _/etc/init.d/adblock_ start/stop/restart/reload/suspend/resume/status or use the LuCI frontend * enable/disable your favored block list sources in _/etc/config/adblock_ - 'adaway', 'disconnect' and 'yoyo' are enabled by default ## LuCI adblock companion package * for easy management of the various block list sources and all other adblock options you can also use a nice & efficient LuCI frontend * install 'luci-app-adblock' (_opkg install luci-app-adblock_) * the application is located in LuCI under 'Services' menu -* _Thanks to Hannu Nyman for this great adblock LuCI frontend!_ ## Tweaks -* **runtime information:** the adblock status is available via ubus service call (see example below) +* **runtime information:** the adblock status is available via _/etc/init.d/adblock status_ (see example below) * **debug logging:** for script debugging please set the config option 'adb\_debug' to '1' and check the runtime output with _logread -e "adblock"_ * **storage expansion:** to process and store all block list sources at once it might helpful to enlarge your temp directory with a swap partition => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details * **add white- / blacklist entries:** add domain white- or blacklist entries to always-allow or -deny certain (sub) domains, by default both lists are empty and located in _/etc/adblock_. Please add one domain per line - ip addresses, wildcards & regex are _not_ allowed (see example below) @@ -115,6 +115,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but * adb\_iface => set the procd interface trigger to a (list of) lan / wan interface(s) (default: 'wan') * adb\_fetch => full path to a different download utility, see example below (default: not set, use wget) * adb\_fetchparm => options for the download utility, see example below (default: not set, use wget options) + * adb\_triggerdelay => additional trigger delay in seconds before adblock processing starts (default: '1') ## Examples **change default dns backend to 'unbound':** @@ -145,37 +146,17 @@ curl: option adb_fetchparm '-s --connect-timeout 10 --insecure -o' -**receive adblock runtime information via ubus:** +**receive adblock runtime information:** 

-ubus call service get_data '{"name":"adblock"}
-This will output the active block lists and other runtime information as JSON, e.g.:
-{
-    "adblock": {
-        "adblock": {
-            "active_lists": [
-                {
-                    "reg_pl": "45",
-                    "blacklist": "144",
-                    "winspy": "138",
-                    "adaway": "379",
-                    "zeus": "399",
-                    "reg_ro": "656",
-                    "reg_ch": "1631",
-                    "yoyo": "2320",
-                    "reg_ru": "2397",
-                    "ransomware": "1681",
-                    "disconnect": "5157",
-                    "adguard": "12799"
-                }
-            ],
-            "adblock_version": "2.5.1",
-            "blocked_domains": "27746",
-            "dns_backend": "dnsmasq",
-            "last_rundate": "04.04.2017 11:02:40",
-            "system": "LEDE Reboot SNAPSHOT r3904-c3778f2647"
-        }
-    }
-}
+root@blackhole:~# /etc/init.d/adblock status
+::: adblock runtime information
+ status          : active
+ adblock_version : 2.6.0
+ blocked_domains : 113711
+ fetch_info      : wget (built-in)
+ dns_backend     : dnsmasq
+ last_rundate    : 12.04.2017 13:08:26
+ system          : LEDE Reboot SNAPSHOT r3900-399d5cf532
**cronjob for a regular block list update (/etc/crontabs/root):** diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index fd3a0855e..acc5518ac 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -5,10 +5,12 @@ config adblock 'global' option adb_enabled '1' option adb_debug '0' option adb_iface 'wan' + option adb_triggerdelay '1' option adb_whitelist '/etc/adblock/adblock.whitelist' option adb_whitelist_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}' option adb_backup '0' option adb_backupdir '/mnt' + option adb_rtfile '/tmp/adb_runtime.json' config source 'adaway' option enabled '1' @@ -36,7 +38,7 @@ config source 'disconnect' config source 'dshield' option enabled '0' - option adb_src 'http://www.dshield.org/feeds/suspiciousdomains_Low.txt' + option adb_src 'https://www.dshield.org/feeds/suspiciousdomains_Low.txt' option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' option adb_src_desc 'generic blocklist, daily updates, approx. 4.500 entries' @@ -82,7 +84,7 @@ config source 'ransomware' option adb_src_rset '\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}' option adb_src_desc 'focus on ransomware, numerous updates on the same day, approx. 130 entries' -config source 'reg_ch' +config source 'reg_cn' option enabled '0' option adb_src 'https://easylist-downloads.adblockplus.org/easylistchina+easylist.txt' option adb_src_rset '{FS=\"[|^]\"} \$0 ~/^\|\|([A-Za-z0-9_-]+\.){1,}[A-Za-z]+\^$/{print tolower(\$3)}' diff --git a/net/adblock/files/adblock.init b/net/adblock/files/adblock.init index 822317e16..ad22abb97 100755 --- a/net/adblock/files/adblock.init +++ b/net/adblock/files/adblock.init @@ -4,23 +4,19 @@ START=90 USE_PROCD=1 -EXTRA_COMMANDS="suspend resume query" +EXTRA_COMMANDS="suspend resume query status" EXTRA_HELP=" suspend Suspend adblock processing resume Resume adblock processing - query Query active blocklists for specific domain" + query Query active blocklists for specific domains + status Print runtime information" adb_init="/etc/init.d/adblock" adb_script="/usr/bin/adblock.sh" boot() { - local list iface="$(uci -q get adblock.global.adb_iface)" - - for name in ${iface} - do - list="${list} network.interface.${name}" - done - ubus -t 60 wait_for network.interface ${list} 2>/dev/null + adb_boot=1 + ubus -t 30 wait_for network.interface 2>/dev/null rc_procd start_service } @@ -28,6 +24,10 @@ start_service() { if [ $("${adb_init}" enabled; printf ${?}) -eq 0 ] then + if [ -n "${adb_boot}" ] + then + return 0 + fi procd_open_instance "adblock" procd_set_param command "${adb_script}" "${@}" procd_set_param stdout 1 @@ -39,6 +39,7 @@ start_service() stop_service() { rc_procd "${adb_script}" stop + rc_procd start_service } restart() @@ -61,10 +62,17 @@ query() rc_procd "${adb_script}" query "${1}" } +status() +{ + rc_procd "${adb_script}" status +} + service_triggers() { local iface="$(uci -q get adblock.global.adb_iface)" + local delay="$(uci -q get adblock.global.adb_triggerdelay)" + PROCD_RELOAD_DELAY=$((${delay:=1} * 1000)) for name in ${iface} do procd_add_interface_trigger "interface.*.up" "${name}" "${adb_init}" start diff --git a/net/adblock/files/adblock.sh b/net/adblock/files/adblock.sh index b2bda5619..c81f6ac0b 100755 --- a/net/adblock/files/adblock.sh +++ b/net/adblock/files/adblock.sh @@ -10,7 +10,7 @@ # LC_ALL=C PATH="/usr/sbin:/usr/bin:/sbin:/bin" -adb_ver="2.5.1" +adb_ver="2.6.0" adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')" adb_enabled=1 adb_debug=0 @@ -22,6 +22,7 @@ adb_fetch="/usr/bin/wget" adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 --no-check-certificate -O" adb_dnslist="dnsmasq unbound" adb_dnsprefix="adb_list" +adb_rtfile="/tmp/adb_runtime.json" # f_envload: load adblock environment # @@ -31,11 +32,12 @@ f_envload() # source in system library # - if [ -r "/lib/functions.sh" ] + if [ -r "/lib/functions.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ] then . "/lib/functions.sh" + . "/usr/share/libubox/jshn.sh" else - f_log "error" "system library not found" + f_log "error" "system libraries not found" fi # set dns backend environment @@ -117,11 +119,13 @@ f_envload() # f_envcheck() { + local ssl_lib + # check 'enabled' option # if [ ${adb_enabled} -ne 1 ] then - if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ] + if [ -n "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)" ] then f_rmdns f_dnsrestart @@ -132,15 +136,36 @@ f_envcheck() # check fetch utility # + ssl_lib="-" + if [ -x "${adb_fetch}" ] + then + if [ "$(readlink -fn "${adb_fetch}")" = "/usr/bin/wget-nossl" ] + then + adb_fetchparm="--no-config --quiet --no-cache --no-cookies --max-redirect=0 --timeout=10 -O" + elif [ "$(readlink -fn "/bin/wget")" = "/bin/busybox" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] + then + adb_fetch="/bin/busybox" + adb_fetchparm="-q -O" + else + ssl_lib="built-in" + fi + fi if [ ! -x "${adb_fetch}" ] && [ "$(readlink -fn "/bin/wget")" = "/bin/uclient-fetch" ] then adb_fetch="/bin/uclient-fetch" - adb_fetchparm="-q --timeout=10 --no-check-certificate -O" + if [ -f "/lib/libustream-ssl.so" ] + then + adb_fetchparm="-q --timeout=10 --no-check-certificate -O" + ssl_lib="libustream-ssl" + else + adb_fetchparm="-q --timeout=10 -O" + fi fi - if [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ] || [ ! -x "${adb_fetch}" ] || [ "$(readlink -fn "${adb_fetch}")" = "/bin/busybox" ] + if [ ! -x "${adb_fetch}" ] || [ -z "${adb_fetch}" ] || [ -z "${adb_fetchparm}" ] then - f_log "error" "required download utility with ssl support not found, e.g. install full 'wget' package" + f_log "error" "no download utility found, please install 'uclient-fetch' with 'libustream-mbedtls' or the full 'wget' package" fi + adb_fetchinfo="${adb_fetch##*/} (${ssl_lib})" # create dns hideout directory # @@ -187,6 +212,7 @@ f_rmdns() rm -f "${adb_dnsdir}/${adb_dnsprefix}"* rm -f "${adb_backupdir}/${adb_dnsprefix}"*.gz rm -rf "${adb_dnshidedir}" + > "${adb_rtfile}" fi } @@ -288,7 +314,7 @@ f_query() if [ -z "${dns_active}" ] then - printf "%s\n" "::: no active block lists found, please start adblock first" + printf "%s\n" "::: no active block lists found, please start / resume adblock first" elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ] then printf "%s\n" "::: invalid domain input, please submit a specific (sub-)domain, e.g. 'www.abc.xyz'" @@ -299,18 +325,44 @@ f_query() search="${domain//./\.}" result="$(grep -Hm1 "[/\"\.]${search}[/\"]" "${adb_dnsprefix}"* | awk -F ':|=|/|\"' '{printf(" %-20s : %s\n",$1,$4)}')" printf "%s\n" "::: distinct results for domain '${domain}'" - if [ -z "${result}" ] - then - printf "%s\n" " no match" - else - printf "%s\n" "${result}" - fi + printf "%s\n" "${result:=" no match"}" domain="${tld}" tld="${domain#*.}" done fi } +# f_status: output runtime information +# +f_status() +{ + local key keylist value + + if [ -s "${adb_rtfile}" ] + then + local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" + local dns_passive="$(find "${adb_dnshidedir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)" + + if [ -n "${dns_active}" ] + then + value="active" + elif [ -n "${dns_passive}" ] || [ -z "${dns_active}" ] + then + value="no domains blocked" + fi + printf "%s\n" "::: adblock runtime information" + printf " %-15s : %s\n" "status" "${value}" + json_load "$(cat "${adb_rtfile}" 2>/dev/null)" + json_select data + json_get_keys keylist + for key in ${keylist} + do + json_get_var value ${key} + printf " %-15s : %s\n" "${key}" "${value}" + done + fi +} + # f_log: write to syslog, exit on error # f_log() @@ -324,7 +376,7 @@ f_log() then logger -t "adblock-[${adb_ver}] ${class}" "Please check 'https://github.com/openwrt/packages/blob/master/net/adblock/files/README.md' (${adb_sysver})" f_rmtemp - if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ] + if [ -n "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)" ] then f_rmdns f_dnsrestart @@ -339,10 +391,11 @@ f_log() f_main() { local enabled url cnt sum_cnt=0 mem_total=0 - local src_name src_rset shalla_archive list active_lists active_triggers + local src_name src_rset shalla_archive mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo" 2>/dev/null)" f_log "info " "start adblock processing ..." + > "${adb_rtfile}" for src_name in ${adb_sources} do eval "enabled=\"\${enabled_${src_name}}\"" @@ -363,7 +416,7 @@ f_main() # download block list # - f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetch}, memory: ${mem_total}" + f_log "debug" "name: ${src_name}, enabled: ${enabled}, backup: ${adb_backup}, dns: ${adb_dns}, fetch: ${adb_fetchinfo}, memory: ${mem_total}" if [ "${src_name}" = "blacklist" ] then cat "${url}" 2>/dev/null > "${adb_tmpload}" @@ -446,16 +499,9 @@ f_main() fi cnt="$(wc -l < "${src_name}")" sum_cnt=$((sum_cnt + cnt)) - list="${src_name/*./}" - if [ -z "${active_lists}" ] - then - active_lists="\"${list}\":\"${cnt}\"" - else - active_lists="${active_lists},\"${list}\":\"${cnt}\"" - fi done - # restart the dns backend and write statistics to procd service instance + # restart the dns backend and export runtime information # mv -f "${adb_tmpdir}/${adb_dnsprefix}"* "${adb_dnsdir}" 2>/dev/null chown "${adb_dns}":"${adb_dns}" "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null @@ -463,21 +509,17 @@ f_main() f_dnsrestart if [ "${adb_dnsup}" = "true" ] then + json_init + json_add_object "data" + json_add_string "adblock_version" "${adb_ver}" + json_add_string "blocked_domains" "${sum_cnt}" + json_add_string "fetch_info" "${adb_fetchinfo}" + json_add_string "dns_backend" "${adb_dns}" + json_add_string "last_rundate" "$(/bin/date "+%d.%m.%Y %H:%M:%S")" + json_add_string "system" "${adb_sysver}" + json_close_object + json_dump > "${adb_rtfile}" f_log "info " "block lists with overall ${sum_cnt} domains loaded successfully (${adb_sysver})" - for name in ${adb_iface} - do - active_triggers="${active_triggers}[\"interface.*.up\",[\"if\",[\"eq\",\"interface\",\"${name}\"],[\"run_script\",\"/etc/init.d/adblock\",\"start\"],1000]]," - done - active_triggers="${active_triggers}[\"config.change\",[\"if\",[\"eq\",\"package\",\"adblock\"],[\"run_script\",\"/etc/init.d/adblock\",\"start\"],1000]]" - ubus call service set "{\"name\":\"adblock\", - \"instances\":{\"adblock\":{\"command\":[\"/usr/bin/adblock.sh\"], - \"data\":{\"active_lists\":[{${active_lists}}], - \"adblock_version\":\"${adb_ver}\", - \"blocked_domains\":\"${sum_cnt}\", - \"dns_backend\":\"${adb_dns}\", - \"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\", - \"system\":\"${adb_sysver}\"}}}, - \"triggers\":[${active_triggers}]}" else f_log "error" "dns backend restart with active block lists failed" fi @@ -507,6 +549,9 @@ case "${1}" in query) f_query "${2}" ;; + status) + f_status + ;; *) f_envcheck f_main