From 1f5ed5d0409495515eb63d84e0547725391f4a8a Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Wed, 11 Nov 2015 19:36:34 +0100 Subject: [PATCH] adblock: uci support, dynamic uhttpd instance support plus various fixes * add uci support * add dynamic uhttpd instance support (no longer rely on uhttpd config changes) * package reordering * plus various fixes Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 18 +- net/adblock/files/{etc/adblock => }/README.md | 25 +- net/adblock/files/adblock-helper.sh | 566 ++++++++++++++++++ .../files/{usr/bin => }/adblock-update.sh | 99 ++- .../files/{etc/adblock => }/adblock.blacklist | 0 net/adblock/files/adblock.conf | 76 +++ .../files/{etc/adblock => }/adblock.whitelist | 0 net/adblock/files/etc/adblock/adblock.conf | 118 ---- net/adblock/files/samples/adblock.conf.sample | 76 +++ .../adblock => }/samples/dhcp.config.sample | 0 .../adblock => }/samples/dnsmasq.conf.sample | 0 .../adblock => }/samples/firewall.user.sample | 0 .../{etc/adblock => }/samples/rc.local.sample | 0 .../adblock => }/samples/root.crontab.sample | 0 net/adblock/files/usr/bin/adblock-helper.sh | 434 -------------- 15 files changed, 790 insertions(+), 622 deletions(-) rename net/adblock/files/{etc/adblock => }/README.md (69%) create mode 100644 net/adblock/files/adblock-helper.sh rename net/adblock/files/{usr/bin => }/adblock-update.sh (59%) rename net/adblock/files/{etc/adblock => }/adblock.blacklist (100%) create mode 100644 net/adblock/files/adblock.conf rename net/adblock/files/{etc/adblock => }/adblock.whitelist (100%) delete mode 100644 net/adblock/files/etc/adblock/adblock.conf create mode 100644 net/adblock/files/samples/adblock.conf.sample rename net/adblock/files/{etc/adblock => }/samples/dhcp.config.sample (100%) rename net/adblock/files/{etc/adblock => }/samples/dnsmasq.conf.sample (100%) rename net/adblock/files/{etc/adblock => }/samples/firewall.user.sample (100%) rename net/adblock/files/{etc/adblock => }/samples/rc.local.sample (100%) rename net/adblock/files/{etc/adblock => }/samples/root.crontab.sample (100%) delete mode 100644 net/adblock/files/usr/bin/adblock-helper.sh diff --git a/net/adblock/Makefile b/net/adblock/Makefile index b37250662..c8582bd3a 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=0.11.0 +PKG_VERSION:=0.20.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken @@ -40,7 +40,7 @@ The script supports the following domain blacklist sites: endef define Package/$(PKG_NAME)/conffiles -/etc/adblock/adblock.conf +/etc/config/adblock /etc/adblock/adblock.whitelist /etc/adblock/adblock.blacklist endef @@ -56,15 +56,19 @@ endef define Package/$(PKG_NAME)/install $(INSTALL_DIR) $(1)/usr/bin - $(INSTALL_BIN) ./files/usr/bin/adblock-update.sh $(1)/usr/bin/ - $(INSTALL_DATA) ./files/usr/bin/adblock-helper.sh $(1)/usr/bin/ + $(INSTALL_BIN) ./files/adblock-update.sh $(1)/usr/bin/ + $(INSTALL_DATA) ./files/adblock-helper.sh $(1)/usr/bin/ + + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/adblock.conf $(1)/etc/config/adblock $(INSTALL_DIR) $(1)/etc/adblock - $(INSTALL_CONF) ./files/etc/adblock/adblock.* $(1)/etc/adblock/ - $(INSTALL_CONF) ./files/etc/adblock/README.md $(1)/etc/adblock/ + $(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock/ + $(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock/ + $(INSTALL_CONF) ./files/README.md $(1)/etc/adblock/ $(INSTALL_DIR) $(1)/etc/adblock/samples - $(INSTALL_CONF) ./files/etc/adblock/samples/*.sample $(1)/etc/adblock/samples/ + $(INSTALL_CONF) ./files/samples/*.sample $(1)/etc/adblock/samples/ $(INSTALL_DIR) $(1)/www/adblock $(INSTALL_DATA) ./files/www/adblock/* $(1)/www/adblock/ diff --git a/net/adblock/files/etc/adblock/README.md b/net/adblock/files/README.md similarity index 69% rename from net/adblock/files/etc/adblock/README.md rename to net/adblock/files/README.md index 19b06eb41..f5f65abd1 100644 --- a/net/adblock/files/etc/adblock/README.md +++ b/net/adblock/files/README.md @@ -5,7 +5,7 @@ A lot of people already use adblocker plugins within their desktop browsers, but what if you are using your (smart) phone, tablet, watch or any other wlan gadget... ...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router. -When the dns server on your router receives dns requests, we’ll sort out queries that ask for the [A] resource records of ad servers +When the dns server on your router receives dns requests, you’ll sort out queries that ask for the [A] resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead. ## Main Features @@ -21,34 +21,35 @@ and return the local ip address of your router and the internal web server deliv * [winhelp2002.mvps.org](http://winhelp2002.mvps.org) * blocklist parsing by fast & flexible regex rulesets * additional white- and blacklist support for manual overrides -* separate adblock loopback network interface (auto-install) -* separate uhttpd instance as pixel server (auto-install) +* separate dynamic adblock network interface +* separate dynamic uhttpd instance as pixel server * optional: quality checks and a powerful backup/restore handling to ensure a reliable dnsmasq service -* optional: adblock updates only on pre-defined interfaces +* optional: adblock updates only on pre-defined wan interfaces * optional: domain query logging as a background service to easily identify free and already blocked domains * optional: ntp time sync * optional: status & error logging (req. ntp time sync) ## Prerequisites -* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work (please adjust *min_release* accordingly) +* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work * additional software packages: * curl * wget (due to an openwrt bug still needed for certain https requests - see ticket #19621) - * busybox find with *-mtime* support (needed only for query logging/housekeeping, will be disabled if not found) + * busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found) * optional: mounted usb stick or any other storage device to overcome limited memory resources on embedded router devices * the above dependencies will be checked during package installation & script runtime, please check *logread -e "adblock"* for errors ## Usage * select & install adblock package (*opkg install adblock*) -* configure /etc/adblock/adblock.conf to your needs +* configure /etc/config/adblock to your needs * start /usr/bin/adblock-update.sh and check *logread -e "adblock"* for errors ## Distributed samples -* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see */etc/adblock/samples/dhcp.config.sample*. -* for script autostart via rc.local and /tmp resizing on the fly see */etc/adblock/samples/rc.local.sample*. -* for scheduled call of *adblock-update.sh* see */etc/adblock/samples/root.crontab.sample*. -* to redirect/force all dns queries to your router see */etc/adblock/samples/firwall.user.sample*. -* for further dnsmasq tweaks see */etc/adblock/samples/dnsmasq.conf.sample*. +* all sample configuration files stored in */etc/adblock/samples*. +* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*. +* for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample*. +* for scheduled call of *adblock-update.sh* see *root.crontab.sample*. +* to redirect/force all dns queries to your router see *firwall.user.sample*. +* for further dnsmasq tweaks see *dnsmasq.conf.sample*. Have fun! Dirk diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh new file mode 100644 index 000000000..206be6ac1 --- /dev/null +++ b/net/adblock/files/adblock-helper.sh @@ -0,0 +1,566 @@ +#!/bin/sh +############################################## +# function library used by adblock-update.sh # +# written by Dirk Brenken (dirk@brenken.org) # +############################################## + +##################################### +# f_envload: load adblock environment +# +f_envload() +{ + # source in openwrt function library + # + if [ -r "/lib/functions.sh" ] + then + . /lib/functions.sh + else + /usr/bin/logger -t "adblock[${pid}]" "error: openwrt function library not found" + f_deltemp + exit 10 + fi + + # source in openwrt json helpers library + # + if [ -r "/usr/share/libubox/jshn.sh" ] + then + . "/usr/share/libubox/jshn.sh" + else + /usr/bin/logger -t "adblock[${pid}]" "error: openwrt json helpers library not found" + f_deltemp + exit 15 + fi + + # get list with all installed openwrt packages + # + pkg_list="$(opkg list-installed 2>/dev/null)" + if [ -z "${pkg_list}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: empty openwrt package list" + f_deltemp + exit 20 + fi +} + +###################################################### +# f_envparse: parse adblock config and set environment +# +f_envparse() +{ + # function to read/set global options by callback, + # prepare list items and build option list for all others + # + config_cb() + { + local type="${1}" + local name="${2}" + if [ "${type}" = "adblock" ] + then + option_cb() + { + local option="${1}" + local value="${2}" + eval "${option}=\"${value}\"" + } + else + option_cb() + { + local option="${1}" + local value="${2}" + local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p')" + if [ -z "${opt_out}" ] + then + all_options="${all_options} ${option}" + fi + } + list_cb() + { + local list="${1}" + local value="${2}" + if [ "${list}" = "adb_wanlist" ] + then + adb_wandev="${adb_wandev} ${value}" + elif [ "${list}" = "adb_ntplist" ] + then + adb_ntpsrv="${adb_ntpsrv} ${value}" + elif [ "${list}" = "adb_catlist" ] + then + adb_cat_shalla="${adb_cat_shalla} ${value}" + fi + } + fi + } + + # function to iterate through option list, read/set all options in "enabled" sections + # + parse_config() + { + local config="${1}" + config_get switch "${config}" "enabled" + if [ "${switch}" = "1" ] + then + for option in ${all_options} + do + config_get value "${config}" "${option}" + if [ -n "${value}" ] + then + local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p')" + if [ -n "${opt_src}" ] + then + adb_sources="${adb_sources} ${value}" + else + eval "${option}=\"${value}\"" + fi + fi + done + elif [ "${config}" = "wancheck" ] + then + unset adb_wandev + elif [ "${config}" = "ntpcheck" ] + then + unset adb_ntpsrv + elif [ "${config}" = "shalla" ] + then + unset adb_cat_shalla + fi + } + + # load adblock config and start parsing functions + # + config_load adblock + config_foreach parse_config service + config_foreach parse_config source + + # set temp variables and counter + # + adb_tmpfile="$(mktemp -tu)" + adb_tmpdir="$(mktemp -d)" + cnt=0 + max_cnt=30 + max_time=60 + + # set adblock source ruleset definitions + # + rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'" + rset_end="sed '/^[#/:;_\s]*$/d'" + rset_default="${rset_start} | ${rset_end}" + rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}" + rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}" + rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}" + rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" + + # set adblock/dnsmasq destination file and format + # + adb_dnsfile="/tmp/dnsmasq.d/adlist.conf" + adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'" +} + +############################################# +# f_envcheck: check environment prerequisites +# +f_envcheck() +{ + # check adblock network device configuration + # + if [ ! -d "/sys/class/net/${adb_dev}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network device input (${adb_dev})" + f_deltemp + exit 25 + fi + + # check adblock network interface configuration + # + check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')" + banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')" + if [ -n "${check_if}" ] || [ -n "${banned_if}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network interface input (${adb_if})" + f_deltemp + exit 30 + fi + + # check adblock ip address configuration + # + check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')" + if [ -z "${check_ip}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock ip address input (${adb_ip})" + f_deltemp + exit 35 + fi + + # check adblock blacklist/whitelist configuration + # + if [ ! -r "${adb_blacklist}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: adblock blacklist not found" + f_deltemp + exit 40 + elif [ ! -r "${adb_whitelist}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: adblock whitelist not found" + f_deltemp + exit 45 + fi + + # check wan update configuration + # + if [ -n "${adb_wandev}" ] + then + wan_ok="true" + else + wan_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "info: wan update check will be disabled" + fi + + # check ntp sync configuration + # + if [ -n "${adb_ntpsrv}" ] + then + ntp_ok="true" + else + ntp_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "info: ntp time sync will be disabled" + fi + + # check backup configuration + # + adb_backupdir="${adb_backupfile%/*}" + if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ] + then + backup_ok="true" + adb_mounts="${adb_backupdir} ${adb_tmpdir}" + else + backup_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "info: backup/restore will be disabled" + fi + + # check error log configuration + # + adb_logdir="${adb_logfile%/*}" + if [ -n "${adb_logfile}" ] && [ "${adb_logfile}" = "/dev/stdout" ] + then + log_ok="true" + adb_logfile="/proc/self/fd/1" + elif [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] && [ "${ntp_ok}" = "true" ] + then + log_ok="true" + adb_mounts="${adb_mounts} ${adb_logdir}" + else + log_ok="false" + adb_logfile="/dev/null" + /usr/bin/logger -t "adblock[${pid}]" "info: error logging will be disabled" + fi + + # check dns query log configuration + # + adb_querydir="${adb_queryfile%/*}" + if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ] + then + # check find capabilities + # + check="$(find --help 2>&1 | grep "mtime")" + if [ -z "${check}" ] + then + query_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "info: busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled" + else + query_ok="true" + query_name="${adb_queryfile##*/}" + query_ip="${adb_ip//./\\.}" + query_pid="/var/run/query.pid" + adb_mounts="${adb_mounts} ${adb_querydir}" + fi + else + query_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "info: dns query logging will be disabled" + fi + + # check mount points & space requirements + # + adb_mounts="${adb_mounts} ${adb_tmpdir}" + for mp in ${adb_mounts} + do + df "${mp}" 2>/dev/null |\ + tail -n1 |\ + while read filesystem overall used available scrap + do + av_space="${available}" + if [ $((av_space)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: no space left on device/not mounted (${mp})" + exit 50 + elif [ $((av_space)) -lt $((adb_minspace)) ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: not enough space on device (${mp})" + exit 55 + fi + done + # subshell return code handling + # + rc=$? + if [ $((rc)) -ne 0 ] + then + f_deltemp + exit ${rc} + fi + done + + # check curl package dependency + # + check="$(printf "${pkg_list}" | grep "^curl")" + if [ -z "${check}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: curl package not found" + f_deltemp + exit 60 + fi + + # check wget package dependency + # + check="$(printf "${pkg_list}" | grep "^wget")" + if [ -z "${check}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: wget package not found" + f_deltemp + exit 65 + fi + + # check dynamic/volatile adblock network interface configuration + # + rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)" + if [ $((rc)) -ne 0 ] + then + json_init + json_add_string name "${adb_if}" + json_add_string ifname "${adb_dev}" + json_add_string proto "static" + json_add_array ipaddr + json_add_string "" "${adb_ip}" + json_close_array + json_close_object + ubus call network add_dynamic "$(json_dump)" + /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile network interface (${adb_if}, ${adb_ip})" + fi + + # check dynamic/volatile adblock uhttpd instance configuration + # + rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf $?)" + if [ $((rc)) -ne 0 ] + then + uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" + /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" + fi +} + +################################################### +# f_deltemp: delete temporary files and directories +# +f_deltemp() +{ + if [ -f "${adb_tmpfile}" ] + then + rm -f "${adb_tmpfile}" 2>/dev/null + fi + if [ -d "${adb_tmpdir}" ] + then + rm -rf "${adb_tmpdir}" 2>/dev/null + fi +} + +################################################################ +# f_remove: remove temporary files, start and maintain query log +# +f_remove() +{ + # delete temporary files and directories + # + f_deltemp + + # kill existing domain query log background process, + # housekeeping and start of a new process on daily basis + # + if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] + then + query_date="$(date "+%Y%m%d")" + if [ -s "${query_pid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ] + then + kill -9 $(< "${query_pid}") 2>/dev/null + > "${query_pid}" + find "${adb_backupdir}" -maxdepth 1 -type f -mtime +${adb_queryhistory} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null + /usr/bin/logger -t "adblock[${pid}]" "info: kill old query log background process and do logfile housekeeping" + fi + if [ ! -s "${query_pid}" ] + then + ( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" & + /usr/bin/logger -t "adblock[${pid}]" "info: start new domain query log background process" + fi + fi + + # final log entry + # + /usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing finished (${adb_version})" +} + +##################################################### +# f_restore: if available, restore last adlist backup +# +f_restore() +{ + if [ -z "${restore_msg}" ] + then + restore_msg="unknown" + fi + + if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ] + then + cp -f "${adb_backupfile}" "${adb_dnsfile}" 2>/dev/null + /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, adlist backup restored" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${adb_logfile}" + else + > "${adb_dnsfile}" + /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, empty adlist generated" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${adb_logfile}" + fi + + # restart dnsmasq + # + /etc/init.d/dnsmasq restart >/dev/null 2>&1 + + # remove files and exit + # + f_remove + exit 100 +} + +####################################################### +# f_wancheck: check for usable adblock update interface +# +f_wancheck() +{ + if [ "${wan_ok}" = "true" ] + then + # wait for wan update interface(s) + # + while [ $((cnt)) -le $((max_cnt)) ] + do + for dev in ${adb_wandev} + do + if [ -d "/sys/class/net/${dev}" ] + then + dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null) + if [ "${dev_out}" = "up" ] + then + /usr/bin/logger -t "adblock[${pid}]" "info: get wan/update interface: ${dev}, after ${cnt} loops" + break 2 + fi + fi + if [ $((cnt)) -eq $((max_cnt)) ] + then + /usr/bin/logger -t "adblock[${pid}]" "error: no wan/update interface(s) found (${adb_wandev})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${adb_wandev})" >> "${adb_logfile}" + restore_msg="no wan/update interface(s)" + f_restore + fi + done + sleep 1 + cnt=$((cnt + 1)) + done + fi +} + +##################################### +# f_ntpcheck: check/get ntp time sync +# +f_ntpcheck() +{ + if [ "${ntp_ok}" = "true" ] + then + # prepare ntp server pool + # + unset ntp_pool + for srv in ${adb_ntpsrv} + do + ntp_pool="${ntp_pool} -p ${srv}" + done + + # wait for ntp time sync + # + while [ $((cnt)) -le $((max_cnt)) ] + do + /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1 + rc=$? + if [ $((rc)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "info: get ntp time sync (${adb_ntpsrv}), after ${cnt} loops" + break + fi + if [ $((cnt)) -eq $((max_cnt)) ] + then + ntp_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "error: ntp time sync failed (${adb_ntpsrv})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${adb_ntpsrv})" >> "${adb_logfile}" + restore_msg="time sync failed" + f_restore + fi + sleep 1 + cnt=$((cnt + 1)) + done + fi +} + +################################################################# +# f_dnscheck: dnsmasq health check with newly generated blocklist +# +f_dnscheck() +{ + # check 1: dnsmasq startup + # + dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")" + if [ -z "${dns_status}" ] + then + # check 2: nslookup probe + # + dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")" + if [ -z "${dns_status}" ] + then + # create backup of new block list only, if both checks are OK and backup enabled + # + if [ "${backup_ok}" = "true" ] + then + cp -f "${adb_dnsfile}" "${adb_backupfile}" 2>/dev/null + /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, backup generated" + else + /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, no backup" + fi + else + restore_msg="nslookup probe failed" + f_restore + fi + else + restore_msg="dnsmasq probe failed" + f_restore + fi +} + +########################################################## +# f_footer: write footer with a few statistics to dns file +# +f_footer() +{ + adb_count="$(wc -l < "${adb_dnsfile}")" + printf "%s\n" "###################################################" >> "${adb_dnsfile}" + printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}" + printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}" + printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}" + for src in ${adb_sources} + do + url="${src//\&ruleset=*/}" + printf "%s\n" "# ${url}" >> "${adb_dnsfile}" + done + printf "%s\n" "###################################################" >> "${adb_dnsfile}" + printf "%s\n" "# domain whitelist source:" >> "${adb_dnsfile}" + printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}" + printf "%s\n" "###################################################" >> "${adb_dnsfile}" +} diff --git a/net/adblock/files/usr/bin/adblock-update.sh b/net/adblock/files/adblock-update.sh similarity index 59% rename from net/adblock/files/usr/bin/adblock-update.sh rename to net/adblock/files/adblock-update.sh index 4e9b88c00..1ca1acf61 100755 --- a/net/adblock/files/usr/bin/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -25,38 +25,21 @@ # set script version # -script_ver="0.11.0" +adb_version="0.20.0" # get current pid and script directory # pid=$$ -script_dir="$(printf "${0}" | sed 's/\(.*\)\/.*/\1/')" - -# set temp variables -# -tmp_file="$(mktemp -tu)" -tmp_dir="$(mktemp -d)" - -# source in adblock configuration -# -if [ -r "/etc/adblock/adblock.conf" ] -then - . "/etc/adblock/adblock.conf" -else - /usr/bin/logger -t "adblock[${pid}]" "adblock configuration not found" - rm -rf "${tmp_dir}" 2>/dev/null - exit 200 -fi +adb_scriptdir="${0%/*}" # source in adblock function library # -if [ -r "${script_dir}/adblock-helper.sh" ] +if [ -r "${adb_scriptdir}/adblock-helper.sh" ] then - . "${script_dir}/adblock-helper.sh" + . "${adb_scriptdir}/adblock-helper.sh" else - /usr/bin/logger -t "adblock[${pid}]" "adblock function library not found" - rm -rf "${tmp_dir}" 2>/dev/null - exit 210 + /usr/bin/logger -t "adblock[${pid}]" "error: adblock function library not found" + exit 200 fi ################ @@ -69,7 +52,15 @@ trap "restore_msg='trap error'; f_restore" 1 2 3 10 11 15 # start logging # -/usr/bin/logger -t "adblock[${pid}]" "domain adblock processing started (${script_ver})" +/usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing started (${adb_version})" + +# load environment +# +f_envload + +# parse environment +# +f_envparse # check environment # @@ -83,32 +74,36 @@ f_wancheck # f_ntpcheck -# download shallalist archive +# check/start shallalist (pre-)processing # -if [ "${shalla_ok}" = "true" ] +if [ -n "${adb_arc_shalla}" ] then - curl --insecure --max-time "${max_time}" "${shalla_url}" -o "${shalla_archive}" 2>/dev/null + # download shallalist archive + # + shalla_archive="${adb_tmpdir}/shallalist.tar.gz" + shalla_file="${adb_tmpdir}/shallalist.txt" + curl --insecure --max-time "${max_time}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null rc=$? if [ $((rc)) -eq 0 ] then - /usr/bin/logger -t "adblock[${pid}]" "shallalist archive download finished" + /usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive download finished" else - /usr/bin/logger -t "adblock[${pid}]" "shallalist archive download failed (${shalla_url})" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${shalla_url})" >> "${log_file}" + /usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive download failed (${adb_arc_shalla})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${adb_arc_shalla})" >> "${adb_logfile}" restore_msg="archive download failed" f_restore fi # extract shallalist archive # - tar -xzf "${shalla_archive}" -C "${tmp_dir}" 2>/dev/null + tar -xzf "${shalla_archive}" -C "${adb_tmpdir}" 2>/dev/null rc=$? if [ $((rc)) -eq 0 ] then - /usr/bin/logger -t "adblock[${pid}]" "shallalist archive extraction finished" + /usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive extraction finished" else - /usr/bin/logger -t "adblock[${pid}]" "shallalist archive extraction failed" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${log_file}" + /usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive extraction failed" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${adb_logfile}" restore_msg="archive extraction failed" f_restore fi @@ -116,11 +111,11 @@ then # merge selected shallalist categories # > "${shalla_file}" - for category in ${shalla_cat} + for category in ${adb_cat_shalla} do - if [ -f "${tmp_dir}/BL/${category}/domains" ] + if [ -f "${adb_tmpdir}/BL/${category}/domains" ] then - cat "${tmp_dir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null + cat "${adb_tmpdir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null rc=$? else rc=220 @@ -135,23 +130,25 @@ then # if [ $((rc)) -eq 0 ] then - /usr/bin/logger -t "adblock[${pid}]" "shallalist (pre-)processing finished (${shalla_cat})" + adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla" + /usr/bin/logger -t "adblock[${pid}]" "info: shallalist (pre-)processing finished (${adb_cat_shalla})" else - /usr/bin/logger -t "adblock[${pid}]" "shallalist category merge failed (${rc}, ${shalla_cat})" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist category merge failed (${rc}, ${shalla_cat})" >> "${log_file}" + /usr/bin/logger -t "adblock[${pid}]" "error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})" >> "${adb_logfile}" restore_msg="shallalist merge failed" f_restore fi fi -# loop through domain source list, +# loop through active adblock domain sources, # prepare output and store all extracted domains in temp file # -for src in ${adb_source} +adb_sources="${adb_sources} file://${adb_blacklist}&ruleset=rset_default" +for src in ${adb_sources} do # download selected adblock sources # - url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')" + url="${src//\&ruleset=*/}" check_url="$(printf "${url}" | sed -n '/^https:/p')" if [ -n "${check_url}" ] then @@ -166,16 +163,16 @@ do # if [ $((rc)) -eq 0 ] && [ -n "${tmp_var}" ] then - eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$rset_/g')" + eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')" tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[[:upper:]]' '[[:lower:]]')" - adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${tmp_file}" | wc -l)" - /usr/bin/logger -t "adblock[${pid}]" "source download finished (${url}, ${adb_count} entries)" + adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)" + /usr/bin/logger -t "adblock[${pid}]" "info: source download finished (${url}, ${adb_count} entries)" elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ] then - /usr/bin/logger -t "adblock[${pid}]" "empty source download finished (${url})" + /usr/bin/logger -t "adblock[${pid}]" "info: empty source download finished (${url})" else - /usr/bin/logger -t "adblock[${pid}]" "source download failed (${url})" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${log_file}" + /usr/bin/logger -t "adblock[${pid}]" "error: source download failed (${url})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${adb_logfile}" restore_msg="download failed" f_restore fi @@ -183,12 +180,12 @@ done # create empty destination file # -> "${dns_file}" +> "${adb_dnsfile}" # rewrite ad/abuse domain information to dns file, # remove duplicates and whitelist entries # -grep -vxf "${adb_whitelist}" < "${tmp_file}" | eval "${dns_format}" | sort -u 2>/dev/null >> "${dns_file}" +grep -vxf "${adb_whitelist}" < "${adb_tmpfile}" | eval "${adb_dnsformat}" | sort -u 2>/dev/null >> "${adb_dnsfile}" # write dns file footer # diff --git a/net/adblock/files/etc/adblock/adblock.blacklist b/net/adblock/files/adblock.blacklist similarity index 100% rename from net/adblock/files/etc/adblock/adblock.blacklist rename to net/adblock/files/adblock.blacklist diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf new file mode 100644 index 000000000..5fcacc0b6 --- /dev/null +++ b/net/adblock/files/adblock.conf @@ -0,0 +1,76 @@ +# adblock configuration +# +config adblock "global" + option adb_ip "192.168.2.1" + option adb_dev "eth0" + option adb_if "adblock" + option adb_domain "heise.de" + option adb_minspace "100000" + option adb_blacklist "/etc/adblock/adblock.blacklist" + option adb_whitelist "/etc/adblock/adblock.whitelist" + +config service "wancheck" + option enabled "0" + list adb_wanlist "wan" + +config service "ntpcheck" + option enabled "0" + list adb_ntplist "0.pool.ntp.org" + list adb_ntplist "1.pool.ntp.org" + list adb_ntplist "2.pool.ntp.org" + list adb_ntplist "3.pool.ntp.org" + +config service "backup" + option enabled "0" + option adb_backupfile "/tmp/adlist.backup" + +config service "errorlog" + option enabled "0" + option adb_logfile "/tmp/error.log" + +config service "querylog" + option enabled "0" + option adb_queryfile "/tmp/query.log" + option adb_queryhistory "1" + +config source "yoyo" + option enabled "1" + option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo" + +config source "malware" + option enabled "0" + option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default" + +config source "zeus" + option enabled "0" + option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default" + +config source "feodo" + option enabled "0" + option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default" + +config source "palevo" + option enabled "0" + option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default" + +config source "dshield" + option enabled "0" + option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default" + +config source "spam404" + option enabled "0" + option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404" + +config source "winhelp" + option enabled "0" + option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp" + +config source "shalla" + option enabled "0" + option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz" + list adb_catlist "adv" + list adb_catlist "costtraps" + list adb_catlist "downloads" + list adb_catlist "spyware" + list adb_catlist "tracker" + list adb_catlist "warez" diff --git a/net/adblock/files/etc/adblock/adblock.whitelist b/net/adblock/files/adblock.whitelist similarity index 100% rename from net/adblock/files/etc/adblock/adblock.whitelist rename to net/adblock/files/adblock.whitelist diff --git a/net/adblock/files/etc/adblock/adblock.conf b/net/adblock/files/etc/adblock/adblock.conf deleted file mode 100644 index 6b2e0cc2c..000000000 --- a/net/adblock/files/etc/adblock/adblock.conf +++ /dev/null @@ -1,118 +0,0 @@ -################################################### -# adblock configuration used by adblock-update.sh # -# written by Dirk Brenken (dirk@brenken.org) # -################################################### - -# adblock ip address and device information -# adb_ip => ip address of the local adblock loopback interface/uhttpd instance -# adb_dev => name of the physical adblock network device -# adb_if => name of the adblock network interface and uhttpd instance -# -adb_ip="192.168.2.1" -adb_dev="eth0" -adb_if="adblock" - -# backup file and default restore message -# backup_dir => full path to directory for logging and backups, an empty entry disables this feature -# backup_file => full path to backup file -# restore_msg => default restore message -# -backup_dir="" -backup_file="${backup_dir}/adlist.backup" -restore_msg="unknown error" - -# wan/update interface(s) -# wan_dev => space separated list of devices that are allowed for adblock updates (check /sys/class/net/), -# an empty entry disables this feature -# -wan_dev="" - -# local adblock black & white list for manual overrides -# adb_blacklist => full path to domain blacklist file -# adb_whitelist => full path to domain whitelist file -# -adb_blacklist="/etc/adblock/adblock.blacklist" -adb_whitelist="/etc/adblock/adblock.whitelist" - -# error logfile -# log_file => full path to additional logfile for error logging, -# "/dev/stdout" redirect errors to console, an empty entry disables this feature -# this feature requires ntp time sync too (see below) -# -log_file="${backup_dir}/error.log" - -# domain query logfile -# query_file => full path to additional domain query logfile, -# an empty entry or an empty backup_dir entry disables this feature -# query_history => delete query logfiles older than n days (req. busybox find with mtime support) -# -query_file="${backup_dir}/query.log" -query_history=1 -query_name="$(printf "${query_file}" | sed 's/.*\///g')" -query_ip="$(printf "${adb_ip}" | sed 's/\./\\./g')" -query_pid="/var/run/query.pid" - -# ntp time server pool -# ntp_srv => space separated list of ntp serverpools, an empty entry disables this feature -# -ntp_srv="ntp0.fau.de ntp1.fau.de ntp2.fau.de ntp3.fau.de" - -# domain for nslookup probe -# check_domain => check the newly generated domain blocklist with this domain -# -check_domain="heise.de" - -# set startup counter and download timeout defaults -# max_cnt => wait n seconds/loops for wan & ntp check in sum -# max_time => wait n seconds for every source download -# -cnt=0 -max_cnt=30 -max_time=60 - -# minimum values for environment checks -# min_release => minimum required openwrt release number -# min_space => minimum required space for backups & logfiles (in kbyte) -# -min_release=47025 -min_space=100000 - -# shallalist url, categories and local naming -# shalla_cat => space separated list of categories to use from shallalist archive -# -shalla_cat="adv costtraps downloads spyware tracker warez" -shalla_url="http://www.shallalist.de/Downloads/shallalist.tar.gz" -shalla_archive="${tmp_dir}/shallalist.tar.gz" -shalla_file="${tmp_dir}/shallalist.txt" - -# remote and local domain block list sources -# adb_source => comment out sources you don't want to use -# -unset adb_source -adb_source="${adb_source} http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=yoyo" -#adb_source="${adb_source} http://mirror1.malwaredomains.com/files/justdomains&ruleset=default" -#adb_source="${adb_source} https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=default" -#adb_source="${adb_source} https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=default" -#adb_source="${adb_source} https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=default" -#adb_source="${adb_source} http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=default" -#adb_source="${adb_source} file:///${shalla_file}&ruleset=shalla" -#adb_source="${adb_source} http://spam404bl.com/spam404scamlist.txt&ruleset=spam404" -#adb_source="${adb_source} http://winhelp2002.mvps.org/hosts.txt&ruleset=winhelp" -adb_source="${adb_source} file://${adb_blacklist}&ruleset=default" - -# adblock source ruleset definitions -# note: please keep the default settings below -# -rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'" -rset_end="sed '/^[#/:;_\s]*$/d'" -rset_default="${rset_start} | ${rset_end}" -rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}" -rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}" -rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}" -rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" - -# dnsmasq destination file and format -# note: please keep the default settings below -# -dns_file="/tmp/dnsmasq.d/adlist.conf" -dns_format="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'" diff --git a/net/adblock/files/samples/adblock.conf.sample b/net/adblock/files/samples/adblock.conf.sample new file mode 100644 index 000000000..5fcacc0b6 --- /dev/null +++ b/net/adblock/files/samples/adblock.conf.sample @@ -0,0 +1,76 @@ +# adblock configuration +# +config adblock "global" + option adb_ip "192.168.2.1" + option adb_dev "eth0" + option adb_if "adblock" + option adb_domain "heise.de" + option adb_minspace "100000" + option adb_blacklist "/etc/adblock/adblock.blacklist" + option adb_whitelist "/etc/adblock/adblock.whitelist" + +config service "wancheck" + option enabled "0" + list adb_wanlist "wan" + +config service "ntpcheck" + option enabled "0" + list adb_ntplist "0.pool.ntp.org" + list adb_ntplist "1.pool.ntp.org" + list adb_ntplist "2.pool.ntp.org" + list adb_ntplist "3.pool.ntp.org" + +config service "backup" + option enabled "0" + option adb_backupfile "/tmp/adlist.backup" + +config service "errorlog" + option enabled "0" + option adb_logfile "/tmp/error.log" + +config service "querylog" + option enabled "0" + option adb_queryfile "/tmp/query.log" + option adb_queryhistory "1" + +config source "yoyo" + option enabled "1" + option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo" + +config source "malware" + option enabled "0" + option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default" + +config source "zeus" + option enabled "0" + option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default" + +config source "feodo" + option enabled "0" + option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default" + +config source "palevo" + option enabled "0" + option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default" + +config source "dshield" + option enabled "0" + option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default" + +config source "spam404" + option enabled "0" + option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404" + +config source "winhelp" + option enabled "0" + option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp" + +config source "shalla" + option enabled "0" + option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz" + list adb_catlist "adv" + list adb_catlist "costtraps" + list adb_catlist "downloads" + list adb_catlist "spyware" + list adb_catlist "tracker" + list adb_catlist "warez" diff --git a/net/adblock/files/etc/adblock/samples/dhcp.config.sample b/net/adblock/files/samples/dhcp.config.sample similarity index 100% rename from net/adblock/files/etc/adblock/samples/dhcp.config.sample rename to net/adblock/files/samples/dhcp.config.sample diff --git a/net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample b/net/adblock/files/samples/dnsmasq.conf.sample similarity index 100% rename from net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample rename to net/adblock/files/samples/dnsmasq.conf.sample diff --git a/net/adblock/files/etc/adblock/samples/firewall.user.sample b/net/adblock/files/samples/firewall.user.sample similarity index 100% rename from net/adblock/files/etc/adblock/samples/firewall.user.sample rename to net/adblock/files/samples/firewall.user.sample diff --git a/net/adblock/files/etc/adblock/samples/rc.local.sample b/net/adblock/files/samples/rc.local.sample similarity index 100% rename from net/adblock/files/etc/adblock/samples/rc.local.sample rename to net/adblock/files/samples/rc.local.sample diff --git a/net/adblock/files/etc/adblock/samples/root.crontab.sample b/net/adblock/files/samples/root.crontab.sample similarity index 100% rename from net/adblock/files/etc/adblock/samples/root.crontab.sample rename to net/adblock/files/samples/root.crontab.sample diff --git a/net/adblock/files/usr/bin/adblock-helper.sh b/net/adblock/files/usr/bin/adblock-helper.sh deleted file mode 100644 index 9ad11555b..000000000 --- a/net/adblock/files/usr/bin/adblock-helper.sh +++ /dev/null @@ -1,434 +0,0 @@ -############################################## -# function library used by adblock-update.sh # -# written by Dirk Brenken (dirk@brenken.org) # -############################################## - -############################################# -# f_envcheck: check environment prerequisites -# -f_envcheck() -{ - # source in json helpers library - # - if [ -r "/usr/share/libubox/jshn.sh" ] - then - . "/usr/share/libubox/jshn.sh" - else - /usr/bin/logger -t "adblock[${pid}]" "json helpers library not found" - f_deltemp - exit 10 - fi - - # check adblock network device configuration - # - if [ ! -d "/sys/class/net/${adb_dev}" ] - then - /usr/bin/logger -t "adblock[${pid}]" "invalid adblock network device input (${adb_dev})" - f_deltemp - exit 15 - fi - - # check adblock network interface configuration - # - check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')" - banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')" - if [ -n "${check_if}" ] || [ -n "${banned_if}" ] - then - /usr/bin/logger -t "adblock[${pid}]" "invalid adblock network interface input (${adb_if})" - f_deltemp - exit 20 - fi - - # check adblock ip address configuration - # - check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')" - if [ -z "${check_ip}" ] - then - /usr/bin/logger -t "adblock[${pid}]" "invalid adblock ip address input (${adb_ip})" - f_deltemp - exit 25 - fi - - # check adblock blacklist/whitelist configuration - # - if [ ! -r "${adb_blacklist}" ] || [ ! -r "${adb_whitelist}" ] - then - /usr/bin/logger -t "adblock[${pid}]" "adblock blacklist or whitelist not found" - f_deltemp - exit 30 - fi - - # check wan update configuration - # - if [ -n "${wan_dev}" ] - then - wan_ok="true" - else - wan_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "wan update check will be disabled" - fi - - # check ntp sync configuration - # - if [ -n "${ntp_srv}" ] - then - ntp_ok="true" - else - ntp_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "ntp time sync will be disabled" - fi - - # check backup configuration - # - if [ -n "${backup_dir}" ] && [ -d "${backup_dir}" ] - then - backup_ok="true" - mounts="${backup_dir} ${tmp_dir}" - else - backup_ok="false" - mounts="${tmp_dir}" - /usr/bin/logger -t "adblock[${pid}]" "backup/restore will be disabled" - fi - - # check error log configuration - # - if [ "${log_file}" = "/dev/stdout" ] - then - log_ok="true" - log_file="/proc/self/fd/1" - elif [ -n "${log_file}" ] && [ "${backup_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] - then - log_ok="true" - else - log_ok="false" - log_file="/dev/null" - /usr/bin/logger -t "adblock[${pid}]" "error logging will be disabled" - fi - - # check dns query log configuration - # - if [ -n "${query_file}" ] && [ "${backup_ok}" = "true" ] - then - # check find capabilities - # - base="$(find --help 2>&1 | grep "mtime")" - if [[ -z "${base}" ]] - then - query_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "no 'find/mtime' support, dns query logging will be disabled" - else - query_ok="true" - fi - else - query_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "dns query logging will be disabled" - fi - - # check shallalist configuration - # - check_shalla="$(printf "${adb_source}" | sed -n '/.*shallalist.txt.*/p')" - if [ -n "${check_shalla}" ] - then - shalla_ok="true" - else - shalla_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "shallalist processing will be disabled" - fi - - # check mount points & space requirements - # - for mp in ${mounts} - do - df "${mp}" 2>/dev/null |\ - tail -n1 |\ - while read filesystem overall used available scrap - do - av_space="${available}" - if [ $((av_space)) -eq 0 ] - then - /usr/bin/logger -t "adblock[${pid}]" "no space left on device, not mounted (${mp})" - exit 35 - elif [ $((av_space)) -lt $((min_space)) ] - then - /usr/bin/logger -t "adblock[${pid}]" "not enough space on device (${mp})" - exit 40 - fi - done - # subshell return code handling - # - rc=$? - if [ $((rc)) -ne 0 ] - then - f_deltemp - exit ${rc} - fi - done - - # get list with all installed packages - # - pkg_list="$(opkg list-installed 2>/dev/null)" - - # check openwrt release - # - base="$(printf "${pkg_list}" | grep "^base-files" | sed 's/\(.*r\)//g')" - if [ $((base)) -lt $((min_release)) ] - then - /usr/bin/logger -t "adblock[${pid}]" "openwrt (r${wrt_release}) seems to be too old" - f_deltemp - exit 45 - fi - - # check curl package dependency - # - base="$(printf "${pkg_list}" | grep "^curl")" - if [ -z "${base}" ] - then - /usr/bin/logger -t "adblock[${pid}]" "curl package not found" - f_deltemp - exit 50 - fi - - # check wget package dependency - # - base="$(printf "${pkg_list}" | grep "^wget")" - if [ -z "${base}" ] - then - /usr/bin/logger -t "adblock[${pid}]" "wget package not found" - f_deltemp - exit 55 - fi - - # check dynamic/volatile adblock network interface configuration - # - rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)" - if [ $((rc)) -ne 0 ] - then - json_init - json_add_string name "${adb_if}" - json_add_string ifname "${adb_dev}" - json_add_string proto "static" - json_add_array ipaddr - json_add_string "" "${adb_ip}" - json_close_array - json_close_object - ubus call network add_dynamic "$(json_dump)" - /usr/bin/logger -t "adblock[${pid}]" "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})" - fi - - # check adblock uhttpd instance configuration - # - if [ -z "$(uci -q get uhttpd.${adb_if} 2>/dev/null)" ] - then - uci -q set uhttpd.${adb_if}="uhttpd" - uci -q set uhttpd.${adb_if}.listen_http="${adb_ip}:80" - uci -q set uhttpd.${adb_if}.home="/www/adblock" - uci -q set uhttpd.${adb_if}.error_page="/adblock.html" - uci -q commit uhttpd - /etc/init.d/uhttpd reload - /usr/bin/logger -t "adblock[${pid}]" "created new uhttpd instance (${adb_if}, ${adb_ip}) in /etc/config/uhttpd" - fi -} - -################################################### -# f_deltemp: delete temporary files and directories -f_deltemp() -{ - if [ -f "${tmp_file}" ] - then - rm -f "${tmp_file}" 2>/dev/null - fi - if [ -d "${tmp_dir}" ] - then - rm -rf "${tmp_dir}" 2>/dev/null - fi -} - -################################################################ -# f_remove: remove temporary files, start and maintain query log -# -f_remove() -{ - # delete temporary files and directories - # - f_deltemp - - # kill existing domain query log background process, - # housekeeping and start of a new process on daily basis - # - if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] - then - query_date="$(date "+%Y%m%d")" - if [ -s "${query_pid}" ] && [ ! -f "${query_file}.${query_date}" ] - then - kill -9 $(< "${query_pid}") 2>/dev/null - > "${query_pid}" - find "${backup_dir}" -maxdepth 1 -type f -mtime +${query_history} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null - /usr/bin/logger -t "adblock[${pid}]" "kill old query log background process and do logfile housekeeping" - fi - if [ ! -s "${query_pid}" ] - then - ( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${query_file}.${query_date}" & - /usr/bin/logger -t "adblock[${pid}]" "start new domain query log background process" - fi - fi - - # final log entry - # - /usr/bin/logger -t "adblock[${pid}]" "domain adblock processing finished (${script_ver})" -} - -##################################################### -# f_restore: if available, restore last adlist backup -# -f_restore() -{ - if [ "${backup_ok}" = "true" ] && [ -f "${backup_file}" ] - then - cp -f "${backup_file}" "${dns_file}" 2>/dev/null - /usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, adlist backup restored" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${log_file}" - else - > "${dns_file}" - /usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, empty adlist generated" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${log_file}" - fi - # restart dnsmasq - # - /etc/init.d/dnsmasq restart >/dev/null 2>&1 - - # remove files and exit - # - f_remove - exit 100 -} - -####################################################### -# f_wancheck: check for usable adblock update interface -# -f_wancheck() -{ - if [ "${wan_ok}" = "true" ] - then - # wait for wan update interface(s) - # - while [ $((cnt)) -le $((max_cnt)) ] - do - for dev in ${wan_dev} - do - dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null) - if [[ "${dev_out}" = "up" ]] - then - /usr/bin/logger -t "adblock[${pid}]" "get wan/update interface: ${dev}, after ${cnt} loops" - break 2 - elif [ $((cnt)) -eq $((max_cnt)) ] - then - /usr/bin/logger -t "adblock[${pid}]" "no wan/update interface(s) found (${wan_dev})" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${wan_dev})" >> "${log_file}" - restore_msg="no wan/update interface(s)" - f_restore - fi - done - sleep 1 - cnt=$((cnt + 1)) - done - fi -} - -##################################### -# f_ntpcheck: check/get ntp time sync -# -f_ntpcheck() -{ - if [ "${ntp_ok}" = "true" ] - then - # prepare ntp server pool - # - unset ntp_pool - for srv in ${ntp_srv} - do - ntp_pool="${ntp_pool} -p ${srv}" - done - - # wait for ntp time sync - # - while [ $((cnt)) -le $((max_cnt)) ] - do - /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1 - rc=$? - if [ $((rc)) -eq 0 ] - then - /usr/bin/logger -t "adblock[${pid}]" "get ntp time sync (${ntp_srv}), after ${cnt} loops" - break - elif [ $((cnt)) -eq $((max_cnt)) ] - then - ntp_ok="false" - /usr/bin/logger -t "adblock[${pid}]" "ntp time sync failed (${ntp_srv})" - printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${ntp_srv})" >> "${log_file}" - restore_msg="time sync failed" - f_restore - fi - sleep 1 - cnt=$((cnt + 1)) - done - fi -} - -################################################################# -# f_dnscheck: dnsmasq health check with newly generated blocklist -# -f_dnscheck() -{ - # check 1: dnsmasq startup - # - dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")" - if [ -z "${dns_status}" ] - then - # check 2: nslookup probe - # - dns_status="$(nslookup "${check_domain}" 2>/dev/null | grep "${adb_ip}")" - if [ -z "${dns_status}" ] - then - # create backup of new block list only, if both checks are OK and backup enabled - # - if [ "${backup_ok}" = "true" ] - then - cp -f "${dns_file}" "${backup_file}" 2>/dev/null - /usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded, backup generated" - else - /usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded" - fi - else - restore_msg="nslookup probe failed" - f_restore - fi - else - restore_msg="dnsmasq probe failed" - f_restore - fi -} - -########################################################## -# f_footer: write footer with a few statistics to dns file -# -f_footer() -{ - # count result of merged domain entries - # - adb_count="$(wc -l < "${dns_file}")" - - # write file footer with timestamp and merged ad count sum - # - printf "%s\n" "###################################################" >> "${dns_file}" - printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${dns_file}" - printf "%s\n" "# ${0##*/} (${script_ver}) - ${adb_count} ad/abuse domains blocked" >> "${dns_file}" - printf "%s\n" "# domain blacklist sources:" >> "${dns_file}" - for src in ${adb_source} - do - url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')" - printf "%s\n" "# ${url}" >> "${dns_file}" - done - printf "%s\n" "###################################################" >> "${dns_file}" - printf "%s\n" "# domain whitelist source:" >> "${dns_file}" - printf "%s\n" "# ${adb_whitelist}" >> "${dns_file}" - printf "%s\n" "###################################################" >> "${dns_file}" -}