From 10bd039c3741ab664e36ed31115c3ad4b4595e12 Mon Sep 17 00:00:00 2001 From: dirk Date: Wed, 28 Oct 2015 00:02:23 +0100 Subject: [PATCH] powerful adblock script to block ad/abuse domains Signed-off-by: dirk --- net/adblock/Makefile | 73 +++ net/adblock/files/etc/adblock/README.md | 54 +++ .../files/etc/adblock/adblock.blacklist | 1 + net/adblock/files/etc/adblock/adblock.conf | 118 +++++ .../files/etc/adblock/adblock.whitelist | 4 + .../etc/adblock/samples/dhcp.config.sample | 7 + .../etc/adblock/samples/dnsmasq.conf.sample | 5 + .../etc/adblock/samples/firewall.user.sample | 5 + .../files/etc/adblock/samples/rc.local.sample | 26 ++ .../etc/adblock/samples/root.crontab.sample | 8 + net/adblock/files/usr/bin/adblock-helper.sh | 434 ++++++++++++++++++ net/adblock/files/usr/bin/adblock-update.sh | 209 +++++++++ net/adblock/files/www/adblock/adblock.html | 6 + net/adblock/files/www/adblock/adblock.png | Bin 0 -> 68 bytes net/adblock/files/www/adblock/index.html | 6 + 15 files changed, 956 insertions(+) create mode 100644 net/adblock/Makefile create mode 100644 net/adblock/files/etc/adblock/README.md create mode 100644 net/adblock/files/etc/adblock/adblock.blacklist create mode 100644 net/adblock/files/etc/adblock/adblock.conf create mode 100644 net/adblock/files/etc/adblock/adblock.whitelist create mode 100644 net/adblock/files/etc/adblock/samples/dhcp.config.sample create mode 100644 net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample create mode 100644 net/adblock/files/etc/adblock/samples/firewall.user.sample create mode 100644 net/adblock/files/etc/adblock/samples/rc.local.sample create mode 100644 net/adblock/files/etc/adblock/samples/root.crontab.sample create mode 100644 net/adblock/files/usr/bin/adblock-helper.sh create mode 100755 net/adblock/files/usr/bin/adblock-update.sh create mode 100644 net/adblock/files/www/adblock/adblock.html create mode 100644 net/adblock/files/www/adblock/adblock.png create mode 100644 net/adblock/files/www/adblock/index.html diff --git a/net/adblock/Makefile b/net/adblock/Makefile new file mode 100644 index 000000000..b37250662 --- /dev/null +++ b/net/adblock/Makefile @@ -0,0 +1,73 @@ +# +# Copyright (C) 2015 OpenWrt.org +# +# This is free software, licensed under the GNU General Public License v3. +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=adblock +PKG_VERSION:=0.11.0 +PKG_RELEASE:=1 +PKG_LICENSE:=GPL-3.0+ +PKG_MAINTAINER:=Dirk Brenken + +include $(INCLUDE_DIR)/package.mk + +define Package/$(PKG_NAME) + SECTION:=net + CATEGORY:=Network + TITLE:=powerful adblock script to block ad/abuse domains + DEPENDS:=+curl +wget + PKGARCH:=all +endef + +define Package/$(PKG_NAME)/description +powerful adblock script to block ad/abuse domains +When the dns server on your router receives dns requests, we will sort out queries that ask for the [A] resource records of ad servers +and return the local ip address of your router and the internal web server delivers a transparent pixel instead. +The script supports the following domain blacklist sites: + http://pgl.yoyo.org/adservers + http://malwaredomains.com + https://zeustracker.abuse.ch + https://feodotracker.abuse.ch + https://palevotracker.abuse.ch + http://dshield.org + http://www.shallalist.de + http://www.spam404.com + http://winhelp2002.mvps.org + +endef + +define Package/$(PKG_NAME)/conffiles +/etc/adblock/adblock.conf +/etc/adblock/adblock.whitelist +/etc/adblock/adblock.blacklist +endef + +define Build/Prepare +endef + +define Build/Configure +endef + +define Build/Compile +endef + +define Package/$(PKG_NAME)/install + $(INSTALL_DIR) $(1)/usr/bin + $(INSTALL_BIN) ./files/usr/bin/adblock-update.sh $(1)/usr/bin/ + $(INSTALL_DATA) ./files/usr/bin/adblock-helper.sh $(1)/usr/bin/ + + $(INSTALL_DIR) $(1)/etc/adblock + $(INSTALL_CONF) ./files/etc/adblock/adblock.* $(1)/etc/adblock/ + $(INSTALL_CONF) ./files/etc/adblock/README.md $(1)/etc/adblock/ + + $(INSTALL_DIR) $(1)/etc/adblock/samples + $(INSTALL_CONF) ./files/etc/adblock/samples/*.sample $(1)/etc/adblock/samples/ + + $(INSTALL_DIR) $(1)/www/adblock + $(INSTALL_DATA) ./files/www/adblock/* $(1)/www/adblock/ +endef + +$(eval $(call BuildPackage,$(PKG_NAME))) diff --git a/net/adblock/files/etc/adblock/README.md b/net/adblock/files/etc/adblock/README.md new file mode 100644 index 000000000..19b06eb41 --- /dev/null +++ b/net/adblock/files/etc/adblock/README.md @@ -0,0 +1,54 @@ +# adblock script for openwrt + +## Description +A lot of people already use adblocker plugins within their desktop browsers, +but what if you are using your (smart) phone, tablet, watch or any other wlan gadget... +...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router. + +When the dns server on your router receives dns requests, we’ll sort out queries that ask for the [A] resource records of ad servers +and return the local ip address of your router and the internal web server delivers a transparent pixel instead. + +## Main Features +* support of the following domain blacklist sites (free for private usage, for commercial use please check their individual licenses): + * [pgl.yoyo.org](http://pgl.yoyo.org/adservers) + * [malwaredomains.com](http://malwaredomains.com) + * [zeustracker.abuse.ch](https://zeustracker.abuse.ch) + * [feodotracker.abuse.ch](https://feodotracker.abuse.ch) + * [palevotracker.abuse.ch](https://palevotracker.abuse.ch) + * [dshield.org](http://dshield.org) + * [shallalist.de](http://www.shallalist.de) (tested with the categories "adv" "costtraps" "downloads" "spyware" "tracker" "warez") + * [spam404.com](http://www.spam404.com) + * [winhelp2002.mvps.org](http://winhelp2002.mvps.org) +* blocklist parsing by fast & flexible regex rulesets +* additional white- and blacklist support for manual overrides +* separate adblock loopback network interface (auto-install) +* separate uhttpd instance as pixel server (auto-install) +* optional: quality checks and a powerful backup/restore handling to ensure a reliable dnsmasq service +* optional: adblock updates only on pre-defined interfaces +* optional: domain query logging as a background service to easily identify free and already blocked domains +* optional: ntp time sync +* optional: status & error logging (req. ntp time sync) + +## Prerequisites +* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work (please adjust *min_release* accordingly) +* additional software packages: + * curl + * wget (due to an openwrt bug still needed for certain https requests - see ticket #19621) + * busybox find with *-mtime* support (needed only for query logging/housekeeping, will be disabled if not found) +* optional: mounted usb stick or any other storage device to overcome limited memory resources on embedded router devices +* the above dependencies will be checked during package installation & script runtime, please check *logread -e "adblock"* for errors + +## Usage +* select & install adblock package (*opkg install adblock*) +* configure /etc/adblock/adblock.conf to your needs +* start /usr/bin/adblock-update.sh and check *logread -e "adblock"* for errors + +## Distributed samples +* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see */etc/adblock/samples/dhcp.config.sample*. +* for script autostart via rc.local and /tmp resizing on the fly see */etc/adblock/samples/rc.local.sample*. +* for scheduled call of *adblock-update.sh* see */etc/adblock/samples/root.crontab.sample*. +* to redirect/force all dns queries to your router see */etc/adblock/samples/firwall.user.sample*. +* for further dnsmasq tweaks see */etc/adblock/samples/dnsmasq.conf.sample*. + +Have fun! +Dirk diff --git a/net/adblock/files/etc/adblock/adblock.blacklist b/net/adblock/files/etc/adblock/adblock.blacklist new file mode 100644 index 000000000..3df75830c --- /dev/null +++ b/net/adblock/files/etc/adblock/adblock.blacklist @@ -0,0 +1 @@ +bild.de diff --git a/net/adblock/files/etc/adblock/adblock.conf b/net/adblock/files/etc/adblock/adblock.conf new file mode 100644 index 000000000..6b2e0cc2c --- /dev/null +++ b/net/adblock/files/etc/adblock/adblock.conf @@ -0,0 +1,118 @@ +################################################### +# adblock configuration used by adblock-update.sh # +# written by Dirk Brenken (dirk@brenken.org) # +################################################### + +# adblock ip address and device information +# adb_ip => ip address of the local adblock loopback interface/uhttpd instance +# adb_dev => name of the physical adblock network device +# adb_if => name of the adblock network interface and uhttpd instance +# +adb_ip="192.168.2.1" +adb_dev="eth0" +adb_if="adblock" + +# backup file and default restore message +# backup_dir => full path to directory for logging and backups, an empty entry disables this feature +# backup_file => full path to backup file +# restore_msg => default restore message +# +backup_dir="" +backup_file="${backup_dir}/adlist.backup" +restore_msg="unknown error" + +# wan/update interface(s) +# wan_dev => space separated list of devices that are allowed for adblock updates (check /sys/class/net/), +# an empty entry disables this feature +# +wan_dev="" + +# local adblock black & white list for manual overrides +# adb_blacklist => full path to domain blacklist file +# adb_whitelist => full path to domain whitelist file +# +adb_blacklist="/etc/adblock/adblock.blacklist" +adb_whitelist="/etc/adblock/adblock.whitelist" + +# error logfile +# log_file => full path to additional logfile for error logging, +# "/dev/stdout" redirect errors to console, an empty entry disables this feature +# this feature requires ntp time sync too (see below) +# +log_file="${backup_dir}/error.log" + +# domain query logfile +# query_file => full path to additional domain query logfile, +# an empty entry or an empty backup_dir entry disables this feature +# query_history => delete query logfiles older than n days (req. busybox find with mtime support) +# +query_file="${backup_dir}/query.log" +query_history=1 +query_name="$(printf "${query_file}" | sed 's/.*\///g')" +query_ip="$(printf "${adb_ip}" | sed 's/\./\\./g')" +query_pid="/var/run/query.pid" + +# ntp time server pool +# ntp_srv => space separated list of ntp serverpools, an empty entry disables this feature +# +ntp_srv="ntp0.fau.de ntp1.fau.de ntp2.fau.de ntp3.fau.de" + +# domain for nslookup probe +# check_domain => check the newly generated domain blocklist with this domain +# +check_domain="heise.de" + +# set startup counter and download timeout defaults +# max_cnt => wait n seconds/loops for wan & ntp check in sum +# max_time => wait n seconds for every source download +# +cnt=0 +max_cnt=30 +max_time=60 + +# minimum values for environment checks +# min_release => minimum required openwrt release number +# min_space => minimum required space for backups & logfiles (in kbyte) +# +min_release=47025 +min_space=100000 + +# shallalist url, categories and local naming +# shalla_cat => space separated list of categories to use from shallalist archive +# +shalla_cat="adv costtraps downloads spyware tracker warez" +shalla_url="http://www.shallalist.de/Downloads/shallalist.tar.gz" +shalla_archive="${tmp_dir}/shallalist.tar.gz" +shalla_file="${tmp_dir}/shallalist.txt" + +# remote and local domain block list sources +# adb_source => comment out sources you don't want to use +# +unset adb_source +adb_source="${adb_source} http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=yoyo" +#adb_source="${adb_source} http://mirror1.malwaredomains.com/files/justdomains&ruleset=default" +#adb_source="${adb_source} https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=default" +#adb_source="${adb_source} https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=default" +#adb_source="${adb_source} https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=default" +#adb_source="${adb_source} http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=default" +#adb_source="${adb_source} file:///${shalla_file}&ruleset=shalla" +#adb_source="${adb_source} http://spam404bl.com/spam404scamlist.txt&ruleset=spam404" +#adb_source="${adb_source} http://winhelp2002.mvps.org/hosts.txt&ruleset=winhelp" +adb_source="${adb_source} file://${adb_blacklist}&ruleset=default" + +# adblock source ruleset definitions +# note: please keep the default settings below +# +rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'" +rset_end="sed '/^[#/:;_\s]*$/d'" +rset_default="${rset_start} | ${rset_end}" +rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}" +rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}" +rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}" +rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" + +# dnsmasq destination file and format +# note: please keep the default settings below +# +dns_file="/tmp/dnsmasq.d/adlist.conf" +dns_format="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'" diff --git a/net/adblock/files/etc/adblock/adblock.whitelist b/net/adblock/files/etc/adblock/adblock.whitelist new file mode 100644 index 000000000..42e0a57c3 --- /dev/null +++ b/net/adblock/files/etc/adblock/adblock.whitelist @@ -0,0 +1,4 @@ +downloads.openwrt.org +ftp.de.debian.org +download.eclipse.org +dl.sourceforge.net diff --git a/net/adblock/files/etc/adblock/samples/dhcp.config.sample b/net/adblock/files/etc/adblock/samples/dhcp.config.sample new file mode 100644 index 000000000..40de7d4a2 --- /dev/null +++ b/net/adblock/files/etc/adblock/samples/dhcp.config.sample @@ -0,0 +1,7 @@ +# tweaks for dnsmasq +# configuration found in /etc/config/dhcp +# +config dnsmasq + option cachesize '1000' + option filterwin2k '0' + option logqueries '1' diff --git a/net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample b/net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample new file mode 100644 index 000000000..d92adc893 --- /dev/null +++ b/net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample @@ -0,0 +1,5 @@ +# tell DHCP clients to not ask for proxy information +# some clients - like Win7 - will constantly ask if not told "No!" +# configuration found in /etc/dnsmasq +# +dhcp-option=252,"\n" diff --git a/net/adblock/files/etc/adblock/samples/firewall.user.sample b/net/adblock/files/etc/adblock/samples/firewall.user.sample new file mode 100644 index 000000000..823c0147d --- /dev/null +++ b/net/adblock/files/etc/adblock/samples/firewall.user.sample @@ -0,0 +1,5 @@ +# redirect/force all dns queries to port 53 of your router +# configuration found in /etc/firewall.user +# +iptables -t nat -I PREROUTING -p tcp --dport 53 -j REDIRECT --to-ports 53 +iptables -t nat -I PREROUTING -p udp --dport 53 -j REDIRECT --to-ports 53 diff --git a/net/adblock/files/etc/adblock/samples/rc.local.sample b/net/adblock/files/etc/adblock/samples/rc.local.sample new file mode 100644 index 000000000..890779e58 --- /dev/null +++ b/net/adblock/files/etc/adblock/samples/rc.local.sample @@ -0,0 +1,26 @@ +# sample startup script +# configuration found in /etc/rc.local +# + +# start logging +# +/usr/bin/logger -t rc.local "start rc.local processing" + +# set home directory +# +export HOME=/root + +# resize /tmp partition to 256 MB +# +/usr/bin/logger -t rc.local "resize /tmp partition to 256 MB" +mount tmpfs /tmp -t tmpfs -o remount,nosuid,nodev,noatime,size=256M + +# start adblock script +# +/usr/bin/logger -t rc.local "start adblock script" +/usr/bin/adblock-update.sh >/dev/null 2>&1 + +# write log and exit +# +/usr/bin/logger -t rc.local "finish rc.local processing" +exit 0 diff --git a/net/adblock/files/etc/adblock/samples/root.crontab.sample b/net/adblock/files/etc/adblock/samples/root.crontab.sample new file mode 100644 index 000000000..bc7bde7cb --- /dev/null +++ b/net/adblock/files/etc/adblock/samples/root.crontab.sample @@ -0,0 +1,8 @@ +# sample crontab script +# configuration found in /etc/crontabs/root +# + +# start adblock script twice a day +# +0 06 * * * /usr/bin/adblock-update.sh & +0 22 * * * /usr/bin/adblock-update.sh & diff --git a/net/adblock/files/usr/bin/adblock-helper.sh b/net/adblock/files/usr/bin/adblock-helper.sh new file mode 100644 index 000000000..9ad11555b --- /dev/null +++ b/net/adblock/files/usr/bin/adblock-helper.sh @@ -0,0 +1,434 @@ +############################################## +# function library used by adblock-update.sh # +# written by Dirk Brenken (dirk@brenken.org) # +############################################## + +############################################# +# f_envcheck: check environment prerequisites +# +f_envcheck() +{ + # source in json helpers library + # + if [ -r "/usr/share/libubox/jshn.sh" ] + then + . "/usr/share/libubox/jshn.sh" + else + /usr/bin/logger -t "adblock[${pid}]" "json helpers library not found" + f_deltemp + exit 10 + fi + + # check adblock network device configuration + # + if [ ! -d "/sys/class/net/${adb_dev}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "invalid adblock network device input (${adb_dev})" + f_deltemp + exit 15 + fi + + # check adblock network interface configuration + # + check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')" + banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')" + if [ -n "${check_if}" ] || [ -n "${banned_if}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "invalid adblock network interface input (${adb_if})" + f_deltemp + exit 20 + fi + + # check adblock ip address configuration + # + check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')" + if [ -z "${check_ip}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "invalid adblock ip address input (${adb_ip})" + f_deltemp + exit 25 + fi + + # check adblock blacklist/whitelist configuration + # + if [ ! -r "${adb_blacklist}" ] || [ ! -r "${adb_whitelist}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "adblock blacklist or whitelist not found" + f_deltemp + exit 30 + fi + + # check wan update configuration + # + if [ -n "${wan_dev}" ] + then + wan_ok="true" + else + wan_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "wan update check will be disabled" + fi + + # check ntp sync configuration + # + if [ -n "${ntp_srv}" ] + then + ntp_ok="true" + else + ntp_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "ntp time sync will be disabled" + fi + + # check backup configuration + # + if [ -n "${backup_dir}" ] && [ -d "${backup_dir}" ] + then + backup_ok="true" + mounts="${backup_dir} ${tmp_dir}" + else + backup_ok="false" + mounts="${tmp_dir}" + /usr/bin/logger -t "adblock[${pid}]" "backup/restore will be disabled" + fi + + # check error log configuration + # + if [ "${log_file}" = "/dev/stdout" ] + then + log_ok="true" + log_file="/proc/self/fd/1" + elif [ -n "${log_file}" ] && [ "${backup_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] + then + log_ok="true" + else + log_ok="false" + log_file="/dev/null" + /usr/bin/logger -t "adblock[${pid}]" "error logging will be disabled" + fi + + # check dns query log configuration + # + if [ -n "${query_file}" ] && [ "${backup_ok}" = "true" ] + then + # check find capabilities + # + base="$(find --help 2>&1 | grep "mtime")" + if [[ -z "${base}" ]] + then + query_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "no 'find/mtime' support, dns query logging will be disabled" + else + query_ok="true" + fi + else + query_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "dns query logging will be disabled" + fi + + # check shallalist configuration + # + check_shalla="$(printf "${adb_source}" | sed -n '/.*shallalist.txt.*/p')" + if [ -n "${check_shalla}" ] + then + shalla_ok="true" + else + shalla_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "shallalist processing will be disabled" + fi + + # check mount points & space requirements + # + for mp in ${mounts} + do + df "${mp}" 2>/dev/null |\ + tail -n1 |\ + while read filesystem overall used available scrap + do + av_space="${available}" + if [ $((av_space)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "no space left on device, not mounted (${mp})" + exit 35 + elif [ $((av_space)) -lt $((min_space)) ] + then + /usr/bin/logger -t "adblock[${pid}]" "not enough space on device (${mp})" + exit 40 + fi + done + # subshell return code handling + # + rc=$? + if [ $((rc)) -ne 0 ] + then + f_deltemp + exit ${rc} + fi + done + + # get list with all installed packages + # + pkg_list="$(opkg list-installed 2>/dev/null)" + + # check openwrt release + # + base="$(printf "${pkg_list}" | grep "^base-files" | sed 's/\(.*r\)//g')" + if [ $((base)) -lt $((min_release)) ] + then + /usr/bin/logger -t "adblock[${pid}]" "openwrt (r${wrt_release}) seems to be too old" + f_deltemp + exit 45 + fi + + # check curl package dependency + # + base="$(printf "${pkg_list}" | grep "^curl")" + if [ -z "${base}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "curl package not found" + f_deltemp + exit 50 + fi + + # check wget package dependency + # + base="$(printf "${pkg_list}" | grep "^wget")" + if [ -z "${base}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "wget package not found" + f_deltemp + exit 55 + fi + + # check dynamic/volatile adblock network interface configuration + # + rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)" + if [ $((rc)) -ne 0 ] + then + json_init + json_add_string name "${adb_if}" + json_add_string ifname "${adb_dev}" + json_add_string proto "static" + json_add_array ipaddr + json_add_string "" "${adb_ip}" + json_close_array + json_close_object + ubus call network add_dynamic "$(json_dump)" + /usr/bin/logger -t "adblock[${pid}]" "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})" + fi + + # check adblock uhttpd instance configuration + # + if [ -z "$(uci -q get uhttpd.${adb_if} 2>/dev/null)" ] + then + uci -q set uhttpd.${adb_if}="uhttpd" + uci -q set uhttpd.${adb_if}.listen_http="${adb_ip}:80" + uci -q set uhttpd.${adb_if}.home="/www/adblock" + uci -q set uhttpd.${adb_if}.error_page="/adblock.html" + uci -q commit uhttpd + /etc/init.d/uhttpd reload + /usr/bin/logger -t "adblock[${pid}]" "created new uhttpd instance (${adb_if}, ${adb_ip}) in /etc/config/uhttpd" + fi +} + +################################################### +# f_deltemp: delete temporary files and directories +f_deltemp() +{ + if [ -f "${tmp_file}" ] + then + rm -f "${tmp_file}" 2>/dev/null + fi + if [ -d "${tmp_dir}" ] + then + rm -rf "${tmp_dir}" 2>/dev/null + fi +} + +################################################################ +# f_remove: remove temporary files, start and maintain query log +# +f_remove() +{ + # delete temporary files and directories + # + f_deltemp + + # kill existing domain query log background process, + # housekeeping and start of a new process on daily basis + # + if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ] + then + query_date="$(date "+%Y%m%d")" + if [ -s "${query_pid}" ] && [ ! -f "${query_file}.${query_date}" ] + then + kill -9 $(< "${query_pid}") 2>/dev/null + > "${query_pid}" + find "${backup_dir}" -maxdepth 1 -type f -mtime +${query_history} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null + /usr/bin/logger -t "adblock[${pid}]" "kill old query log background process and do logfile housekeeping" + fi + if [ ! -s "${query_pid}" ] + then + ( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${query_file}.${query_date}" & + /usr/bin/logger -t "adblock[${pid}]" "start new domain query log background process" + fi + fi + + # final log entry + # + /usr/bin/logger -t "adblock[${pid}]" "domain adblock processing finished (${script_ver})" +} + +##################################################### +# f_restore: if available, restore last adlist backup +# +f_restore() +{ + if [ "${backup_ok}" = "true" ] && [ -f "${backup_file}" ] + then + cp -f "${backup_file}" "${dns_file}" 2>/dev/null + /usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, adlist backup restored" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${log_file}" + else + > "${dns_file}" + /usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, empty adlist generated" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${log_file}" + fi + # restart dnsmasq + # + /etc/init.d/dnsmasq restart >/dev/null 2>&1 + + # remove files and exit + # + f_remove + exit 100 +} + +####################################################### +# f_wancheck: check for usable adblock update interface +# +f_wancheck() +{ + if [ "${wan_ok}" = "true" ] + then + # wait for wan update interface(s) + # + while [ $((cnt)) -le $((max_cnt)) ] + do + for dev in ${wan_dev} + do + dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null) + if [[ "${dev_out}" = "up" ]] + then + /usr/bin/logger -t "adblock[${pid}]" "get wan/update interface: ${dev}, after ${cnt} loops" + break 2 + elif [ $((cnt)) -eq $((max_cnt)) ] + then + /usr/bin/logger -t "adblock[${pid}]" "no wan/update interface(s) found (${wan_dev})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${wan_dev})" >> "${log_file}" + restore_msg="no wan/update interface(s)" + f_restore + fi + done + sleep 1 + cnt=$((cnt + 1)) + done + fi +} + +##################################### +# f_ntpcheck: check/get ntp time sync +# +f_ntpcheck() +{ + if [ "${ntp_ok}" = "true" ] + then + # prepare ntp server pool + # + unset ntp_pool + for srv in ${ntp_srv} + do + ntp_pool="${ntp_pool} -p ${srv}" + done + + # wait for ntp time sync + # + while [ $((cnt)) -le $((max_cnt)) ] + do + /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1 + rc=$? + if [ $((rc)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "get ntp time sync (${ntp_srv}), after ${cnt} loops" + break + elif [ $((cnt)) -eq $((max_cnt)) ] + then + ntp_ok="false" + /usr/bin/logger -t "adblock[${pid}]" "ntp time sync failed (${ntp_srv})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${ntp_srv})" >> "${log_file}" + restore_msg="time sync failed" + f_restore + fi + sleep 1 + cnt=$((cnt + 1)) + done + fi +} + +################################################################# +# f_dnscheck: dnsmasq health check with newly generated blocklist +# +f_dnscheck() +{ + # check 1: dnsmasq startup + # + dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")" + if [ -z "${dns_status}" ] + then + # check 2: nslookup probe + # + dns_status="$(nslookup "${check_domain}" 2>/dev/null | grep "${adb_ip}")" + if [ -z "${dns_status}" ] + then + # create backup of new block list only, if both checks are OK and backup enabled + # + if [ "${backup_ok}" = "true" ] + then + cp -f "${dns_file}" "${backup_file}" 2>/dev/null + /usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded, backup generated" + else + /usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded" + fi + else + restore_msg="nslookup probe failed" + f_restore + fi + else + restore_msg="dnsmasq probe failed" + f_restore + fi +} + +########################################################## +# f_footer: write footer with a few statistics to dns file +# +f_footer() +{ + # count result of merged domain entries + # + adb_count="$(wc -l < "${dns_file}")" + + # write file footer with timestamp and merged ad count sum + # + printf "%s\n" "###################################################" >> "${dns_file}" + printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${dns_file}" + printf "%s\n" "# ${0##*/} (${script_ver}) - ${adb_count} ad/abuse domains blocked" >> "${dns_file}" + printf "%s\n" "# domain blacklist sources:" >> "${dns_file}" + for src in ${adb_source} + do + url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')" + printf "%s\n" "# ${url}" >> "${dns_file}" + done + printf "%s\n" "###################################################" >> "${dns_file}" + printf "%s\n" "# domain whitelist source:" >> "${dns_file}" + printf "%s\n" "# ${adb_whitelist}" >> "${dns_file}" + printf "%s\n" "###################################################" >> "${dns_file}" +} diff --git a/net/adblock/files/usr/bin/adblock-update.sh b/net/adblock/files/usr/bin/adblock-update.sh new file mode 100755 index 000000000..4e9b88c00 --- /dev/null +++ b/net/adblock/files/usr/bin/adblock-update.sh @@ -0,0 +1,209 @@ +#!/bin/sh +####################################################### +# ad/abuse domain blocking script for dnsmasq/openwrt # +# written by Dirk Brenken (dirk@brenken.org) # +####################################################### + +# LICENSE +# ======== +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +############### +# environment # +############### + +# set script version +# +script_ver="0.11.0" + +# get current pid and script directory +# +pid=$$ +script_dir="$(printf "${0}" | sed 's/\(.*\)\/.*/\1/')" + +# set temp variables +# +tmp_file="$(mktemp -tu)" +tmp_dir="$(mktemp -d)" + +# source in adblock configuration +# +if [ -r "/etc/adblock/adblock.conf" ] +then + . "/etc/adblock/adblock.conf" +else + /usr/bin/logger -t "adblock[${pid}]" "adblock configuration not found" + rm -rf "${tmp_dir}" 2>/dev/null + exit 200 +fi + +# source in adblock function library +# +if [ -r "${script_dir}/adblock-helper.sh" ] +then + . "${script_dir}/adblock-helper.sh" +else + /usr/bin/logger -t "adblock[${pid}]" "adblock function library not found" + rm -rf "${tmp_dir}" 2>/dev/null + exit 210 +fi + +################ +# main program # +################ + +# call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM) +# +trap "restore_msg='trap error'; f_restore" 1 2 3 10 11 15 + +# start logging +# +/usr/bin/logger -t "adblock[${pid}]" "domain adblock processing started (${script_ver})" + +# check environment +# +f_envcheck + +# check wan update interface(s) +# +f_wancheck + +# check for ntp time sync +# +f_ntpcheck + +# download shallalist archive +# +if [ "${shalla_ok}" = "true" ] +then + curl --insecure --max-time "${max_time}" "${shalla_url}" -o "${shalla_archive}" 2>/dev/null + rc=$? + if [ $((rc)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "shallalist archive download finished" + else + /usr/bin/logger -t "adblock[${pid}]" "shallalist archive download failed (${shalla_url})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${shalla_url})" >> "${log_file}" + restore_msg="archive download failed" + f_restore + fi + + # extract shallalist archive + # + tar -xzf "${shalla_archive}" -C "${tmp_dir}" 2>/dev/null + rc=$? + if [ $((rc)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "shallalist archive extraction finished" + else + /usr/bin/logger -t "adblock[${pid}]" "shallalist archive extraction failed" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${log_file}" + restore_msg="archive extraction failed" + f_restore + fi + + # merge selected shallalist categories + # + > "${shalla_file}" + for category in ${shalla_cat} + do + if [ -f "${tmp_dir}/BL/${category}/domains" ] + then + cat "${tmp_dir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null + rc=$? + else + rc=220 + fi + if [ $((rc)) -ne 0 ] + then + break + fi + done + + # finish shallalist (pre-)processing + # + if [ $((rc)) -eq 0 ] + then + /usr/bin/logger -t "adblock[${pid}]" "shallalist (pre-)processing finished (${shalla_cat})" + else + /usr/bin/logger -t "adblock[${pid}]" "shallalist category merge failed (${rc}, ${shalla_cat})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist category merge failed (${rc}, ${shalla_cat})" >> "${log_file}" + restore_msg="shallalist merge failed" + f_restore + fi +fi + +# loop through domain source list, +# prepare output and store all extracted domains in temp file +# +for src in ${adb_source} +do + # download selected adblock sources + # + url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')" + check_url="$(printf "${url}" | sed -n '/^https:/p')" + if [ -n "${check_url}" ] + then + tmp_var="$(wget --timeout="${max_time}" --tries=1 --output-document=- "${url}" 2>/dev/null)" + rc=$? + else + tmp_var="$(curl --insecure --max-time "${max_time}" "${url}" 2>/dev/null)" + rc=$? + fi + + # check download result and prepare domain output by regex patterns + # + if [ $((rc)) -eq 0 ] && [ -n "${tmp_var}" ] + then + eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$rset_/g')" + tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[[:upper:]]' '[[:lower:]]')" + adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${tmp_file}" | wc -l)" + /usr/bin/logger -t "adblock[${pid}]" "source download finished (${url}, ${adb_count} entries)" + elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ] + then + /usr/bin/logger -t "adblock[${pid}]" "empty source download finished (${url})" + else + /usr/bin/logger -t "adblock[${pid}]" "source download failed (${url})" + printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${log_file}" + restore_msg="download failed" + f_restore + fi +done + +# create empty destination file +# +> "${dns_file}" + +# rewrite ad/abuse domain information to dns file, +# remove duplicates and whitelist entries +# +grep -vxf "${adb_whitelist}" < "${tmp_file}" | eval "${dns_format}" | sort -u 2>/dev/null >> "${dns_file}" + +# write dns file footer +# +f_footer + +# restart dnsmasq with newly generated block list +# +/etc/init.d/dnsmasq restart >/dev/null 2>&1 +sleep 2 + +# dnsmasq health check +# +f_dnscheck + +# remove files and exit +# +f_remove +exit 0 diff --git a/net/adblock/files/www/adblock/adblock.html b/net/adblock/files/www/adblock/adblock.html new file mode 100644 index 000000000..cc8d21357 --- /dev/null +++ b/net/adblock/files/www/adblock/adblock.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/net/adblock/files/www/adblock/adblock.png b/net/adblock/files/www/adblock/adblock.png new file mode 100644 index 0000000000000000000000000000000000000000..9da19eacad3b03bb08bbddbbf4ac48dd78b3d838 GIT binary patch literal 68 zcmeAS@N?(olHy`uVBq!ia0vp^j3CUx0wlM}@Gt=>Zci7-kcv6Uzs@r-FtIZ-&5|)J Q1PU{Fy85}Sb4q9e0B4a5jsO4v literal 0 HcmV?d00001 diff --git a/net/adblock/files/www/adblock/index.html b/net/adblock/files/www/adblock/index.html new file mode 100644 index 000000000..cc8d21357 --- /dev/null +++ b/net/adblock/files/www/adblock/index.html @@ -0,0 +1,6 @@ + + + + + +