Browse Source

Merge pull request #2304 from dibdot/adblock

adblock: update to 0.60.0
lilik-openwrt-22.03
Hannu Nyman 9 years ago
parent
commit
5bdd69c1f6
15 changed files with 634 additions and 664 deletions
  1. +8
    -6
      net/adblock/Makefile
  2. +47
    -75
      net/adblock/files/README.md
  3. +387
    -385
      net/adblock/files/adblock-helper.sh
  4. +146
    -120
      net/adblock/files/adblock-update.sh
  5. +0
    -1
      net/adblock/files/adblock.blacklist
  6. +8
    -21
      net/adblock/files/adblock.conf
  7. +0
    -1
      net/adblock/files/adblock.whitelist
  8. +33
    -40
      net/adblock/files/samples/adblock.conf.sample
  9. +1
    -1
      net/adblock/files/samples/dhcp.config.sample
  10. +1
    -1
      net/adblock/files/samples/dnsmasq.conf.sample
  11. +0
    -5
      net/adblock/files/samples/firewall.user.sample
  12. +1
    -2
      net/adblock/files/samples/root.crontab.sample
  13. +0
    -4
      net/adblock/files/samples/uhttpd.config.sample
  14. +1
    -1
      net/adblock/files/www/adblock/adblock.html
  15. +1
    -1
      net/adblock/files/www/adblock/index.html

+ 8
- 6
net/adblock/Makefile View File

@ -1,5 +1,5 @@
#
# Copyright (C) 2015 OpenWrt.org
# Copyright (C) 2015-2016 OpenWrt.org
#
# This is free software, licensed under the GNU General Public License v3.
#
@ -7,10 +7,10 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=adblock
PKG_VERSION:=0.40.2
PKG_VERSION:=0.60.0
PKG_RELEASE:=1
PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org>
PKG_MAINTAINER:=Dirk Brenken <openwrt@brenken.org>
include $(INCLUDE_DIR)/package.mk
@ -18,21 +18,23 @@ define Package/$(PKG_NAME)
SECTION:=net
CATEGORY:=Network
TITLE:=Powerful adblock script to block ad/abuse domains
DEPENDS:=+curl +wget
DEPENDS:=+wget
PKGARCH:=all
endef
define Package/$(PKG_NAME)/description
Powerful adblock script to block ad/abuse domains
When the dns server on your router receives dns requests, we will sort out queries that ask for the [A]
resource records of ad servers and return the local ip address of your router and the internal web server
When the dns server on your router receives dns requests, we will sort out queries that ask for the resource
records of ad/abuse servers and return the local ip address of your router and the internal web server
delivers a transparent pixel instead.
The script supports the following domain blacklist sites:
https://adaway.org
https://disconnect.me
http://dshield.org
https://feodotracker.abuse.ch
http://malwaredomains.com
http://www.malwaredomainlist.com
https://palevotracker.abuse.ch
http://www.shallalist.de
http://www.spam404.com


+ 47
- 75
net/adblock/files/README.md View File

@ -5,20 +5,22 @@ A lot of people already use adblocker plugins within their desktop browsers,
but what if you are using your (smart) phone, tablet, watch or any other wlan gadget...
...getting rid of annoying ads, trackers and other abuse sites (like facebook ;-) is simple: block them with your router.
When the dns server on your router receives dns requests, you’ll sort out queries that ask for the [A] resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
When the dns server on your router receives dns requests, you will sort out queries that ask for the resource records of ad servers and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
## Main Features
* support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses):
* [adaway.org](https://adaway.org)
* => infrequent updates, approx. 400 entries
* => infrequent updates, approx. 400 entries (enabled by default)
* [disconnect.me](https://disconnect.me)
* => numerous updates on the same day, approx. 6.500 entries
* => numerous updates on the same day, approx. 6.500 entries (enabled by default)
* [dshield.org](http://dshield.org)
* => daily updates, approx. 4.500 entries
* [feodotracker.abuse.ch](https://feodotracker.abuse.ch)
* => daily updates, approx. 0-10 entries
* [malwaredomains.com](http://malwaredomains.com)
* => daily updates, approx. 16.000 entries
* [malwaredomainlist.com](http://www.malwaredomainlist.com)
* => daily updates, approx. 1.500 entries
* [palevotracker.abuse.ch](https://palevotracker.abuse.ch)
* => daily updates, approx. 15 entries
* [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
@ -33,93 +35,63 @@ When the dns server on your router receives dns requests, you’ll sort out quer
* => weekly updates, approx. 2.500 entries (enabled by default)
* [zeustracker.abuse.ch](https://zeustracker.abuse.ch)
* => daily updates, approx. 440 entries
* zero-conf like automatic installation & setup, usually no manual changes needed (i.e. ip address, network devices etc.)
* full IPv4 and IPv6 support
* each blocklist source will be updated and processed separately
* timestamp check to download and process only updated blocklists
* overall duplicate removal in separate blocklists (will be automatically disabled on low memory systems)
* blocklist parsing by fast & flexible regex rulesets
* timestamp check to download and process only updated adblock list sources
* overall duplicate removal in separate adblock lists (will be automatically disabled on low memory systems)
* adblock source list parsing by fast & flexible regex rulesets
* additional white- and blacklist support for manual overrides
* use of dynamic adblock network interface
* use of dynamic uhttpd instance as pixel server
* use of quality checks after adblocklist updates to ensure a reliable dnsmasq service
* quality checks during/after update of adblock lists to ensure a reliable dnsmasq service
* wan update check, to wait for an active wan uplink before update
* basic adblock statistics via iptables packet counters
* status & error logging to stdout and syslog
* use of dynamic uhttpd instance as adblock pixel server
* optional features (disabled by default):
* powerful adblock list backup/restore handling
* wan interface check, useful for rc.local based autostart and (mobile) multiwan setups to update only on pre-defined wan interfaces
* domain query logging as a background service to easily identify free and already blocked domains (see example output below)
* adblock statistics (req. query logging)
* ntp time sync
* status & error logging to separate file (req. ntp time sync)
* adblock list backup/restore
* debug logging to separate file
## Prerequisites
* [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work
* additional software packages:
* curl
* wget (due to an openwrt bug still needed for certain https requests - see ticket #19621)
* optional: busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found)
* the above dependencies and requirements will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors
* [openwrt](https://openwrt.org), tested with latest stable release (Chaos Calmer 15.05) and with current trunk (Designated Driver > r47025)
* required software packages:
* wget
* optional for IPv6 support: kmod-ipt-nat6
* the above dependencies and requirements will be checked during package installation & script runtime, please check console output or *logread -e "adblock"* for errors
## Usage
* select & install adblock package (*opkg install adblock*)
* configure */etc/config/adblock* to your needs, see additional comments in *adblock.conf.sample*
* at least configure the ip address of the local adblock interface/uhttpd instance, it needs to be a different subnet from the normal LAN
* recommendation: there is no need to enable all blacklist sites at once, for normal use one to three lists should be sufficient
* recommendation: to handle all blacklists at once add an usb stick or any other storage device to supersize your /tmp directory with a swap partition
* install the adblock package (*opkg install adblock*)
* optional: for an update installation please replace your existing */etc/config/adblock* with a copy of */etc/samples/adblock.conf.sample* to get the latest changes
* optional: enable/disable your required adblock list sources in */etc/config/adblock* - 'adaway', 'disconnect' and 'yoyo' are enabled by default
* start */usr/bin/adblock-update.sh* and check console output or *logread -e "adblock"* for errors
## Tweaks
* there is no need to enable all blacklist sites at once, for normal use one to three adblock list sources should be sufficient
* if you really need to handle all blacklists at once add an usb stick or any other storage device to supersize your /tmp directory with a swap partition
* => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details
* add additional domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), both lists are located in */etc/adblock*
* by default openwrts main uhttpd instance is bind to all ports of your router,
* for a working adblock setup you have to bind uhttpd to the standard LAN port only, please change listen_http accordingly
* start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors
* add static, personal domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), by default both lists are located in */etc/adblock*
* enable the backup/restore feature, to restore automatically the latest, stable backup of your adblock lists in case of any processing error
* enable the debug log feature for continuous logfile writing to monitor the adblock runs over a longer period
## Distributed samples
* all sample configuration files stored in */etc/adblock/samples*
* to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*
* to bind uhttpd to standard LAN port only, see *uhttpd.config.sample*
* for a fully blown adblock configuration with all explained options see *adblock.conf.sample*
* for some dnsmasq tweaks see *dhcp.config.sample* and *dnsmasq.conf.sample*
* for rc.local based autostart and /tmp resizing on the fly see *rc.local.sample*
* for scheduled call of *adblock-update.sh* see *root.crontab.sample*
* to redirect/force all dns queries to your router see *firwall.user.sample*
* for further dnsmasq tweaks see *dnsmasq.conf.sample*
## Examples
adblock log file excerpt:
adblock[11780] info : domain adblock processing started (0.40.1, r47929, 20.12.2015 21:59:56)
adblock[11780] info : backup/restore will be disabled
adblock[11780] info : wan update check will be disabled
adblock[11780] info : get ntp time sync (192.168.254.254), after 0 loops
adblock[11780] info : source doesn't change, no update required (whocares)
adblock[11780] info : adblocklists with overall 11619 domains loaded, backups disabled
adblock[11780] info : adblock statistics for query date 20151220 (total: 2495, blocked: 154)
adblock[11780] info : domain adblock processing finished (0.40.1, r47929, 20.12.2015 22:00:03)
domain blocklist for dnsmasq (disconnect.me after overall duplicate removal):
address=/0000mps.webpreview.dsl.net/192.168.2.1
address=/0001.2waky.com/192.168.2.1
address=/001wen.com/192.168.2.1
address=/002it.com/192.168.2.1
address=/00game.net/192.168.2.1
[...]
address=/zzsgssxh.com/192.168.2.1
address=/zzshw.net/192.168.2.1
address=/zztxdown.com/192.168.2.1
address=/zzxcws.com/192.168.2.1
#------------------------------------------------------------------
# adblock-update.sh (0.40.0) - 3710 ad/abuse domains blocked
# source: https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt
# last modified: Thu, 17 Dec 2015 09:21:17 GMT
domain query log excerpt:
query[A] www.seenby.de from fe80::6257:18ff:fe6b:4667
query[A] tarifrechner.heise.de from 192.168.1.131
query[A] www.mittelstandswiki.de from fe80::6257:18ff:fe6b:4667
query[A] ad.doubleclick.net from 192.168.1.131
ad.doubleclick.net is 192.168.2.1
## Background
This adblock package is a dns/dnsmasq based adblock solution for openwrt.
Queries to ad/abuse domains are never forwarded and always replied with a local IP address which may be IPv4 or IPv6.
For that purpose adblock uses an ip address from the old reserved [class 'E'](https://tools.ietf.org/html/rfc1700) subnet (254.0.0.1 / ::ffff:fe00:0001) by default.
Furthermore all ad/abuse queries will be filtered by ip(6)tables and redirected to internal adblock pixel server (in PREROUTING chain) or rejected (in FORWARD chain).
All iptables and uhttpd related adblock additions are non-destructive, no hard-coded changes in 'firewall.user', 'uhttpd' config or any other openwrt related config files.
The first three queries are OK (not blocked), the last one has been blocked and answered by local dnsmasq instance.
## Removal
* remove the adblock package (*opkg remove adblock*)
* remove all script generated adblock lists in */tmp/dnsmasq.d/*
* kill the running adblock uhttpd instance (ps | grep "[u]httpd.*\-h /www/adblock")
* run /etc/init.d/dnsmasq restart
* run /etc/init.d/firewall restart
Have fun!
Dirk

+ 387
- 385
net/adblock/files/adblock-helper.sh View File

@ -1,8 +1,8 @@
#!/bin/sh
##############################################
# function library used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
##############################################
#################################################
# function library used by adblock-update.sh #
# written by Dirk Brenken (openwrt@brenken.org) #
#################################################
#####################################
# f_envload: load adblock environment
@ -15,20 +15,20 @@ f_envload()
then
. "/lib/functions.sh" 2>/dev/null
else
rc=500
rc=110
f_log "openwrt function library not found" "${rc}"
f_deltemp
f_exit
fi
# source in openwrt json helpers library
# source in openwrt network library
#
if [ -r "/usr/share/libubox/jshn.sh" ]
if [ -r "/lib/functions/network.sh" ]
then
. "/usr/share/libubox/jshn.sh" 2>/dev/null
. "/lib/functions/network.sh" 2>/dev/null
else
rc=505
f_log "openwrt json helpers library not found" "${rc}"
f_deltemp
rc=115
f_log "openwrt network library not found" "${rc}"
f_exit
fi
# get list with all installed openwrt packages
@ -36,9 +36,9 @@ f_envload()
pkg_list="$(opkg list-installed 2>/dev/null)"
if [ -z "${pkg_list}" ]
then
rc=510
rc=120
f_log "empty openwrt package list" "${rc}"
f_deltemp
f_exit
fi
}
@ -52,27 +52,19 @@ f_envparse()
#
LC_ALL=C
# set initial defaults (may be overwritten by setting appropriate adblock config options)
# set initial defaults,
# may be overwritten by setting appropriate adblock config options in global section of /etc/config/adblock
#
adb_if="adblock"
adb_minspace="20000"
adb_wanif="wan"
adb_lanif="lan"
adb_port="65535"
adb_nullipv4="254.0.0.1"
adb_nullipv6="::ffff:fe00:0001"
adb_maxtime="60"
adb_maxloop="5"
adb_unique="1"
adb_maxloop="20"
adb_blacklist="/etc/adblock/adblock.blacklist"
adb_whitelist="/etc/adblock/adblock.whitelist"
# adblock device name auto detection
# derived from first entry in openwrt lan ifname config
#
adb_dev="$(uci get network.lan.ifname 2>/dev/null)"
adb_dev="${adb_dev/ *}"
# adblock ntp server name auto detection
# derived from ntp list found in openwrt ntp server config
#
adb_ntpsrv="$(uci get system.ntp.server 2>/dev/null)"
# function to read/set global options by callback,
# prepare list items and build option list for all others
#
@ -103,13 +95,7 @@ f_envparse()
{
local list="${1}"
local value="${2}"
if [ "${list}" = "adb_wanlist" ]
then
adb_wandev="${adb_wandev} ${value}"
elif [ "${list}" = "adb_ntplist" ]
then
adb_ntpsrv="${adb_ntpsrv} ${value}"
elif [ "${list}" = "adb_catlist" ]
if [ "${list}" = "adb_catlist" ]
then
adb_cat_shalla="${adb_cat_shalla} ${value}"
fi
@ -139,15 +125,6 @@ f_envparse()
fi
fi
done
elif [ "${config}" = "wancheck" ]
then
unset adb_wandev
elif [ "${config}" = "ntpcheck" ]
then
unset adb_ntpsrv
elif [ "${config}" = "shalla" ]
then
unset adb_cat_shalla
fi
}
@ -157,10 +134,14 @@ f_envparse()
config_foreach parse_config service
config_foreach parse_config source
# set temp variables and defaults
# set more script defaults (can't be overwritten by adblock config options)
#
adb_minspace="20000"
adb_unique="1"
adb_tmpfile="$(mktemp -tu 2>/dev/null)"
adb_tmpdir="$(mktemp -p /tmp -d 2>/dev/null)"
adb_dnsdir="/tmp/dnsmasq.d"
adb_dnsprefix="adb_list"
unset adb_srcfind
unset adb_revsrcfind
@ -174,6 +155,7 @@ f_envparse()
rset_dshield="${rset_start} | ${rset_end}"
rset_feodo="${rset_start} | ${rset_end}"
rset_malware="${rset_start} | ${rset_end}"
rset_malwarelist="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
rset_palevo="${rset_start} | ${rset_end}"
rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
@ -182,146 +164,106 @@ f_envparse()
rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
rset_zeus="${rset_start} | ${rset_end}"
# set dnsmasq defaults
#
adb_dnsdir="/tmp/dnsmasq.d"
adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
adb_dnsprefix="adb_list"
}
#############################################
# f_envcheck: check environment prerequisites
#
f_envcheck()
{
# check adblock config file
# get logical wan update interfaces
#
check_config="$(grep -F "ruleset=rset_default" /etc/config/adblock 2>/dev/null)"
if [ -n "${check_config}" ]
network_find_wan adb_wanif4 2>/dev/null
network_find_wan6 adb_wanif6 2>/dev/null
if [ "${adb_wanif4}" = "${adb_lanif}" ] || [ "${adb_wanif6}" = "${adb_lanif}" ]
then
rc=515
grep -Fv "#" "/etc/adblock/samples/adblock.conf.sample" > /etc/config/adblock
f_log "new default adblock config applied, please check your configuration settings in /etc/config/adblock" "${rc}"
f_deltemp
rc=125
f_log "LAN only (${adb_lanif}) network, no valid IPv4/IPv6 wan update interface found" "${rc}"
f_exit
elif [ -z "${adb_wanif4}" ] && [ -z "${adb_wanif6}" ]
then
rc=125
f_log "no valid IPv4/IPv6 wan update interface found" "${rc}"
f_exit
fi
# check required config options
# get lan ip addresses
#
adb_varlist="adb_ip adb_dev adb_domain"
for var in ${adb_varlist}
do
if [ -z "$(eval printf \"\$"${var}"\")" ]
then
rc=520
f_log "missing adblock config option (${var})" "${rc}"
f_deltemp
fi
done
# check main uhttpd configuration
#
check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -Fo "0.0.0.0" 2>/dev/null)"
if [ -n "${check_uhttpd}" ]
network_get_ipaddr adb_ipv4 "${adb_lanif}" 2>/dev/null
network_get_ipaddr6 adb_ipv6 "${adb_lanif}" 2>/dev/null
if [ -z "${adb_ipv4}" ] && [ -z "${adb_ipv6}" ]
then
rc=525
lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)"
f_log "please bind main uhttpd instance to LAN only (lan ip: ${lan_ip})" "${rc}"
f_deltemp
rc=130
f_log "no valid IPv4/IPv6 configuration for given logical LAN interface found (${adb_lanif}), please set 'adb_lanif' manually" "${rc}"
f_exit
fi
# check adblock network device configuration
# read system ntp server names
#
if [ ! -d "/sys/class/net/${adb_dev}" ]
then
rc=530
f_log "invalid adblock network device input (${adb_dev})" "${rc}"
f_deltemp
fi
adb_ntpsrv="$(uci get system.ntp.server 2>/dev/null)"
}
#################################################
# f_envcheck: check/set environment prerequisites
#
f_envcheck()
{
# check general package dependencies
#
f_depend "wget"
f_depend "iptables"
f_depend "kmod-ipt-nat"
# check adblock network interface configuration
# check ipv6 related package dependencies
#
check_if="$(printf "${adb_if}" | sed -n '/[^._0-9A-Za-z]/p' 2>/dev/null)"
banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p' 2>/dev/null)"
if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
if [ -n "${adb_wanif6}" ]
then
rc=535
f_log "invalid adblock network interface input (${adb_if})" "${rc}"
f_deltemp
check="$(printf "${pkg_list}" | grep "^ip6tables -" 2>/dev/null)"
if [ -z "${check}" ]
then
f_log "package 'ip6tables' not found, IPv6 support wíll be disabled"
unset adb_wanif6
else
check="$(printf "${pkg_list}" | grep "^kmod-ipt-nat6 -" 2>/dev/null)"
if [ -z "${check}" ]
then
f_log "package 'kmod-ipt-nat6' not found, IPv6 support wíll be disabled"
unset adb_wanif6
fi
fi
fi
# check adblock ip address configuration
# check ca-certificates package and set wget parms accordingly
#
check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' 2>/dev/null)"
lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)"
if [ -z "${check_ip}" ]
then
rc=540
f_log "invalid adblock ip address input (${adb_ip})" "${rc}"
f_deltemp
elif [ "${adb_ip}" = "${lan_ip}" ]
check="$(printf "${pkg_list}" | grep "^ca-certificates -" 2>/dev/null)"
if [ -z "${check}" ]
then
rc=545
f_log "adblock ip needs to be a different subnet from the normal LAN (adblock ip: ${adb_ip})" "${rc}"
f_deltemp
wget_parm="--no-config --no-check-certificate --quiet"
else
wget_parm="--no-config --quiet"
fi
# check adblock blacklist/whitelist configuration
#
if [ ! -r "${adb_blacklist}" ]
then
rc=550
f_log "adblock blacklist not found" "${rc}"
f_deltemp
rc=135
f_log "adblock blacklist not found (${adb_blacklist})" "${rc}"
f_exit
elif [ ! -r "${adb_whitelist}" ]
then
rc=555
f_log "adblock whitelist not found" "${rc}"
f_deltemp
rc=135
f_log "adblock whitelist not found (${adb_whitelist})" "${rc}"
f_exit
fi
# check adblock temp directory
#
if [ -n "${adb_tmpdir}" ] && [ -d "${adb_tmpdir}" ]
then
f_space "${adb_tmpdir}"
tmp_ok="true"
f_space "${adb_tmpdir}" "please supersize your /tmp directory"
if [ "${space_ok}" = "false" ]
then
rc=140
f_exit
fi
else
rc=560
tmp_ok="false"
rc=140
f_log "temp directory not found" "${rc}"
f_deltemp
fi
# check curl package dependency
#
check="$(printf "${pkg_list}" | grep "^curl -" 2>/dev/null)"
if [ -z "${check}" ]
then
rc=565
f_log "curl package not found" "${rc}"
f_deltemp
fi
# check wget package dependency
#
check="$(printf "${pkg_list}" | grep "^wget -" 2>/dev/null)"
if [ -z "${check}" ]
then
rc=570
f_log "wget package not found" "${rc}"
f_deltemp
fi
# check ca-certificates package and set wget/curl options accordingly
#
check="$(printf "${pkg_list}" | grep "^ca-certificates -" 2>/dev/null)"
if [ -z "${check}" ]
then
curl_parm="-q --insecure --silent"
wget_parm="--no-config --no-hsts --no-check-certificate --quiet"
else
curl_parm="-q --silent"
wget_parm="--no-config --no-hsts --quiet"
f_exit
fi
# check total and swap memory
@ -340,41 +282,16 @@ f_envcheck()
#
if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
then
f_space "${adb_backupdir}"
backup_ok="true"
else
backup_ok="false"
f_log "backup/restore will be disabled"
fi
# check dns query log configuration
#
adb_querydir="${adb_queryfile%/*}"
adb_querypid="/var/run/adb_query.pid"
if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
then
# check find capabilities
#
check="$(find --help 2>&1 | grep -F "mtime" 2>/dev/null)"
if [ -z "${check}" ]
f_space "${adb_backupdir}" "backup/restore will be disabled"
if [ "${space_ok}" = "false" ]
then
query_ok="false"
f_log "busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
backup_ok="false"
else
f_space "${adb_querydir}"
query_ok="true"
query_name="${adb_queryfile##*/}"
query_ip="${adb_ip//./\\.}"
backup_ok="true"
fi
else
query_ok="false"
f_log "dns query logging will be disabled"
if [ -s "${adb_querypid}" ]
then
kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1
f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}" 2>/dev/null))"
> "${adb_querypid}"
fi
backup_ok="false"
f_log "backup/restore will be disabled"
fi
# check debug log configuration
@ -382,86 +299,229 @@ f_envcheck()
adb_logdir="${adb_logfile%/*}"
if [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ]
then
f_space "${adb_logdir}"
log_ok="true"
f_space "${adb_logdir}" "debug logging will be disabled"
if [ "${space_ok}" = "false" ]
then
log_ok="false"
else
log_ok="true"
fi
else
log_ok="false"
f_log "debug logging will be disabled"
fi
# check wan update configuration
# check ipv4/iptables configuration
#
if [ -n "${adb_wandev}" ]
if [ -n "${adb_wanif4}" ]
then
f_wancheck "${adb_maxloop}"
else
wan_ok="false"
f_log "wan update check will be disabled"
f_firewall "IPv4" "nat" "I" "PREROUTING" "adb-nat: tcp, port 80, DNAT" "-p tcp -d ${adb_nullipv4} --dport 80 -j DNAT --to-destination ${adb_ipv4}:${adb_port}"
f_firewall "IPv4" "nat" "A" "PREROUTING" "adb-dns: udp, port 53, DNAT" "-p udp --dport 53 -j DNAT --to-destination ${adb_ipv4}"
f_firewall "IPv4" "nat" "A" "PREROUTING" "adb-dns: tcp, port 53, DNAT" "-p tcp --dport 53 -j DNAT --to-destination ${adb_ipv4}"
f_firewall "IPv4" "filter" "I" "FORWARD" "adb-rej: all protocols, all ports, REJECT" "-d ${adb_nullipv4} -j REJECT"
fi
# check ntp sync configuration
# check ipv6/ip6tables configuration
#
if [ -n "${adb_ntpsrv}" ]
if [ -n "${adb_wanif6}" ]
then
f_ntpcheck "${adb_maxloop}"
else
ntp_ok="false"
f_log "ntp time sync will be disabled"
f_firewall "IPv6" "nat" "I" "PREROUTING" "adb-nat: tcp, port 80, DNAT" "-p tcp -d ${adb_nullipv6} --dport 80 -j DNAT --to-destination [${adb_ipv6}]:${adb_port}"
f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-dns: udp, port 53, DNAT" "-p udp --dport 53 -j DNAT --to-destination ${adb_ipv6}"
f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-dns: tcp, port 53, DNAT" "-p tcp --dport 53 -j DNAT --to-destination ${adb_ipv6}"
f_firewall "IPv6" "filter" "I" "FORWARD" "adb-rej: all protocols, all ports, REJECT" "-d ${adb_nullipv6} -j REJECT"
fi
# check dynamic/volatile adblock network interface configuration
# check volatile adblock uhttpd instance configuration
#
rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf ${?})"
rc="$(ps | grep "[u]httpd.*\-h /www/adblock" >/dev/null 2>&1; printf ${?})"
if [ $((rc)) -ne 0 ]
then
json_init
json_add_string name "${adb_if}"
json_add_string ifname "${adb_dev}"
json_add_string proto "static"
json_add_array ipaddr
json_add_string "" "${adb_ip}"
json_close_array
json_close_object
ubus call network add_dynamic "$(json_dump)"
rc=${?}
if [ $((rc)) -eq 0 ]
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
f_log "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
else
f_log "failed to initialize new dynamic/volatile network interface (${adb_if}, ${adb_ip})" "${rc}"
f_remove
uhttpd -h "/www/adblock" -k 0 -N 100 -T 5 -D -E "/adblock.html" -p "${adb_ipv4}:${adb_port}" -p "[${adb_ipv6}]:${adb_port}">/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
then
f_log "created volatile uhttpd instance (${adb_ipv4}:${adb_port}, [${adb_ipv6}]:${adb_port})"
else
f_log "failed to initialize volatile uhttpd instance (${adb_ipv4}:${adb_port}, [${adb_ipv6}]:${adb_port})" "${rc}"
f_restore
fi
elif [ -n "${adb_wanif4}" ]
then
uhttpd -h "/www/adblock" -k 0 -N 100 -T 5 -D -E "/adblock.html" -p "${adb_ipv4}:${adb_port}" >/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
then
f_log "created volatile uhttpd instance (${adb_ipv4}:${adb_port})"
else
f_log "failed to initialize volatile uhttpd instance (${adb_ipv4}:${adb_port})" "${rc}"
f_restore
fi
elif [ -n "${adb_wanif6}" ]
then
uhttpd -h "/www/adblock" -k 0 -N 100 -T 5 -D -E "/adblock.html" -p "[${adb_ipv6}]:${adb_port}" >/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
then
f_log "created volatile uhttpd instance ([${adb_ipv6}]:${adb_port})"
else
f_log "failed to initialize volatile uhttpd instance ([${adb_ipv6}]:${adb_port})" "${rc}"
f_restore
fi
fi
fi
# check dynamic/volatile adblock uhttpd instance configuration
# wait for active wan update interface
#
rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf ${?})"
if [ $((rc)) -ne 0 ]
cnt=0
while [ $((cnt)) -le $((adb_maxloop)) ]
do
for interface in ${adb_wanif}
do
network_get_device adb_wandev "${interface}" 2>/dev/null
if [ -z "${adb_wandev}" ] || [ ! -d "/sys/class/net/${adb_wandev}" ]
then
if [ -n "${adb_wanif4}" ]
then
network_get_device adb_wandev "${adb_wanif4}" 2>/dev/null
else
network_get_device adb_wandev "${adb_wanif6}" 2>/dev/null
fi
if [ -z "${adb_wandev}" ] || [ ! -d "/sys/class/net/${adb_wandev}" ]
then
rc=145
f_log "no valid network device for given logical WAN interface found, please set 'adb_wanif' manually" "${rc}"
f_restore
fi
fi
check="$(cat /sys/class/net/${adb_wandev}/operstate 2>/dev/null)"
if [ "${check}" = "up" ]
then
f_log "get active wan update interface/device (${adb_wanif}/${adb_wandev}) after ${cnt} loops"
break 2
elif [ $((cnt)) -eq $((adb_maxloop)) ]
then
rc=145
f_log "wan update interface/device not running (${adb_wanif}/${adb_wandev}) after ${cnt} loops" "${rc}"
f_restore
fi
cnt=$((cnt + 1))
sleep 1
done
done
# wait for ntp sync
#
if [ -n "${adb_ntpsrv}" ]
then
uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" >/dev/null 2>&1
cnt=0
unset ntp_pool
for srv in ${adb_ntpsrv}
do
ntp_pool="${ntp_pool} -p ${srv}"
done
/usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
then
f_log "created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
ntp_ok="true"
f_log "get ntp time sync"
else
f_log "failed to initialize new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" "${rc}"
f_remove
rc=0
ntp_ok="false"
f_log "ntp time sync failed"
fi
fi
# remove no longer used package list
# set dnsmasq defaults
#
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
adb_dnsformat="awk -v ipv4="${adb_nullipv4}" -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv4\"\n\"\"address=/\"\$0\"/\"ipv6}'"
elif [ -n "${adb_wanif4}" ]
then
adb_dnsformat="awk -v ipv4="${adb_nullipv4}" '{print \"address=/\"\$0\"/\"ipv4}'"
elif [ -n "${adb_wanif6}" ]
then
adb_dnsformat="awk -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv6}'"
fi
# remove no longer used opkg package list
#
unset pkg_list
}
################################################
# f_log: log messages to stdout, syslog, logfile
######################################
# f_depend: check package dependencies
#
f_depend()
{
local rc_func
local package="${1}"
check="$(printf "${pkg_list}" | grep "^${package} -" 2>/dev/null)"
if [ -z "${check}" ]
then
rc_func=150
f_log "package '${package}' not found" "${rc_func}"
f_exit
fi
}
##############################################
# f_firewall: set iptables rules for ipv4/ipv6
#
f_firewall()
{
local rc_func
local ipt
local iptv4="/usr/sbin/iptables"
local iptv6="/usr/sbin/ip6tables"
local proto="${1}"
local table="${2}"
local ctype="${3}"
local chain="${4}"
local notes="${5}"
local rules="${6}"
# select appropriate iptables executable
#
if [ "${proto}" = "IPv4" ]
then
ipt="${iptv4}"
else
ipt="${iptv6}"
fi
# check whether iptables rule already applied and proceed accordingly
#
rc_func="$("${ipt}" -w -t "${table}" -C "${chain}" -m comment --comment "${notes}" ${rules} >/dev/null 2>&1; printf ${?})"
if [ $((rc_func)) -ne 0 ]
then
"${ipt}" -w -t "${table}" -"${ctype}" "${chain}" -m comment --comment "${notes}" ${rules} >/dev/null 2>&1
rc_func=${?}
if [ $((rc_func)) -eq 0 ]
then
f_log "created volatile ${proto} firewall rule in '${chain}' chain (${notes})"
else
f_log "failed to initialize volatile ${proto} firewall rule in '${chain}' chain (${notes})" "${rc_func}"
f_restore
fi
fi
}
###################################################
# f_log: log messages to stdout, syslog and logfile
#
f_log()
{
local log_msg="${1}"
local log_rc="${2}"
local class="info "
# log to different output devices, set log class accordingly
#
if [ -n "${log_msg}" ]
then
if [ $((log_rc)) -ne 0 ]
@ -483,7 +543,12 @@ f_log()
#
f_space()
{
local rc_func
local mp="${1}"
local notes="${2}"
# check relevant mount points in a subshell
#
if [ -d "${mp}" ]
then
df "${mp}" 2>/dev/null |\
@ -493,114 +558,49 @@ f_space()
av_space="${available}"
if [ $((av_space)) -eq 0 ]
then
rc=575
f_log "no space left on device/not mounted (${mp})" "${rc}"
exit ${rc}
rc_func=155
f_log "no space left on device/not mounted (${mp}), ${notes}"
exit ${rc_func}
elif [ $((av_space)) -lt $((adb_minspace)) ]
then
rc=580
f_log "not enough space left on device (${mp})" "${rc}"
exit ${rc}
rc_func=155
f_log "not enough space left on device (${mp}), ${notes}"
exit ${rc_func}
fi
done
rc=${?}
if [ $((rc)) -eq 0 ]
then
space_ok="true"
else
space_ok="false"
f_deltemp
fi
fi
}
####################################################
# f_deltemp: delete temp files, directories and exit
#
f_deltemp()
{
if [ -f "${adb_tmpfile}" ]
then
rm -f "${adb_tmpfile}" >/dev/null 2>&1
fi
if [ -d "${adb_tmpdir}" ]
then
rm -rf "${adb_tmpdir}" >/dev/null 2>&1
fi
f_log "domain adblock processing finished (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
exit ${rc}
}
####################################################
# f_remove: maintain and (re-)start domain query log
#
f_remove()
{
local query_pid
local query_date
local query_total
local query_blocked
if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
query_date="$(date "+%Y%m%d")"
if [ -s "${adb_querypid}" ] && [ -f "${adb_queryfile}.${query_date}" ]
then
query_total="$(grep -F "query[A]" "${adb_queryfile}.${query_date}" 2>/dev/null | wc -l)"
query_blocked="$(grep -Fv "query[A]" "${adb_queryfile}.${query_date}" 2>/dev/null | wc -l)"
f_log "adblock statistics for query date ${query_date} (total: ${query_total}, blocked: ${query_blocked})"
fi
if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
then
query_pid="$(cat "${adb_querypid}" 2>/dev/null)"
> "${adb_querypid}"
kill -9 "${query_pid}" >/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
then
find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f "{}" \; 2>/dev/null
rc=${?}
if [ $((rc)) -eq 0 ]
then
f_log "remove old domain query background process (pid: ${query_pid}) and do logfile housekeeping"
else
f_log "error during domain query logfile housekeeping" "${rc}"
fi
else
f_log "error during domain query background process removal (pid: ${query_pid})" "${rc}"
fi
fi
if [ ! -s "${adb_querypid}" ]
# subshell return code handling, set space trigger accordingly
#
rc_func=${?}
if [ $((rc_func)) -ne 0 ]
then
(logread -f 2>/dev/null & printf ${!} > "${adb_querypid}") | grep -Eo "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" 2>/dev/null >> "${adb_queryfile}.${query_date}" &
rc=${?}
if [ $((rc)) -eq 0 ]
then
sleep 1
f_log "new domain query log background process started (pid: $(cat "${adb_querypid}" 2>/dev/null))"
else
f_log "error during domain query background process start" "${rc}"
fi
space_ok="false"
fi
fi
f_deltemp
}
################################################################
# f_restore: restore last adblocklist backup and restart dnsmasq
##################################################################
# f_restore: restore last adblock list backups and restart dnsmasq
#
f_restore()
{
# remove bogus adblocklists
local rc_func
local removal_done
local restore_done
# remove bogus adblock lists
#
if [ -n "${adb_revsrclist}" ]
then
find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec rm -f "{}" \; 2>/dev/null
if [ $((rc)) -eq 0 ]
rc_func=${?}
if [ $((rc_func)) -ne 0 ]
then
f_log "bogus adblocklists removed"
f_log "error during removal of bogus adblock lists" "${rc_func}"
f_exit
else
f_log "error during removal of bogus adblocklists" "${rc}"
f_remove
removal_done="true"
f_log "all bogus adblock lists removed"
fi
fi
@ -608,87 +608,89 @@ f_restore()
#
if [ "${backup_ok}" = "true" ] && [ -d "${adb_backupdir}" ] && [ "$(printf "${adb_backupdir}/${adb_dnsprefix}."*)" != "${adb_backupdir}/${adb_dnsprefix}.*" ]
then
cp -f "${adb_backupdir}/${adb_dnsprefix}."* "${adb_dnsdir}" >/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
for file in ${adb_backupdir}/${adb_dnsprefix}.*
do
filename="${file##*/}"
cp -pf "${file}" "${adb_dnsdir}" 2>/dev/null
rc_func=${?}
if [ $((rc_func)) -ne 0 ]
then
f_log "error during restore of adblock list (${filename})" "${rc_func}"
f_exit
fi
restore_done="true"
done
f_log "all available backups restored"
else
f_log "no backups found, nothing to restore"
fi
# (re-)try dnsmasq restart without bogus adblock lists / with backups
#
if [ "${restore_done}" = "true" ] || [ "${removal_done}" = "true" ]
then
/etc/init.d/dnsmasq restart >/dev/null 2>&1
sleep 2
dns_status="$(ps 2>/dev/null | grep "[d]nsmasq" 2>/dev/null)"
if [ -n "${dns_status}" ]
then
f_log "all available backups restored"
rc=0
else
f_log "error during restore" "${rc}"
f_remove
rc=160
f_log "dnsmasq restart failed, please check 'logread' output" "${rc}"
f_restore
fi
fi
/etc/init.d/dnsmasq restart >/dev/null 2>&1
f_remove
f_exit
}
#######################################################
# f_wancheck: check for usable adblock update interface
###################################
# f_exit: delete (temporary) files,
# generate statistics and exit
#
f_wancheck()
f_exit()
{
local cnt=0
local cnt_max="${1}"
local dev
local dev_out
while [ $((cnt)) -le $((cnt_max)) ]
do
for dev in ${adb_wandev}
do
if [ -d "/sys/class/net/${dev}" ]
then
dev_out="$(cat /sys/class/net/${dev}/operstate 2>/dev/null)"
rc=${?}
if [ "${dev_out}" = "up" ]
then
wan_ok="true"
f_log "get wan/update interface (${dev}), after ${cnt} loops"
break 2
fi
fi
done
sleep 1
cnt=$((cnt + 1))
done
if [ -z "${wan_ok}" ]
local ipv4_nat
local ipv4_rej
local ipv6_nat
local ipv6_rej
# delete temporary files & directories
#
if [ -f "${adb_tmpfile}" ]
then
rc=585
wan_ok="false"
f_log "no wan/update interface(s) found (${adb_wandev# })" "${rc}"
f_restore
rm -f "${adb_tmpfile}" >/dev/null 2>&1
fi
if [ -d "${adb_tmpdir}" ]
then
rm -rf "${adb_tmpdir}" >/dev/null 2>&1
fi
}
#####################################
# f_ntpcheck: check/get ntp time sync
#
f_ntpcheck()
{
local cnt=0
local cnt_max="${1}"
local ntp_pool
for srv in ${adb_ntpsrv}
do
ntp_pool="${ntp_pool} -p ${srv}"
done
while [ $((cnt)) -le $((cnt_max)) ]
do
/usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
rc=${?}
if [ $((rc)) -eq 0 ]
# final log message and iptables statistics
#
if [ $((rc)) -eq 0 ]
then
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
ntp_ok="true"
f_log "get ntp time sync (${adb_ntpsrv# }), after ${cnt} loops"
break
ipv4_nat="$(iptables -t nat -vnL | grep -F "adb-nat" | grep -Eo "[0-9]+" | head -n1)"
ipv4_rej="$(iptables -vnL | grep -F "adb-rej" | grep -Eo "[0-9]+" | head -n1)"
ipv6_nat="$(ip6tables -t nat -vnL | grep -F "adb-nat" | grep -Eo "[0-9]+" | head -n1)"
ipv6_rej="$(ip6tables -vnL | grep -F "adb-rej" | grep -Eo "[0-9]+" | head -n1)"
f_log "adblock firewall statistics (IPv4/IPv6): ${ipv4_nat}/${ipv6_nat} packets redirected in PREROUTING chain, ${ipv4_rej}/${ipv6_rej} packets rejected in FORWARD chain"
elif [ -n "${adb_wanif4}" ]
then
ipv4_nat="$(iptables -t nat -vnL | grep -F "adb-nat" | grep -Eo "[0-9]+" | head -n1)"
ipv4_rej="$(iptables -vnL | grep -F "adb-rej" | grep -Eo "[0-9]+" | head -n1)"
f_log "adblock firewall statistics (IPv4): ${ipv4_nat} packets redirected in PREROUTING chain, ${ipv4_rej} packets rejected in FORWARD chain"
elif [ -n "${adb_wanif6}" ]
then
ipv6_nat="$(ip6tables -t nat -vnL | grep -F "adb-nat" | grep -Eo "[0-9]+" | head -n1)"
ipv6_rej="$(ip6tables -vnL | grep -F "adb-rej" | grep -Eo "[0-9]+" | head -n1)"
f_log "adblock firewall statistics (IPv6): ${ipv6_nat} packets redirected in PREROUTING chain, ${ipv6_rej} packets rejected in FORWARD chain"
fi
sleep 1
cnt=$((cnt + 1))
done
if [ -z "${ntp_ok}" ]
then
rc=590
ntp_ok="false"
f_log "ntp time sync failed (${adb_ntpsrv# })" "${rc}"
f_restore
f_log "domain adblock processing finished successfully (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
else
f_log "domain adblock processing failed (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
fi
exit ${rc}
}

+ 146
- 120
net/adblock/files/adblock-update.sh View File

@ -1,7 +1,7 @@
#!/bin/sh
#######################################################
# ad/abuse domain blocking script for dnsmasq/openwrt #
# written by Dirk Brenken (dirk@brenken.org) #
# written by Dirk Brenken (openwrt@brenken.org) #
#######################################################
# LICENSE
@ -25,7 +25,7 @@
# set script version
#
adb_version="0.40.2"
adb_version="0.60.0"
# get current pid, script directory and openwrt version
#
@ -39,7 +39,7 @@ if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
then
. "${adb_scriptdir}/adblock-helper.sh" 2>/dev/null
else
rc=600
rc=100
/usr/bin/logger -s -t "adblock[${pid}] error" "adblock function library not found, rc: ${rc}"
exit ${rc}
fi
@ -50,7 +50,7 @@ fi
# call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM)
#
trap "f_log 'trap error' '700'; f_restore" 1 2 3 10 11 15
trap "rc=255; f_log 'trap error' '${rc}'; f_restore" 1 2 3 10 11 15
# start logging
#
@ -80,71 +80,77 @@ then
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)"
list_time="${list_time/*: /}"
f_log "=> (pre-)processing adblock source '${src_name}'"
# only process shallalist archive with updated timestamp
# only process shallalist archive with updated timestamp,
# extract and merge only domains of selected shallalist categories
#
shalla_time="$(curl ${curl_parm} --max-time 5 --head "${adb_arc_shalla}" 2>/dev/null | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
shalla_time="$(wget ${wget_parm} --timeout=5 --server-response --spider "${adb_arc_shalla}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
shalla_time="${shalla_time/*: /}"
if [ -z "${shalla_time}" ]
then
shalla_time="$(date)"
f_log "no online timestamp received, current date will be used (${src_name})"
f_log " no online timestamp received, current date will be used"
fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ]
then
f_log "shallalist (pre-)processing started ..."
curl ${curl_parm} --max-time "${adb_maxtime}" "${adb_arc_shalla}" --output "${shalla_archive}" 2>/dev/null
wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document="${shalla_archive}" "${adb_arc_shalla}" 2>/dev/null
rc=${?}
if [ $((rc)) -ne 0 ]
if [ $((rc)) -eq 0 ]
then
f_log "source download failed (${src_name})" "${rc}"
f_restore
fi
> "${shalla_file}"
for category in ${adb_cat_shalla}
do
tar -xOzf "${shalla_archive}" BL/${category}/domains 2>/dev/null >> "${shalla_file}"
rc=${?}
if [ $((rc)) -ne 0 ]
then
f_log " archive extraction failed (${category})"
break
fi
done
# extract and merge only domains of selected shallalist categories
#
> "${shalla_file}"
for category in ${adb_cat_shalla}
do
tar -xOzf "${shalla_archive}" BL/${category}/domains 2>/dev/null >> "${shalla_file}"
rc=${?}
if [ $((rc)) -ne 0 ]
# remove temporary files
#
rm -f "${shalla_archive}" >/dev/null 2>&1
rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1
if [ $((rc)) -eq 0 ]
then
f_log "source archive extraction failed (${category})" "${rc}"
f_restore
adb_sources="${adb_sources} ${shalla_file}&ruleset=rset_shalla"
f_log " source archive (pre-)processing finished"
else
rc=0
fi
done
# remove temporary files
#
rm -f "${shalla_archive}" >/dev/null 2>&1
rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1
else
f_log " source archive download failed"
rc=0
fi
else
adb_srcfind="! -name ${adb_dnsprefix}.${src_name}"
f_log " source archive doesn't change, no update required"
fi
adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla"
fi
# add blacklist source to active adblock domain sources
#
if [ -n "${adb_sources}" ] && [ -s "${adb_blacklist}" ]
if [ -s "${adb_blacklist}" ]
then
adb_sources="${adb_sources} file://${adb_blacklist}&ruleset=rset_blacklist"
adb_sources="${adb_sources} ${adb_blacklist}&ruleset=rset_blacklist"
fi
# loop through active adblock domain sources,
# prepare output and store all extracted domains in temp file
# download sources, prepare output and store all extracted domains in temp file
#
for src in ${adb_sources}
do
# download selected adblock sources
#
url="${src/\&ruleset=*/}"
check_url="$(printf "${url}" | sed -n '/^https:/p' 2>/dev/null)"
src_name="${src/*\&ruleset=rset_/}"
adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)"
list_time="${list_time/*: /}"
f_log "=> processing adblock source '${src_name}'"
# prepare find statement for all active adblocklist sources
# prepare find statement with active adblock list sources
#
if [ -z "${adb_srcfind}" ]
then
@ -153,47 +159,40 @@ do
adb_srcfind="${adb_srcfind} -a ! -name ${adb_dnsprefix}.${src_name}"
fi
# wget/curl switch
# only download blocklist with newer/updated timestamp
# only download adblock list with newer/updated timestamp
#
if [ -n "${check_url}" ]
if [ "${src_name}" = "blacklist" ]
then
url_time="$(date -r "${adb_blacklist}" 2>/dev/null)"
elif [ "${src_name}" = "shalla" ]
then
url_time="${shalla_time}"
else
url_time="$(wget ${wget_parm} --timeout=5 --server-response --spider "${url}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
url_time="${url_time/*: /}"
if [ -z "${url_time}" ]
then
url_time="$(date)"
f_log "no online timestamp received, current date will be used (${src_name})"
fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
fi
if [ -z "${url_time}" ]
then
url_time="$(date)"
f_log " no online timestamp received, current date will be used"
fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
then
if [ "${src_name}" = "blacklist" ]
then
tmp_domains="$(wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
tmp_domains="$(cat "${adb_blacklist}" 2>/dev/null)"
rc=${?}
else
f_log "source doesn't change, no update required (${src_name})"
continue
fi
else
if [ "${src_name}" = "shalla" ]
then
url_time="${shalla_time}"
else
url_time="$(curl ${curl_parm} --max-time 5 --head "${url}" 2>/dev/null | grep -F "Last-Modified: " | tr -d '\r')"
url_time="${url_time/*: /}"
fi
if [ -z "${url_time}" ]
elif [ "${src_name}" = "shalla" ]
then
url_time="$(date)"
f_log "no online timestamp received, current date will be used (${src_name})"
fi
if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
then
tmp_domains="$(curl ${curl_parm} --max-time "${adb_maxtime}" "${url}" 2>/dev/null)"
tmp_domains="$(cat "${shalla_file}" 2>/dev/null)"
rc=${?}
else
f_log "source doesn't change, no update required (${src_name})"
continue
tmp_domains="$(wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
rc=${?}
fi
else
f_log " source doesn't change, no update required"
continue
fi
# check download result and prepare domain output by regex patterns
@ -202,7 +201,7 @@ do
then
eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')"
count="$(printf "%s\n" "${tmp_domains}" | tr '[A-Z]' '[a-z]' | eval "${ruleset}" | tee "${adb_tmpfile}" | wc -l)"
f_log "source download finished (${url}, ${count} entries)"
f_log " source download finished (${count} entries)"
if [ "${src_name}" = "shalla" ]
then
rm -f "${shalla_file}" >/dev/null 2>&1
@ -210,11 +209,12 @@ do
unset tmp_domains
elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
then
f_log "empty source download finished (${src_name})"
f_log " empty source download finished"
continue
else
f_log "source download failed (${src_name})" "${rc}"
f_restore
f_log " source download failed"
rc=0
continue
fi
# remove whitelist domains, sort domains and make them unique,
@ -231,7 +231,7 @@ do
rc=${?}
fi
# prepare find statement for revised adblocklist sources
# prepare find statement with revised adblock list sources
#
if [ -z "${adb_revsrcfind}" ]
then
@ -240,35 +240,43 @@ do
adb_revsrcfind="${adb_revsrcfind} -o -name ${adb_dnsprefix}.${src_name}"
fi
# write preliminary adblocklist footer
# write preliminary adblock list footer
#
if [ $((rc)) -eq 0 ]
then
count="$(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}")"
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
count="$(($(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}") / 2))"
else
count="$(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}")"
fi
printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsfile}"
printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsfile}"
printf "%s\n" "# source: ${url}" >> "${adb_dnsfile}"
printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}"
f_log "domain merging finished (${src_name})"
f_log " domain merging finished"
else
f_log "domain merging failed (${src_name})" "${rc}"
f_log " domain merging failed" "${rc}"
f_restore
fi
else
f_log "empty domain input received (${src_name})"
f_log " empty domain input received"
continue
fi
done
# remove old adblocklists and their backups
# remove old adblock lists and their backups
#
if [ -n "${adb_srcfind}" ]
then
find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srcfind} \) -exec rm -f "{}" \; 2>/dev/null
if [ $((rc)) -ne 0 ]
adb_rmfind="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srcfind} \) -print -exec rm -f "{}" \; 2>/dev/null)"
if [ $((rc)) -eq 0 ] && [ -n "${adb_rmfind}" ]
then
f_log "no longer used adblock lists removed" "${rc}"
elif [ $((rc)) -ne 0 ]
then
f_log "error during removal of old adblocklists" "${rc}"
f_remove
f_log "error during removal of old adblock lists" "${rc}"
f_exit
fi
if [ "${backup_ok}" = "true" ]
then
@ -276,7 +284,7 @@ then
if [ $((rc)) -ne 0 ]
then
f_log "error during removal of old backups" "${rc}"
f_remove
f_exit
fi
fi
else
@ -284,19 +292,19 @@ else
if [ "${backup_ok}" = "true" ]
then
rm -f "${adb_backupdir}/${adb_dnsprefix}."* >/dev/null 2>&1
f_log "all available adblocklists and backups removed"
f_log "all available adblock lists and backups removed"
else
f_log "all available adblocklists removed"
f_log "all available adblock lists removed"
fi
fi
# make separate adblocklists unique
# make separate adblock lists unique
#
if [ $((adb_unique)) -eq 1 ]
then
if [ -n "${adb_revsrcfind}" ]
then
f_log "remove duplicates in separate adblocklists ..."
f_log "remove duplicates in separate adblock lists"
# generate a temporary, unique overall list
#
@ -320,7 +328,12 @@ then
# write final adblocklist footer
#
count="$(wc -l < "${adb_dnsdir}/tmp.${list}")"
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
count="$(($(wc -l < "${adb_dnsdir}/tmp.${list}") / 2))"
else
count="$(wc -l < "${adb_dnsdir}/tmp.${list}")"
fi
printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsdir}/tmp.${list}"
printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/tmp.${list}"
tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}"
@ -330,49 +343,62 @@ then
fi
fi
# restart dnsmasq with newly generated block lists
# get overall count
#
/etc/init.d/dnsmasq restart >/dev/null 2>&1
sleep 3
if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
then
adb_count="$(($(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l) / 2))"
else
adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)"
fi
# dnsmasq health check
# restart dnsmasq with newly generated or deleted adblock lists,
# check dnsmasq startup afterwards
#
dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up" 2>/dev/null)"
if [ -z "${dns_status}" ]
if [ -n "${adb_revsrcfind}" ] || [ -n "${adb_rmfind}" ]
then
dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep -F "${adb_ip}" 2>/dev/null)"
if [ -z "${dns_status}" ]
/etc/init.d/dnsmasq restart >/dev/null 2>&1
sleep 2
dns_status="$(ps 2>/dev/null | grep "[d]nsmasq" 2>/dev/null)"
if [ -n "${dns_status}" ]
then
adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)"
if [ "${backup_ok}" = "true" ]
f_log "adblock lists with overall ${adb_count} domains loaded"
else
rc=105
f_log "dnsmasq restart failed, please check 'logread' output" "${rc}"
f_restore
fi
else
f_log "adblock lists with overall ${adb_count} domains are still valid, no dnsmasq restart required"
fi
# create adblock list backups
#
if [ "${backup_ok}" = "true" ] && [ "$(printf "${adb_dnsdir}/${adb_dnsprefix}."*)" != "${adb_dnsdir}/${adb_dnsprefix}.*" ]
then
for file in ${adb_dnsdir}/${adb_dnsprefix}.*
do
filename="${file##*/}"
if [ ! -f "${adb_backupdir}/${filename}" ] || [ "${file}" -nt "${adb_backupdir}/${filename}" ]
then
if [ -n "${adb_revsrcfind}" ]
cp -pf "${file}" "${adb_backupdir}" 2>/dev/null
rc=${?}
if [ $((rc)) -ne 0 ]
then
find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec cp -f "{}" "${adb_backupdir}" \; 2>/dev/null
rc=${?}
if [ $((rc)) -ne 0 ]
then
f_log "error during backup of adblocklists" "${rc}"
f_remove
fi
f_log "adblocklists with overall ${adb_count} domains loaded, new backups generated"
else
f_log "adblocklists with overall ${adb_count} domains loaded, no new backups"
f_log "error during backup of adblock list (${filename})" "${rc}"
f_exit
fi
else
f_log "adblocklists with overall ${adb_count} domains loaded, backups disabled"
backup_done="true"
fi
done
if [ "${backup_done}" = "true" ]
then
f_log "new adblock list backups generated"
else
rc=605
f_log "nslookup probe failed" "${rc}"
f_restore
f_log "adblock list backups are still valid, no new backups required"
fi
else
rc=610
f_log "dnsmasq probe failed" "${rc}"
f_restore
fi
# remove temporary files and exit
#
f_remove
f_exit

+ 0
- 1
net/adblock/files/adblock.blacklist View File

@ -1 +0,0 @@

+ 8
- 21
net/adblock/files/adblock.conf View File

@ -1,20 +1,8 @@
# adblock configuration, for further information
# please read /etc/adblock/samples/adblock.conf.sample
# and /etc/adblock/README.md
#
config adblock "global"
option adb_ip "192.168.2.1"
option adb_domain "heise.de"
option adb_blacklist "/etc/adblock/adblock.blacklist"
option adb_whitelist "/etc/adblock/adblock.whitelist"
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
config service "ntpcheck"
option enabled "0"
config service "backup"
option enabled "0"
option adb_backupdir "/tmp"
@ -23,17 +11,12 @@ config service "debuglog"
option enabled "0"
option adb_logfile "/tmp/adb_debug.log"
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/adb_query.log"
option adb_queryhistory "1"
config source "adaway"
option enabled "0"
option enabled "1"
option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
config source "disconnect"
option enabled "0"
option enabled "1"
option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
config source "dshield"
@ -46,7 +29,11 @@ config source "feodo"
config source "malware"
option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
option adb_src_malware "https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware"
config source "malwarelist"
option enabled "0"
option adb_src_malwarelist "http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist"
config source "palevo"
option enabled "0"


+ 0
- 1
net/adblock/files/adblock.whitelist View File

@ -1 +0,0 @@

+ 33
- 40
net/adblock/files/samples/adblock.conf.sample View File

@ -1,18 +1,30 @@
# adblock configuration, for further information
# please read /etc/adblock/samples/adblock.conf.sample
# please see /etc/adblock/samples/adblock.conf.sample
# and /etc/adblock/README.md
#
# generic options (always required)
#
# generic options (all optional!)
config adblock "global"
# ip address of the local adblock interface/uhttpd instance,
# needs to be a different subnet from the normal LAN
option adb_ip "192.168.2.1"
# name (or space separated list of names) of the logical wan interface(s)
option adb_wanif "wan"
# name of an "always accessible" domain,
# this domain will be used for the final nslookup check
option adb_domain "heise.de"
# name of the logical lan interface
option adb_lanif "lan"
# uhttpd port for adblock instance
option adb_port "65535"
# ipv4 blackhole ip address for dnsmasq
# old class 'E' subnet, reserved for future use (certainly not used in normal setups)
option adb_nullipv4 "254.0.0.1"
# ipv6 mapped blackhole ip address for dnsmasq
option adb_nullipv6 "::ffff:fe00:0001"
# download timeout (in seconds) to get the adblock list sources
option adb_maxtime "60"
# max loops/timeout for wan check
option adb_maxloop "20"
# full path to static domain blacklist file (one domain per line)
# wildcards or regex expressions are not allowed
@ -22,20 +34,6 @@ config adblock "global"
# wildcards or regex expressions are not allowed
option adb_whitelist "/etc/adblock/adblock.whitelist"
# list of wan devices that are allowed for adblock updates (check /sys/class/net/<dev>),
# if no one found the last adlist backup will be restored,
# useful only for (mobile) multiwan setups
# disabled by default
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
# check that ntp has adjusted the system time on this device,
# will be used for logfile writing and logfile housekeeping
# disabled by default
config service "ntpcheck"
option enabled "0"
# full path to backup directory for adlist backups
# disabled by default
config service "backup"
@ -43,32 +41,23 @@ config service "backup"
option adb_backupdir "/tmp"
# full path to debug logfile
# by default adblock logs to syslog and stdout only
# normally adblock logs to syslog and stdout only
# disabled by default
config service "debuglog"
option enabled "0"
option adb_logfile "/tmp/adb_debug.log"
# full path to domain query logfile
# a background task will trace every dns request to file, to easily identify free and already blocked domains,
# for this to work, you've to enable the dnsmasq option "logqueries" too.
# the "queryhistory" option deletes query logfiles older than n days (req. busybox find with mtime support)
# disabled by default
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/adb_query.log"
option adb_queryhistory "1"
# different adblock list sources
# please do not change the urls listed below,
# adblock list source definition
# please do not change the URLs listed below,
# enable/disable sources as needed
# for shallalist you can also enable/disable different ad categories
# 'adaway', 'disconnect' and 'yoyo' are enabled by default
config source "adaway"
option enabled "0"
option enabled "1"
option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
config source "disconnect"
option enabled "0"
option enabled "1"
option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
config source "dshield"
@ -81,7 +70,11 @@ config source "feodo"
config source "malware"
option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
option adb_src_malware "https://mirror.cedia.org.ec/malwaredomains/justdomains&ruleset=rset_malware"
config source "malwarelist"
option enabled "0"
option adb_src_malwarelist "http://www.malwaredomainlist.com/hostslist/hosts.txt&ruleset=rset_malwarelist"
config source "palevo"
option enabled "0"


+ 1
- 1
net/adblock/files/samples/dhcp.config.sample View File

@ -4,4 +4,4 @@
config dnsmasq
option cachesize '1000'
option filterwin2k '0'
option logqueries '1'

+ 1
- 1
net/adblock/files/samples/dnsmasq.conf.sample View File

@ -1,4 +1,4 @@
# tell DHCP clients to not ask for proxy information
# tell DHCP clients not to ask for proxy information
# some clients - like Win7 - will constantly ask if not told "No!"
# configuration found in /etc/dnsmasq
#


+ 0
- 5
net/adblock/files/samples/firewall.user.sample View File

@ -1,5 +0,0 @@
# redirect/force all dns queries to port 53 of your router
# configuration found in /etc/firewall.user
#
iptables -t nat -I PREROUTING -p tcp --dport 53 -j REDIRECT --to-ports 53
iptables -t nat -I PREROUTING -p udp --dport 53 -j REDIRECT --to-ports 53

+ 1
- 2
net/adblock/files/samples/root.crontab.sample View File

@ -2,7 +2,6 @@
# configuration found in /etc/crontabs/root
#
# start adblock script twice a day
# start adblock script once a day at 6 o'clock
#
0 06 * * * /usr/bin/adblock-update.sh &
0 22 * * * /usr/bin/adblock-update.sh &

+ 0
- 4
net/adblock/files/samples/uhttpd.config.sample View File

@ -1,4 +0,0 @@
# main uhttpd instance listens only to internal LAN
#
config uhttpd 'main'
list listen_http '192.168.1.1:80'

+ 1
- 1
net/adblock/files/www/adblock/adblock.html View File

@ -1,6 +1,6 @@
<html>
<head></head>
<body>
<img src="/adblock.png" border=0 alt=""></img>
<img src="/adblock.png" border="0" alt=""></img>
</body>
</html>

+ 1
- 1
net/adblock/files/www/adblock/index.html View File

@ -1,6 +1,6 @@
<html>
<head></head>
<body>
<img src="/adblock.png" border=0 alt=""></img>
<img src="/adblock.png" border="0" alt=""></img>
</body>
</html>

Loading…
Cancel
Save