Browse Source

adblock: uci support, dynamic uhttpd instance support plus various fixes

* add uci support
* add dynamic uhttpd instance support
  (no longer rely on uhttpd config changes)
* package reordering
* plus various fixes

Signed-off-by: Dirk Brenken <dirk@brenken.org>
lilik-openwrt-22.03
Dirk Brenken 9 years ago
parent
commit
1f5ed5d040
15 changed files with 790 additions and 622 deletions
  1. +11
    -7
      net/adblock/Makefile
  2. +13
    -12
      net/adblock/files/README.md
  3. +566
    -0
      net/adblock/files/adblock-helper.sh
  4. +48
    -51
      net/adblock/files/adblock-update.sh
  5. +0
    -0
      net/adblock/files/adblock.blacklist
  6. +76
    -0
      net/adblock/files/adblock.conf
  7. +0
    -0
      net/adblock/files/adblock.whitelist
  8. +0
    -118
      net/adblock/files/etc/adblock/adblock.conf
  9. +76
    -0
      net/adblock/files/samples/adblock.conf.sample
  10. +0
    -0
      net/adblock/files/samples/dhcp.config.sample
  11. +0
    -0
      net/adblock/files/samples/dnsmasq.conf.sample
  12. +0
    -0
      net/adblock/files/samples/firewall.user.sample
  13. +0
    -0
      net/adblock/files/samples/rc.local.sample
  14. +0
    -0
      net/adblock/files/samples/root.crontab.sample
  15. +0
    -434
      net/adblock/files/usr/bin/adblock-helper.sh

+ 11
- 7
net/adblock/Makefile View File

@ -7,7 +7,7 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=adblock
PKG_VERSION:=0.11.0
PKG_VERSION:=0.20.0
PKG_RELEASE:=1
PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org>
@ -40,7 +40,7 @@ The script supports the following domain blacklist sites:
endef
define Package/$(PKG_NAME)/conffiles
/etc/adblock/adblock.conf
/etc/config/adblock
/etc/adblock/adblock.whitelist
/etc/adblock/adblock.blacklist
endef
@ -56,15 +56,19 @@ endef
define Package/$(PKG_NAME)/install
$(INSTALL_DIR) $(1)/usr/bin
$(INSTALL_BIN) ./files/usr/bin/adblock-update.sh $(1)/usr/bin/
$(INSTALL_DATA) ./files/usr/bin/adblock-helper.sh $(1)/usr/bin/
$(INSTALL_BIN) ./files/adblock-update.sh $(1)/usr/bin/
$(INSTALL_DATA) ./files/adblock-helper.sh $(1)/usr/bin/
$(INSTALL_DIR) $(1)/etc/config
$(INSTALL_CONF) ./files/adblock.conf $(1)/etc/config/adblock
$(INSTALL_DIR) $(1)/etc/adblock
$(INSTALL_CONF) ./files/etc/adblock/adblock.* $(1)/etc/adblock/
$(INSTALL_CONF) ./files/etc/adblock/README.md $(1)/etc/adblock/
$(INSTALL_CONF) ./files/adblock.blacklist $(1)/etc/adblock/
$(INSTALL_CONF) ./files/adblock.whitelist $(1)/etc/adblock/
$(INSTALL_CONF) ./files/README.md $(1)/etc/adblock/
$(INSTALL_DIR) $(1)/etc/adblock/samples
$(INSTALL_CONF) ./files/etc/adblock/samples/*.sample $(1)/etc/adblock/samples/
$(INSTALL_CONF) ./files/samples/*.sample $(1)/etc/adblock/samples/
$(INSTALL_DIR) $(1)/www/adblock
$(INSTALL_DATA) ./files/www/adblock/* $(1)/www/adblock/


net/adblock/files/etc/adblock/README.md → net/adblock/files/README.md View File


+ 566
- 0
net/adblock/files/adblock-helper.sh View File

@ -0,0 +1,566 @@
#!/bin/sh
##############################################
# function library used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
##############################################
#####################################
# f_envload: load adblock environment
#
f_envload()
{
# source in openwrt function library
#
if [ -r "/lib/functions.sh" ]
then
. /lib/functions.sh
else
/usr/bin/logger -t "adblock[${pid}]" "error: openwrt function library not found"
f_deltemp
exit 10
fi
# source in openwrt json helpers library
#
if [ -r "/usr/share/libubox/jshn.sh" ]
then
. "/usr/share/libubox/jshn.sh"
else
/usr/bin/logger -t "adblock[${pid}]" "error: openwrt json helpers library not found"
f_deltemp
exit 15
fi
# get list with all installed openwrt packages
#
pkg_list="$(opkg list-installed 2>/dev/null)"
if [ -z "${pkg_list}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: empty openwrt package list"
f_deltemp
exit 20
fi
}
######################################################
# f_envparse: parse adblock config and set environment
#
f_envparse()
{
# function to read/set global options by callback,
# prepare list items and build option list for all others
#
config_cb()
{
local type="${1}"
local name="${2}"
if [ "${type}" = "adblock" ]
then
option_cb()
{
local option="${1}"
local value="${2}"
eval "${option}=\"${value}\""
}
else
option_cb()
{
local option="${1}"
local value="${2}"
local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p')"
if [ -z "${opt_out}" ]
then
all_options="${all_options} ${option}"
fi
}
list_cb()
{
local list="${1}"
local value="${2}"
if [ "${list}" = "adb_wanlist" ]
then
adb_wandev="${adb_wandev} ${value}"
elif [ "${list}" = "adb_ntplist" ]
then
adb_ntpsrv="${adb_ntpsrv} ${value}"
elif [ "${list}" = "adb_catlist" ]
then
adb_cat_shalla="${adb_cat_shalla} ${value}"
fi
}
fi
}
# function to iterate through option list, read/set all options in "enabled" sections
#
parse_config()
{
local config="${1}"
config_get switch "${config}" "enabled"
if [ "${switch}" = "1" ]
then
for option in ${all_options}
do
config_get value "${config}" "${option}"
if [ -n "${value}" ]
then
local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p')"
if [ -n "${opt_src}" ]
then
adb_sources="${adb_sources} ${value}"
else
eval "${option}=\"${value}\""
fi
fi
done
elif [ "${config}" = "wancheck" ]
then
unset adb_wandev
elif [ "${config}" = "ntpcheck" ]
then
unset adb_ntpsrv
elif [ "${config}" = "shalla" ]
then
unset adb_cat_shalla
fi
}
# load adblock config and start parsing functions
#
config_load adblock
config_foreach parse_config service
config_foreach parse_config source
# set temp variables and counter
#
adb_tmpfile="$(mktemp -tu)"
adb_tmpdir="$(mktemp -d)"
cnt=0
max_cnt=30
max_time=60
# set adblock source ruleset definitions
#
rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
rset_end="sed '/^[#/:;_\s]*$/d'"
rset_default="${rset_start} | ${rset_end}"
rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
# set adblock/dnsmasq destination file and format
#
adb_dnsfile="/tmp/dnsmasq.d/adlist.conf"
adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
}
#############################################
# f_envcheck: check environment prerequisites
#
f_envcheck()
{
# check adblock network device configuration
#
if [ ! -d "/sys/class/net/${adb_dev}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network device input (${adb_dev})"
f_deltemp
exit 25
fi
# check adblock network interface configuration
#
check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')"
banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network interface input (${adb_if})"
f_deltemp
exit 30
fi
# check adblock ip address configuration
#
check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
if [ -z "${check_ip}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock ip address input (${adb_ip})"
f_deltemp
exit 35
fi
# check adblock blacklist/whitelist configuration
#
if [ ! -r "${adb_blacklist}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: adblock blacklist not found"
f_deltemp
exit 40
elif [ ! -r "${adb_whitelist}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: adblock whitelist not found"
f_deltemp
exit 45
fi
# check wan update configuration
#
if [ -n "${adb_wandev}" ]
then
wan_ok="true"
else
wan_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: wan update check will be disabled"
fi
# check ntp sync configuration
#
if [ -n "${adb_ntpsrv}" ]
then
ntp_ok="true"
else
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: ntp time sync will be disabled"
fi
# check backup configuration
#
adb_backupdir="${adb_backupfile%/*}"
if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
then
backup_ok="true"
adb_mounts="${adb_backupdir} ${adb_tmpdir}"
else
backup_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: backup/restore will be disabled"
fi
# check error log configuration
#
adb_logdir="${adb_logfile%/*}"
if [ -n "${adb_logfile}" ] && [ "${adb_logfile}" = "/dev/stdout" ]
then
log_ok="true"
adb_logfile="/proc/self/fd/1"
elif [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] && [ "${ntp_ok}" = "true" ]
then
log_ok="true"
adb_mounts="${adb_mounts} ${adb_logdir}"
else
log_ok="false"
adb_logfile="/dev/null"
/usr/bin/logger -t "adblock[${pid}]" "info: error logging will be disabled"
fi
# check dns query log configuration
#
adb_querydir="${adb_queryfile%/*}"
if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
then
# check find capabilities
#
check="$(find --help 2>&1 | grep "mtime")"
if [ -z "${check}" ]
then
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
else
query_ok="true"
query_name="${adb_queryfile##*/}"
query_ip="${adb_ip//./\\.}"
query_pid="/var/run/query.pid"
adb_mounts="${adb_mounts} ${adb_querydir}"
fi
else
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "info: dns query logging will be disabled"
fi
# check mount points & space requirements
#
adb_mounts="${adb_mounts} ${adb_tmpdir}"
for mp in ${adb_mounts}
do
df "${mp}" 2>/dev/null |\
tail -n1 |\
while read filesystem overall used available scrap
do
av_space="${available}"
if [ $((av_space)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: no space left on device/not mounted (${mp})"
exit 50
elif [ $((av_space)) -lt $((adb_minspace)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: not enough space on device (${mp})"
exit 55
fi
done
# subshell return code handling
#
rc=$?
if [ $((rc)) -ne 0 ]
then
f_deltemp
exit ${rc}
fi
done
# check curl package dependency
#
check="$(printf "${pkg_list}" | grep "^curl")"
if [ -z "${check}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: curl package not found"
f_deltemp
exit 60
fi
# check wget package dependency
#
check="$(printf "${pkg_list}" | grep "^wget")"
if [ -z "${check}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: wget package not found"
f_deltemp
exit 65
fi
# check dynamic/volatile adblock network interface configuration
#
rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)"
if [ $((rc)) -ne 0 ]
then
json_init
json_add_string name "${adb_if}"
json_add_string ifname "${adb_dev}"
json_add_string proto "static"
json_add_array ipaddr
json_add_string "" "${adb_ip}"
json_close_array
json_close_object
ubus call network add_dynamic "$(json_dump)"
/usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
fi
# check dynamic/volatile adblock uhttpd instance configuration
#
rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf $?)"
if [ $((rc)) -ne 0 ]
then
uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80"
/usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
fi
}
###################################################
# f_deltemp: delete temporary files and directories
#
f_deltemp()
{
if [ -f "${adb_tmpfile}" ]
then
rm -f "${adb_tmpfile}" 2>/dev/null
fi
if [ -d "${adb_tmpdir}" ]
then
rm -rf "${adb_tmpdir}" 2>/dev/null
fi
}
################################################################
# f_remove: remove temporary files, start and maintain query log
#
f_remove()
{
# delete temporary files and directories
#
f_deltemp
# kill existing domain query log background process,
# housekeeping and start of a new process on daily basis
#
if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
query_date="$(date "+%Y%m%d")"
if [ -s "${query_pid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
then
kill -9 $(< "${query_pid}") 2>/dev/null
> "${query_pid}"
find "${adb_backupdir}" -maxdepth 1 -type f -mtime +${adb_queryhistory} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "info: kill old query log background process and do logfile housekeeping"
fi
if [ ! -s "${query_pid}" ]
then
( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" &
/usr/bin/logger -t "adblock[${pid}]" "info: start new domain query log background process"
fi
fi
# final log entry
#
/usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing finished (${adb_version})"
}
#####################################################
# f_restore: if available, restore last adlist backup
#
f_restore()
{
if [ -z "${restore_msg}" ]
then
restore_msg="unknown"
fi
if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ]
then
cp -f "${adb_backupfile}" "${adb_dnsfile}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, adlist backup restored"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${adb_logfile}"
else
> "${adb_dnsfile}"
/usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, empty adlist generated"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${adb_logfile}"
fi
# restart dnsmasq
#
/etc/init.d/dnsmasq restart >/dev/null 2>&1
# remove files and exit
#
f_remove
exit 100
}
#######################################################
# f_wancheck: check for usable adblock update interface
#
f_wancheck()
{
if [ "${wan_ok}" = "true" ]
then
# wait for wan update interface(s)
#
while [ $((cnt)) -le $((max_cnt)) ]
do
for dev in ${adb_wandev}
do
if [ -d "/sys/class/net/${dev}" ]
then
dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null)
if [ "${dev_out}" = "up" ]
then
/usr/bin/logger -t "adblock[${pid}]" "info: get wan/update interface: ${dev}, after ${cnt} loops"
break 2
fi
fi
if [ $((cnt)) -eq $((max_cnt)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "error: no wan/update interface(s) found (${adb_wandev})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${adb_wandev})" >> "${adb_logfile}"
restore_msg="no wan/update interface(s)"
f_restore
fi
done
sleep 1
cnt=$((cnt + 1))
done
fi
}
#####################################
# f_ntpcheck: check/get ntp time sync
#
f_ntpcheck()
{
if [ "${ntp_ok}" = "true" ]
then
# prepare ntp server pool
#
unset ntp_pool
for srv in ${adb_ntpsrv}
do
ntp_pool="${ntp_pool} -p ${srv}"
done
# wait for ntp time sync
#
while [ $((cnt)) -le $((max_cnt)) ]
do
/usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
rc=$?
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "info: get ntp time sync (${adb_ntpsrv}), after ${cnt} loops"
break
fi
if [ $((cnt)) -eq $((max_cnt)) ]
then
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "error: ntp time sync failed (${adb_ntpsrv})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${adb_ntpsrv})" >> "${adb_logfile}"
restore_msg="time sync failed"
f_restore
fi
sleep 1
cnt=$((cnt + 1))
done
fi
}
#################################################################
# f_dnscheck: dnsmasq health check with newly generated blocklist
#
f_dnscheck()
{
# check 1: dnsmasq startup
#
dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
if [ -z "${dns_status}" ]
then
# check 2: nslookup probe
#
dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")"
if [ -z "${dns_status}" ]
then
# create backup of new block list only, if both checks are OK and backup enabled
#
if [ "${backup_ok}" = "true" ]
then
cp -f "${adb_dnsfile}" "${adb_backupfile}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, backup generated"
else
/usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, no backup"
fi
else
restore_msg="nslookup probe failed"
f_restore
fi
else
restore_msg="dnsmasq probe failed"
f_restore
fi
}
##########################################################
# f_footer: write footer with a few statistics to dns file
#
f_footer()
{
adb_count="$(wc -l < "${adb_dnsfile}")"
printf "%s\n" "###################################################" >> "${adb_dnsfile}"
printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}"
printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}"
printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}"
for src in ${adb_sources}
do
url="${src//\&ruleset=*/}"
printf "%s\n" "# ${url}" >> "${adb_dnsfile}"
done
printf "%s\n" "###################################################" >> "${adb_dnsfile}"
printf "%s\n" "# domain whitelist source:" >> "${adb_dnsfile}"
printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}"
printf "%s\n" "###################################################" >> "${adb_dnsfile}"
}

net/adblock/files/usr/bin/adblock-update.sh → net/adblock/files/adblock-update.sh View File


net/adblock/files/etc/adblock/adblock.blacklist → net/adblock/files/adblock.blacklist View File


+ 76
- 0
net/adblock/files/adblock.conf View File

@ -0,0 +1,76 @@
# adblock configuration
#
config adblock "global"
option adb_ip "192.168.2.1"
option adb_dev "eth0"
option adb_if "adblock"
option adb_domain "heise.de"
option adb_minspace "100000"
option adb_blacklist "/etc/adblock/adblock.blacklist"
option adb_whitelist "/etc/adblock/adblock.whitelist"
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
config service "ntpcheck"
option enabled "0"
list adb_ntplist "0.pool.ntp.org"
list adb_ntplist "1.pool.ntp.org"
list adb_ntplist "2.pool.ntp.org"
list adb_ntplist "3.pool.ntp.org"
config service "backup"
option enabled "0"
option adb_backupfile "/tmp/adlist.backup"
config service "errorlog"
option enabled "0"
option adb_logfile "/tmp/error.log"
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/query.log"
option adb_queryhistory "1"
config source "yoyo"
option enabled "1"
option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
config source "malware"
option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default"
config source "zeus"
option enabled "0"
option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default"
config source "feodo"
option enabled "0"
option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default"
config source "palevo"
option enabled "0"
option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default"
config source "dshield"
option enabled "0"
option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default"
config source "spam404"
option enabled "0"
option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404"
config source "winhelp"
option enabled "0"
option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
config source "shalla"
option enabled "0"
option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
list adb_catlist "adv"
list adb_catlist "costtraps"
list adb_catlist "downloads"
list adb_catlist "spyware"
list adb_catlist "tracker"
list adb_catlist "warez"

net/adblock/files/etc/adblock/adblock.whitelist → net/adblock/files/adblock.whitelist View File


+ 0
- 118
net/adblock/files/etc/adblock/adblock.conf View File

@ -1,118 +0,0 @@
###################################################
# adblock configuration used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
###################################################
# adblock ip address and device information
# adb_ip => ip address of the local adblock loopback interface/uhttpd instance
# adb_dev => name of the physical adblock network device
# adb_if => name of the adblock network interface and uhttpd instance
#
adb_ip="192.168.2.1"
adb_dev="eth0"
adb_if="adblock"
# backup file and default restore message
# backup_dir => full path to directory for logging and backups, an empty entry disables this feature
# backup_file => full path to backup file
# restore_msg => default restore message
#
backup_dir=""
backup_file="${backup_dir}/adlist.backup"
restore_msg="unknown error"
# wan/update interface(s)
# wan_dev => space separated list of devices that are allowed for adblock updates (check /sys/class/net/<dev>),
# an empty entry disables this feature
#
wan_dev=""
# local adblock black & white list for manual overrides
# adb_blacklist => full path to domain blacklist file
# adb_whitelist => full path to domain whitelist file
#
adb_blacklist="/etc/adblock/adblock.blacklist"
adb_whitelist="/etc/adblock/adblock.whitelist"
# error logfile
# log_file => full path to additional logfile for error logging,
# "/dev/stdout" redirect errors to console, an empty entry disables this feature
# this feature requires ntp time sync too (see below)
#
log_file="${backup_dir}/error.log"
# domain query logfile
# query_file => full path to additional domain query logfile,
# an empty entry or an empty backup_dir entry disables this feature
# query_history => delete query logfiles older than n days (req. busybox find with mtime support)
#
query_file="${backup_dir}/query.log"
query_history=1
query_name="$(printf "${query_file}" | sed 's/.*\///g')"
query_ip="$(printf "${adb_ip}" | sed 's/\./\\./g')"
query_pid="/var/run/query.pid"
# ntp time server pool
# ntp_srv => space separated list of ntp serverpools, an empty entry disables this feature
#
ntp_srv="ntp0.fau.de ntp1.fau.de ntp2.fau.de ntp3.fau.de"
# domain for nslookup probe
# check_domain => check the newly generated domain blocklist with this domain
#
check_domain="heise.de"
# set startup counter and download timeout defaults
# max_cnt => wait n seconds/loops for wan & ntp check in sum
# max_time => wait n seconds for every source download
#
cnt=0
max_cnt=30
max_time=60
# minimum values for environment checks
# min_release => minimum required openwrt release number
# min_space => minimum required space for backups & logfiles (in kbyte)
#
min_release=47025
min_space=100000
# shallalist url, categories and local naming
# shalla_cat => space separated list of categories to use from shallalist archive
#
shalla_cat="adv costtraps downloads spyware tracker warez"
shalla_url="http://www.shallalist.de/Downloads/shallalist.tar.gz"
shalla_archive="${tmp_dir}/shallalist.tar.gz"
shalla_file="${tmp_dir}/shallalist.txt"
# remote and local domain block list sources
# adb_source => comment out sources you don't want to use
#
unset adb_source
adb_source="${adb_source} http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=yoyo"
#adb_source="${adb_source} http://mirror1.malwaredomains.com/files/justdomains&ruleset=default"
#adb_source="${adb_source} https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=default"
#adb_source="${adb_source} https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=default"
#adb_source="${adb_source} https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=default"
#adb_source="${adb_source} http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=default"
#adb_source="${adb_source} file:///${shalla_file}&ruleset=shalla"
#adb_source="${adb_source} http://spam404bl.com/spam404scamlist.txt&ruleset=spam404"
#adb_source="${adb_source} http://winhelp2002.mvps.org/hosts.txt&ruleset=winhelp"
adb_source="${adb_source} file://${adb_blacklist}&ruleset=default"
# adblock source ruleset definitions
# note: please keep the default settings below
#
rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
rset_end="sed '/^[#/:;_\s]*$/d'"
rset_default="${rset_start} | ${rset_end}"
rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
# dnsmasq destination file and format
# note: please keep the default settings below
#
dns_file="/tmp/dnsmasq.d/adlist.conf"
dns_format="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"

+ 76
- 0
net/adblock/files/samples/adblock.conf.sample View File

@ -0,0 +1,76 @@
# adblock configuration
#
config adblock "global"
option adb_ip "192.168.2.1"
option adb_dev "eth0"
option adb_if "adblock"
option adb_domain "heise.de"
option adb_minspace "100000"
option adb_blacklist "/etc/adblock/adblock.blacklist"
option adb_whitelist "/etc/adblock/adblock.whitelist"
config service "wancheck"
option enabled "0"
list adb_wanlist "wan"
config service "ntpcheck"
option enabled "0"
list adb_ntplist "0.pool.ntp.org"
list adb_ntplist "1.pool.ntp.org"
list adb_ntplist "2.pool.ntp.org"
list adb_ntplist "3.pool.ntp.org"
config service "backup"
option enabled "0"
option adb_backupfile "/tmp/adlist.backup"
config service "errorlog"
option enabled "0"
option adb_logfile "/tmp/error.log"
config service "querylog"
option enabled "0"
option adb_queryfile "/tmp/query.log"
option adb_queryhistory "1"
config source "yoyo"
option enabled "1"
option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"
config source "malware"
option enabled "0"
option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default"
config source "zeus"
option enabled "0"
option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default"
config source "feodo"
option enabled "0"
option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default"
config source "palevo"
option enabled "0"
option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default"
config source "dshield"
option enabled "0"
option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default"
config source "spam404"
option enabled "0"
option adb_src_spam404 "http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404"
config source "winhelp"
option enabled "0"
option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
config source "shalla"
option enabled "0"
option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
list adb_catlist "adv"
list adb_catlist "costtraps"
list adb_catlist "downloads"
list adb_catlist "spyware"
list adb_catlist "tracker"
list adb_catlist "warez"

net/adblock/files/etc/adblock/samples/dhcp.config.sample → net/adblock/files/samples/dhcp.config.sample View File


net/adblock/files/etc/adblock/samples/dnsmasq.conf.sample → net/adblock/files/samples/dnsmasq.conf.sample View File


net/adblock/files/etc/adblock/samples/firewall.user.sample → net/adblock/files/samples/firewall.user.sample View File


net/adblock/files/etc/adblock/samples/rc.local.sample → net/adblock/files/samples/rc.local.sample View File


net/adblock/files/etc/adblock/samples/root.crontab.sample → net/adblock/files/samples/root.crontab.sample View File


+ 0
- 434
net/adblock/files/usr/bin/adblock-helper.sh View File

@ -1,434 +0,0 @@
##############################################
# function library used by adblock-update.sh #
# written by Dirk Brenken (dirk@brenken.org) #
##############################################
#############################################
# f_envcheck: check environment prerequisites
#
f_envcheck()
{
# source in json helpers library
#
if [ -r "/usr/share/libubox/jshn.sh" ]
then
. "/usr/share/libubox/jshn.sh"
else
/usr/bin/logger -t "adblock[${pid}]" "json helpers library not found"
f_deltemp
exit 10
fi
# check adblock network device configuration
#
if [ ! -d "/sys/class/net/${adb_dev}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "invalid adblock network device input (${adb_dev})"
f_deltemp
exit 15
fi
# check adblock network interface configuration
#
check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')"
banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "invalid adblock network interface input (${adb_if})"
f_deltemp
exit 20
fi
# check adblock ip address configuration
#
check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
if [ -z "${check_ip}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "invalid adblock ip address input (${adb_ip})"
f_deltemp
exit 25
fi
# check adblock blacklist/whitelist configuration
#
if [ ! -r "${adb_blacklist}" ] || [ ! -r "${adb_whitelist}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "adblock blacklist or whitelist not found"
f_deltemp
exit 30
fi
# check wan update configuration
#
if [ -n "${wan_dev}" ]
then
wan_ok="true"
else
wan_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "wan update check will be disabled"
fi
# check ntp sync configuration
#
if [ -n "${ntp_srv}" ]
then
ntp_ok="true"
else
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "ntp time sync will be disabled"
fi
# check backup configuration
#
if [ -n "${backup_dir}" ] && [ -d "${backup_dir}" ]
then
backup_ok="true"
mounts="${backup_dir} ${tmp_dir}"
else
backup_ok="false"
mounts="${tmp_dir}"
/usr/bin/logger -t "adblock[${pid}]" "backup/restore will be disabled"
fi
# check error log configuration
#
if [ "${log_file}" = "/dev/stdout" ]
then
log_ok="true"
log_file="/proc/self/fd/1"
elif [ -n "${log_file}" ] && [ "${backup_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
log_ok="true"
else
log_ok="false"
log_file="/dev/null"
/usr/bin/logger -t "adblock[${pid}]" "error logging will be disabled"
fi
# check dns query log configuration
#
if [ -n "${query_file}" ] && [ "${backup_ok}" = "true" ]
then
# check find capabilities
#
base="$(find --help 2>&1 | grep "mtime")"
if [[ -z "${base}" ]]
then
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "no 'find/mtime' support, dns query logging will be disabled"
else
query_ok="true"
fi
else
query_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "dns query logging will be disabled"
fi
# check shallalist configuration
#
check_shalla="$(printf "${adb_source}" | sed -n '/.*shallalist.txt.*/p')"
if [ -n "${check_shalla}" ]
then
shalla_ok="true"
else
shalla_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "shallalist processing will be disabled"
fi
# check mount points & space requirements
#
for mp in ${mounts}
do
df "${mp}" 2>/dev/null |\
tail -n1 |\
while read filesystem overall used available scrap
do
av_space="${available}"
if [ $((av_space)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "no space left on device, not mounted (${mp})"
exit 35
elif [ $((av_space)) -lt $((min_space)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "not enough space on device (${mp})"
exit 40
fi
done
# subshell return code handling
#
rc=$?
if [ $((rc)) -ne 0 ]
then
f_deltemp
exit ${rc}
fi
done
# get list with all installed packages
#
pkg_list="$(opkg list-installed 2>/dev/null)"
# check openwrt release
#
base="$(printf "${pkg_list}" | grep "^base-files" | sed 's/\(.*r\)//g')"
if [ $((base)) -lt $((min_release)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "openwrt (r${wrt_release}) seems to be too old"
f_deltemp
exit 45
fi
# check curl package dependency
#
base="$(printf "${pkg_list}" | grep "^curl")"
if [ -z "${base}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "curl package not found"
f_deltemp
exit 50
fi
# check wget package dependency
#
base="$(printf "${pkg_list}" | grep "^wget")"
if [ -z "${base}" ]
then
/usr/bin/logger -t "adblock[${pid}]" "wget package not found"
f_deltemp
exit 55
fi
# check dynamic/volatile adblock network interface configuration
#
rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)"
if [ $((rc)) -ne 0 ]
then
json_init
json_add_string name "${adb_if}"
json_add_string ifname "${adb_dev}"
json_add_string proto "static"
json_add_array ipaddr
json_add_string "" "${adb_ip}"
json_close_array
json_close_object
ubus call network add_dynamic "$(json_dump)"
/usr/bin/logger -t "adblock[${pid}]" "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
fi
# check adblock uhttpd instance configuration
#
if [ -z "$(uci -q get uhttpd.${adb_if} 2>/dev/null)" ]
then
uci -q set uhttpd.${adb_if}="uhttpd"
uci -q set uhttpd.${adb_if}.listen_http="${adb_ip}:80"
uci -q set uhttpd.${adb_if}.home="/www/adblock"
uci -q set uhttpd.${adb_if}.error_page="/adblock.html"
uci -q commit uhttpd
/etc/init.d/uhttpd reload
/usr/bin/logger -t "adblock[${pid}]" "created new uhttpd instance (${adb_if}, ${adb_ip}) in /etc/config/uhttpd"
fi
}
###################################################
# f_deltemp: delete temporary files and directories
f_deltemp()
{
if [ -f "${tmp_file}" ]
then
rm -f "${tmp_file}" 2>/dev/null
fi
if [ -d "${tmp_dir}" ]
then
rm -rf "${tmp_dir}" 2>/dev/null
fi
}
################################################################
# f_remove: remove temporary files, start and maintain query log
#
f_remove()
{
# delete temporary files and directories
#
f_deltemp
# kill existing domain query log background process,
# housekeeping and start of a new process on daily basis
#
if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
then
query_date="$(date "+%Y%m%d")"
if [ -s "${query_pid}" ] && [ ! -f "${query_file}.${query_date}" ]
then
kill -9 $(< "${query_pid}") 2>/dev/null
> "${query_pid}"
find "${backup_dir}" -maxdepth 1 -type f -mtime +${query_history} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "kill old query log background process and do logfile housekeeping"
fi
if [ ! -s "${query_pid}" ]
then
( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${query_file}.${query_date}" &
/usr/bin/logger -t "adblock[${pid}]" "start new domain query log background process"
fi
fi
# final log entry
#
/usr/bin/logger -t "adblock[${pid}]" "domain adblock processing finished (${script_ver})"
}
#####################################################
# f_restore: if available, restore last adlist backup
#
f_restore()
{
if [ "${backup_ok}" = "true" ] && [ -f "${backup_file}" ]
then
cp -f "${backup_file}" "${dns_file}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, adlist backup restored"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${log_file}"
else
> "${dns_file}"
/usr/bin/logger -t "adblock[${pid}]" "${restore_msg}, empty adlist generated"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${log_file}"
fi
# restart dnsmasq
#
/etc/init.d/dnsmasq restart >/dev/null 2>&1
# remove files and exit
#
f_remove
exit 100
}
#######################################################
# f_wancheck: check for usable adblock update interface
#
f_wancheck()
{
if [ "${wan_ok}" = "true" ]
then
# wait for wan update interface(s)
#
while [ $((cnt)) -le $((max_cnt)) ]
do
for dev in ${wan_dev}
do
dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null)
if [[ "${dev_out}" = "up" ]]
then
/usr/bin/logger -t "adblock[${pid}]" "get wan/update interface: ${dev}, after ${cnt} loops"
break 2
elif [ $((cnt)) -eq $((max_cnt)) ]
then
/usr/bin/logger -t "adblock[${pid}]" "no wan/update interface(s) found (${wan_dev})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${wan_dev})" >> "${log_file}"
restore_msg="no wan/update interface(s)"
f_restore
fi
done
sleep 1
cnt=$((cnt + 1))
done
fi
}
#####################################
# f_ntpcheck: check/get ntp time sync
#
f_ntpcheck()
{
if [ "${ntp_ok}" = "true" ]
then
# prepare ntp server pool
#
unset ntp_pool
for srv in ${ntp_srv}
do
ntp_pool="${ntp_pool} -p ${srv}"
done
# wait for ntp time sync
#
while [ $((cnt)) -le $((max_cnt)) ]
do
/usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
rc=$?
if [ $((rc)) -eq 0 ]
then
/usr/bin/logger -t "adblock[${pid}]" "get ntp time sync (${ntp_srv}), after ${cnt} loops"
break
elif [ $((cnt)) -eq $((max_cnt)) ]
then
ntp_ok="false"
/usr/bin/logger -t "adblock[${pid}]" "ntp time sync failed (${ntp_srv})"
printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${ntp_srv})" >> "${log_file}"
restore_msg="time sync failed"
f_restore
fi
sleep 1
cnt=$((cnt + 1))
done
fi
}
#################################################################
# f_dnscheck: dnsmasq health check with newly generated blocklist
#
f_dnscheck()
{
# check 1: dnsmasq startup
#
dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
if [ -z "${dns_status}" ]
then
# check 2: nslookup probe
#
dns_status="$(nslookup "${check_domain}" 2>/dev/null | grep "${adb_ip}")"
if [ -z "${dns_status}" ]
then
# create backup of new block list only, if both checks are OK and backup enabled
#
if [ "${backup_ok}" = "true" ]
then
cp -f "${dns_file}" "${backup_file}" 2>/dev/null
/usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded, backup generated"
else
/usr/bin/logger -t "adblock[${pid}]" "new block list with ${adb_count} domains loaded"
fi
else
restore_msg="nslookup probe failed"
f_restore
fi
else
restore_msg="dnsmasq probe failed"
f_restore
fi
}
##########################################################
# f_footer: write footer with a few statistics to dns file
#
f_footer()
{
# count result of merged domain entries
#
adb_count="$(wc -l < "${dns_file}")"
# write file footer with timestamp and merged ad count sum
#
printf "%s\n" "###################################################" >> "${dns_file}"
printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${dns_file}"
printf "%s\n" "# ${0##*/} (${script_ver}) - ${adb_count} ad/abuse domains blocked" >> "${dns_file}"
printf "%s\n" "# domain blacklist sources:" >> "${dns_file}"
for src in ${adb_source}
do
url="$(printf "${src}" | sed 's/\(\&ruleset=.*\)//g')"
printf "%s\n" "# ${url}" >> "${dns_file}"
done
printf "%s\n" "###################################################" >> "${dns_file}"
printf "%s\n" "# domain whitelist source:" >> "${dns_file}"
printf "%s\n" "# ${adb_whitelist}" >> "${dns_file}"
printf "%s\n" "###################################################" >> "${dns_file}"
}

Loading…
Cancel
Save