From 8a0e3c09408e5ca7e7f78985259ef2f95401b623 Mon Sep 17 00:00:00 2001 From: Dirk Brenken Date: Fri, 4 Mar 2016 15:23:01 +0100 Subject: [PATCH] adblock: 0.91.0 * fix blocklist footer/counter * optimize/rework blocklist rulesets * change source url for spam404 and yoyo * code cosmetics & cleanups Signed-off-by: Dirk Brenken --- net/adblock/Makefile | 2 +- net/adblock/files/adblock-helper.sh | 35 ++++++++++++++--------------- net/adblock/files/adblock-update.sh | 23 +++++++++++-------- net/adblock/files/adblock.conf | 6 ++--- 4 files changed, 35 insertions(+), 31 deletions(-) diff --git a/net/adblock/Makefile b/net/adblock/Makefile index ca3a183ad..e58ffaf84 100644 --- a/net/adblock/Makefile +++ b/net/adblock/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock -PKG_VERSION:=0.90.0 +PKG_VERSION:=0.91.0 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0+ PKG_MAINTAINER:=Dirk Brenken diff --git a/net/adblock/files/adblock-helper.sh b/net/adblock/files/adblock-helper.sh index d96ce4cca..8c5480b44 100644 --- a/net/adblock/files/adblock-helper.sh +++ b/net/adblock/files/adblock-helper.sh @@ -169,24 +169,23 @@ f_envparse() # set adblock source ruleset definitions # - rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost.*//g; s/[\^#/:;_\.\t ]*$//g'" - rset_end="tr -cd '[0-9a-z\.\-]\n' | sed -r 's/^[ \.\-].*$|^[a-z0-9]*[ \.\-]*$//g; /^[#/:;_\s]*$/d'" - rset_adaway="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" - rset_blacklist="${rset_start} | ${rset_end}" - rset_disconnect="${rset_start} | ${rset_end}" - rset_dshield="${rset_start} | ${rset_end}" - rset_feodo="${rset_start} | ${rset_end}" - rset_malware="${rset_start} | ${rset_end}" - rset_malwarelist="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" - rset_openphish="sed -e 's|^[^/]*//||' -e 's|/.*$||'" - rset_palevo="${rset_start} | ${rset_end}" - rset_ruadlist="sed -e '/^\|\|/! s/.*//; /\^$/! s/.*//; s/\^$//g; /[\.]/! s/.*//; s/^[\|]\{1,2\}//g' | ${rset_end}" - rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}" - rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}" - rset_whocares="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" - rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}" - rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}" - rset_zeus="${rset_start} | ${rset_end}" + rset_core="([A-Za-z0-9_-]+\.){1,}[A-Za-z]+" + rset_adaway="awk '\$0 ~/^127\.0\.0\.1[ ]+${rset_core}/{print tolower(\$2)}'" + rset_blacklist="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_disconnect="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_dshield="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_feodo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_malware="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_malwarelist="awk '\$0 ~/^127\.0\.0\.1[ ]+${rset_core}/{print tolower(\$2)}'" + rset_openphish="awk -F '/' '\$3 ~/^${rset_core}/{print tolower(\$3)}'" + rset_palevo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_ruadlist="awk -F '[|^]' '\$0 ~/^\|\|${rset_core}\^$/{print tolower(\$3)}'" + rset_shalla="awk -F '/' '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_spam404="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_whocares="awk '\$0 ~/^127\.0\.0\.1[ ]+${rset_core}/{print tolower(\$2)}'" + rset_winhelp="awk '\$0 ~/^0\.0\.0\.0[ ]+${rset_core}/{print tolower(\$2)}'" + rset_yoyo="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" + rset_zeus="awk '\$1 ~/^${rset_core}/{print tolower(\$1)}'" # get logical wan update interfaces (with default route) and their device names # diff --git a/net/adblock/files/adblock-update.sh b/net/adblock/files/adblock-update.sh index 32a48b7d3..9ffd09f79 100755 --- a/net/adblock/files/adblock-update.sh +++ b/net/adblock/files/adblock-update.sh @@ -27,7 +27,7 @@ fi # get current directory, script- and openwrt version # adb_scriptdir="${0%/*}" -adb_scriptver="0.90.0" +adb_scriptver="0.91.0" openwrt_version="$(cat /etc/openwrt_version 2>/dev/null)" # source in adblock function library @@ -189,8 +189,8 @@ do # if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ] then - eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')" - count="$(printf "%s\n" "${tmp_domains}" | tr '[A-Z]' '[a-z]' | eval "${ruleset}" | tee "${adb_tmpfile}" | wc -l)" + eval "src_rset=\${rset_${src_name}}" + count="$(printf "%s\n" "${tmp_domains}" | eval "${src_rset}" | tee "${adb_tmpfile}" | wc -l)" f_log " source download finished (${count} entries)" if [ "${src_name}" = "shalla" ] then @@ -220,10 +220,10 @@ do then if [ -s "${adb_whitelist}" ] then - grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort 2>/dev/null | uniq -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}" + grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}" rc=${?} else - sort "${adb_tmpfile}" 2>/dev/null | uniq -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}" + sort -u "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}" rc=${?} fi @@ -242,6 +242,7 @@ do then printf "%s\n" "#---------------------------------------------" >> "${adb_dnsfile}" printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}" + printf "%s\n" "##" >> "${adb_dnsfile}" f_log " domain merging finished" else f_log " domain merging failed" "${rc}" @@ -332,7 +333,7 @@ then # generate a temporary unique overall list # - head -qn -2 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall" + head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall" # loop through all separate lists, ordered by size (ascending) # @@ -351,7 +352,7 @@ then # write unique result back to original separate list (with list footer) # - tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}" + tail -qn 3 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}" mv -f "${adb_dnsdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}" >/dev/null 2>&1 done rm -f "${adb_dnsdir}/tmp.overall" >/dev/null 2>&1 @@ -363,12 +364,16 @@ fi for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null) do list="${list/*./}" - count="$(head -qn -2 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)" + count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)" if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ] then count=$((count / 2)) fi - printf "%s\n" "# ${0##*/} (${adb_scriptver}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/${adb_dnsprefix}.${list}" + if [ "$(tail -qn 1 "${adb_dnsdir}/${adb_dnsprefix}.${list}")" = "##" ] + then + last_line="# ${0##*/} (${adb_scriptver}) - ${count} ad\/abuse domains blocked" + sed -i "s/^##$/${last_line}/" "${adb_dnsdir}/${adb_dnsprefix}.${list}" + fi adb_count=$((adb_count + count)) done diff --git a/net/adblock/files/adblock.conf b/net/adblock/files/adblock.conf index 09d8cfa44..ce81264c7 100644 --- a/net/adblock/files/adblock.conf +++ b/net/adblock/files/adblock.conf @@ -3,7 +3,7 @@ config adblock 'global' option adb_enabled '1' - option adb_cfgver '0.90' + option adb_cfgver '0.91' option adb_blacklist '/etc/adblock/adblock.blacklist' option adb_whitelist '/etc/adblock/adblock.whitelist' option adb_forcedns '1' @@ -69,7 +69,7 @@ config source 'shalla' config source 'spam404' option enabled '0' - option adb_src 'http://spam404bl.com/spam404scamlist.txt&ruleset=rset_spam404' + option adb_src 'https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt&ruleset=rset_spam404' option adb_srcdesc 'generic blocklist for suspicious domains, infrequent updates, approx. 5.000 entries' config source 'whocares' @@ -84,7 +84,7 @@ config source 'winhelp' config source 'yoyo' option enabled '1' - option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo' + option adb_src 'https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml&showintro=0&mimetype=plaintext&ruleset=rset_yoyo' option adb_srcdesc 'focus on ad related domains, weekly updates, approx. 2.500 entries' config source 'zeus'