Merge pull request #3902 from stangri/simple-adblock

simple-adblock: initial version
8 years ago · 521be0fb84
--- a/net/simple-adblock/Makefile
+++ b/net/simple-adblock/Makefile
@ -0,0 +1,50 @@
 # Copyright (c) 2017 Stan Grishin (stangri@melmac.net)
 # This is free software, licensed under the GNU General Public License v3.
 include $(TOPDIR)/rules.mk
 PKG_NAME:=simple-adblock
 PKG_VERSION:=1.5.6
 PKG_RELEASE:=6
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Stan Grishin <stangri@melmac.net>
 include $(INCLUDE_DIR)/package.mk
 define Package/$(PKG_NAME)
 	SECTION:=net
 	CATEGORY:=Network
 	TITLE:=Simple AdBlock Service
 	PKGARCH:=all
 endef
 define Package/$(PKG_NAME)/description
 This service provides dnsmasq-based ad blocking.
 Please see the README for further information.
 endef
 define Package/$(PKG_NAME)/conffiles
 /etc/config/simple-adblock
 endef
 define Build/Prepare
 	mkdir -p $(PKG_BUILD_DIR)/files/
 	$(CP) ./files/simple-adblock.init $(PKG_BUILD_DIR)/files/simple-adblock.init
 	sed -i "s|^\(PKG_VERSION\).*|\1='$(PKG_VERSION)-$(PKG_RELEASE)'|" $(PKG_BUILD_DIR)/files/simple-adblock.init
 endef
 define Build/Configure
 endef
 define Build/Compile
 endef
 define Package/$(PKG_NAME)/install
 	$(INSTALL_DIR) $(1)/etc/init.d
 	$(INSTALL_BIN) $(PKG_BUILD_DIR)/files/simple-adblock.init $(1)/etc/init.d/simple-adblock
 	$(INSTALL_DIR) $(1)/etc/config
 	$(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock
 endef
 $(eval $(call BuildPackage,$(PKG_NAME)))
--- a/net/simple-adblock/files/README.md
+++ b/net/simple-adblock/files/README.md
@ -0,0 +1,118 @@
 # Simple AdBlock
 A simple DNSMASQ-based AdBlocking service for OpenWrt/LEDE Project. Loosely based on [bole5's](https://forum.openwrt.org/profile.php?id=45571) idea with major performance improvements, added features and Web UI (as a separate package); inspired by @dibdot's innovation.
 ## Features
 - Supports OpenWrt Designated Driver and LEDE Project.
 - Super-fast due to the nature of supported block lists and backgrounding of already downloaded data while next list is downloading.
 - Supports both hosts files and domains lists for blocking (to keep it lean and fast).
 - Everything is configurable from Web UI.
 - Allows you to easily add your own domains to whitelist or blacklist.
 - Allows you to easily add URLs to your own blocked hosts or domains lists to block/whitelist (just put whitelisted domains one per line).
 - Requires no configuration for the download utility wherever you want to use wget/libopenssl or uclient-fetch/libustream-mbedtls.
 - Installs dependencies automatically (DD/LEDE-default uclient-fetch libustream-mbedtls).
 - Doesn't stay in memory -- creates the list of blocked domains and then uses DNSMASQ and firewall rules to serve "domain not found reply".
 - As some of the default lists are using https, reliably works with either wget/libopenssl or uclient-fetch/libustream-mbedtls.
 - Very lightweight and easily hackable, the whole script is just one /etc/init.d/simple-adblock file.
 - Logs single entry in the system log with the number of blocked domains if verbosity is set to 0.
 - Retains the downloaded/sorted adblocking list on service stop and reuses it on service start (use reload if you want to force re-download of the list).
 - Blocks ads served over https.
 - Proudly made in Canada, using locally-sourced electrons.
 If you want a more robust AdBlocking, supporting free memory detection and complex block lists, check out [@dibdot's adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files).
 ## Screenshot (luci-app-simple-adblock)
 ![screenshot](https://raw.githubusercontent.com/stangri/screenshots/master/simple-adblock/screenshot04.png "screenshot")
 ## Requirements
 This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` and either ```wget``` and ```libopenssl``` (for OpenWrt CC 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for OpenWrt DD trunk and all LEDE Project release and snapshot builds). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing.
 To satisfy the requirements for connect to your router via ssh and run the following commands:
 ###### OpenWrt CC 15.05.1
 ```sh
 opkg update; opkg install wget libopenssl coreutils-sort dnsmasq
 ```
 ###### LEDE Project and OpenWrt DD trunk
 ```sh
 opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq
 ```
 ###### IPv6 Support
 For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-nat6``` packages from Web UI or run the following in the command line:
 ```sh
 opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6
 ```
 ###### Speed up blocklist processing with coreutils-sort
 The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```.
 #### Unmet dependencies
 If you are running a development (trunk/snapshot) build of OpenWrt/LEDE Project on your router and your build is outdated (meaning that packages of the same revision/commit hash are no longer available and when you try to satisfy the [requirements](#requirements) you get errors), please flash either current LEDE release image or current development/snapshot image.
 ## How to install
 Install ```simple-adblock``` and  ```luci-app-simple-adblock``` packages from Web UI or run the following in the command line:
 ```sh
 opkg update; opkg install simple-adblock luci-app-simple-adblock
 ```
 If ```simple-adblock``` and  ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](#add-custom-repo-to-your-router) first.
 #### Add custom repo to your router
 If your router is not set up with the access to repository containing these packages you will need to add custom repository to your router by connecting to your router via ssh and running the following commands:
 ```sh
 echo -e -n 'untrusted comment: public key 7ffc7517c4cc0c56\nRWR//HUXxMwMVnx7fESOKO7x8XoW4/dRidJPjt91hAAU2L59mYvHy0Fa\n' > /tmp/stangri-repo.pub && opkg-key add /tmp/stangri-repo.pub
 ! grep -q 'stangri_repo' /etc/opkg/customfeeds.conf && echo 'src/gz stangri_repo https://raw.githubusercontent.com/stangri/openwrt-repo/master' >> /etc/opkg/customfeeds.conf
 opkg update
 ```
 #### Default Settings
 Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1```) and selected ad/malware lists suitable for routers with 64Mb RAM. The configuration file has lists in descending order starting with biggest ones, comment out or delete the lists you don't want or your router can't handle.
 ## How to customize
 You can use Web UI (found in Services/Simple AdBlock) to add/remove/edit links to:
 - hosts files (127.0.0.1 or 0.0.0.0 followed by space and domain name per line) to be blocked.
 - domains lists (one domain name per line) to be blocked.
 - domains lists (one domain name per line) to be whitelisted. It is useful if you want to run simple-adblock on multiple routers and maintain one centralized whitelist which you can publish on a web-server.
 Please note that these lists **have** to include either ```http://``` or ```https://``` prefix. Some of the top block lists (both hosts files and domains lists) suitable for routers with at least 8MB RAM are used in the default simple-adblock installation.
 You can also use Web UI to add individual domains to be blocked or whitelisted.
 If you want to use CLI to customize simple-adblock config, you can probably figure out how to do it by looking at the contents of ```/etc/config/simple-adblock``` or output of the ```uci show simple-adblock``` command.
 ## How does it work
 This service downloads (and processes in the background, removing comments and other useless data) lists of hosts and domains to be blocked, combines those lists into one big block list, removes duplicates and sorts it and then removes your whitelisted domains from the block list before converting to to dnsmasq-compatible file and restarting dnsmasq. The result of the process is that dnsmasq returns "domain not found" for the blocked domains.
 If you specify ```google.com``` as a domain to be whitelisted, you will have access to ```google.com```, ```www.google.com```, ```analytics.google.com```, but not fake domains like ```email-google.com``` or ```drive.google.com.verify.signin.normandeassociation.com``` for example. If you only want to allow ```www.google.com``` while blocking all other ```google.com``` subdomains, just specify ```www.google.com``` as domain to be whitelisted.
 In general, whatever domain is specified to be whitelisted; it, along with with its subdomains will be whitelisted, but not any fake domains containing it.
 ## Documentation / Discussion
 Please head to [OpenWrt Forum](https://forum.openwrt.org/viewtopic.php?pid=307950) or [LEDE Project Forum](https://forum.lede-project.org/t/simple-adblock-fast-lightweight-and-fully-uci-luci-configurable-ad-blocking/) for discussion of this package.
 ## What's New
 1.5.6:
 - Better handling of service start/enable from Web UI and enabled flag management.
 1.5.5:
 - Implemented support to set one of the router LEDs on/off based on the AdBlocking status.
 - Fixed the output bug when verbosity=1.
 1.5.3:
 - No longer using enabled in config file, Simple AdBlocking Web UI now enables/disables service directly.
 1.5.1:
 - Reworked console/system log output logic and formatting.
 1.5.0:
 - Processes already downloaded lists in the background while downloading next list from config, dramatically increasing overall speed.
 1.0.0:
 - Initial release
--- a/net/simple-adblock/files/simple-adblock.conf
+++ b/net/simple-adblock/files/simple-adblock.conf
@ -0,0 +1,26 @@
 config simple-adblock 'config'
 	option enabled '0'
 	option verbosity '2'
 	option force_dns '1'
 	option run_in_background '1'
 	option hosts_file '/var/dnsmasq.d/simple-adblock'
  list whitelist_domain 'raw.githubusercontent.com'
 #	list blacklist_hosts_url 'http://support.it-mate.co.uk/downloads/hosts.txt'
 #	list blacklist_hosts_url 'http://hostsfile.mine.nu/Hosts'
 #	list blacklist_hosts_url 'http://hosts-file.net/.\ad_servers.txt'
 #	list blacklist_hosts_url 'http://sysctl.org/cameleon/hosts.win'
 	list blacklist_hosts_url 'http://www.mvps.org/winhelp2002/hosts.txt'
 	list blacklist_hosts_url 'http://pgl.yoyo.org/as/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext'
 	list blacklist_hosts_url 'http://www.malwaredomainlist.com/hostslist/hosts.txt'
 	list blacklist_hosts_url 'https://adaway.org/hosts.txt'
 	list blacklist_hosts_url 'http://someonewhocares.org/hosts/hosts'
 	list blacklist_hosts_url 'https://zeustracker.abuse.ch/blocklist.php?download=hostfile'
 	list blacklist_domains_url 'http://mirror1.malwaredomains.com/files/justdomains'
 	list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt'
 	list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt'
 	list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt'
 	list blacklist_domains_url 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt'
 	list blacklist_domains_url 'https://gitlab.com/gwillem/public-snippets/snippets/28813/raw'
 	list blacklist_domains_url 'http://dshield.org/feeds/suspiciousdomains_High.txt'
 #	list blacklist_domains_url 'http://dshield.org/feeds/suspiciousdomains_Medium.txt'
 #	list blacklist_domains_url 'http://dshield.org/feeds/suspiciousdomains_Low.txt'
--- a/net/simple-adblock/files/simple-adblock.init
+++ b/net/simple-adblock/files/simple-adblock.init
@ -0,0 +1,201 @@
 #!/bin/sh /etc/rc.common
 PKG_VERSION=
 export START=94
 export USE_PROCD=1
 #PROCD_DEBUG=1
 readonly A_TMP='/var/hosts.allowed.tmp'
 readonly B_TMP='/var/hosts.blocked.tmp'
 readonly T_TMP='/var/simple-adblock.hosts'
 readonly dl='wget --no-check-certificate -qO-'
 readonly h_filter='/localhost/d;/^#/d;/^$/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;'
 readonly d_filter='/localhost/d;/^#/d;/^$/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/^</d;'
 readonly f_filter='s|^|local=/|;s|$|/|'
 readonly _ok_='\033[0;32m\xe2\x9c\x93\033[0m'
 readonly _fail_='\033[0;31m\xe2\x9c\x97\033[0m'
 readonly __ok__='\033[0;32m[\xe2\x9c\x93]\033[0m'
 readonly __fail__='\033[0;31m[\xe2\x9c\x97]\033[0m'
 readonly _error_='\033[0;31mERROR\033[0m'
 export verbosity=2 force_dns=1 bgrun=0 hosts_file='/var/dnsmasq.d/simple-adblock' led wan_if wan_gw wanphysdev
 ok() { case $verbosity in 1) output "$_ok_";; 2) output "$__ok__\n";; esac; }
 okn() { case $verbosity in 1) output "$_ok_\n";; 2) output "$__ok__\n";; esac; }
 fail() { case $verbosity in 1) output "$_fail_";; 2) output "$__fail__\n";; esac; }
 failn() { case $verbosity in 1) output "$_fail_\n";; 2) output "$__fail__\n";; esac; }
 output() { [[ $# -ne 1 ]] && { [[ ! $((verbosity & $1)) -gt 0 ]] && return 0 || shift; }; local msg; msg=$(echo -n "${1/$p_name /service }" | sed 's|\\033\[[0-9]\?;\?[0-9]\?[0-9]\?m||g'); [[ -t 1 ]] && echo -e -n "$1"; [[ $(echo -e -n "$msg" | wc -l) -gt 0 ]] && logger -t "${PKG_NAME:-service} [$$]" "$(echo -e -n ${logmsg}${msg})" && logmsg='' || logmsg=${logmsg}${msg}; }
 PKG_NAME="${PKG_NAME:-simple-adblock}"; p_name="${PKG_NAME} ${PKG_VERSION}"
 led_on(){ [[ -n "$led" && -e "$led/trigger" ]] && echo "default-on" > "$led/trigger"; }
 led_off(){ [[ -n "$led" && -e "$led/trigger" ]] && echo "none" > "$led/trigger"; }
 is_enabled () {
 		local c=1 enabled
 		config_load $PKG_NAME
 		config_get_bool enabled   'config' 'enabled' 1
 		config_get_bool bgrun     'config' 'run_in_background' 0
 		config_get_bool force_dns 'config' 'force_dns' 1
 		config_get verbosity      'config' 'verbosity' '2'
 		config_get hosts_file  	  'config' 'hosts_file' '/var/dnsmasq.d/simple-adblock'
 		config_get led        	  'config' 'led'
 		led="${led:+/sys/class/leds/$led}"
 		[[ $enabled -gt 0 ]] || { output "$_error_: $p_name is not enabled.\n"; return 1; }
 		source 	/lib/functions/network.sh
 		while : ; do
 			network_find_wan wan_if;	[ -n "$wan_if" ] && network_get_gateway wan_gw $wan_if;
 			[[ $c -ge 25 || -n "$wan_gw" ]] && break
 			output "$p_name waiting for wan gateway...\n"; sleep 2; network_flush_cache; let "c+=1";
 		done
 		[ -n "$wan_gw" ] && return 0 || { output "$_error_: $p_name failed to discover WAN gateway.\n"; return 1; }
 }
 reset_iptables() {
  [[ $force_dns -eq 0 ]] && return 0
 	[ -z "$PKG_NAME" ] && return 1
 	iptables-save | grep -Fv -- "$PKG_NAME" | iptables-restore
 	lsmod | grep -q ip6table_nat && ip6tables-save | grep -Fv -- "$PKG_NAME" | ip6tables-restore
 	[ ! "$1" == "quiet" ] && output 'No longer forcing local DNS server.\n'
 }
 set_iptables() {
 	local ip ipv6 label ipv6wan brname
 	network_get_ipaddr ip lan; network_get_ipaddr6 ipv6 lan; network_get_device brname lan; network_get_physdev wanphysdev wan;
 	ipv6wan=$(ifconfig $wanphysdev | grep inet6 | awk '{print $3}')
 	if [[ $force_dns -ne 0 ]]; then
 		[ -n "$ip" ] && iptables -t nat -A prerouting_rule -i $brname -p tcp --dport 53 -j DNAT --to $ip -m comment --comment "$PKG_NAME"
 		[ -n "$ip" ] && iptables -t nat -A prerouting_rule -i $brname -p udp --dport 53 -j DNAT --to $ip -m comment --comment "$PKG_NAME"
 		if [[ -n "$ipv6" && -n "$ipv6wan" ]] && lsmod | grep -q ip6table_nat; then
 			ip6tables -t nat -A PREROUTING -i $brname -p tcp --dport 53 -j DNAT --to-destination [$ipv6] -m comment --comment "$PKG_NAME"
 			ip6tables -t nat -A PREROUTING -i $brname -p udp --dport 53 -j DNAT --to-destination [$ipv6] -m comment --comment "$PKG_NAME"
 			label="$ip/$ipv6"
 		else
 			label="$ip"
 		fi
 		[ -n "$label" ] && output "Forcing local DNS server: $label.\n" || output "$_error_: $p_name failed to obtain LAN IP address for DNS forcing!\n"
 	fi
 }
 stop_adblocking () {
    [ -f $hosts_file ] && mv $hosts_file $T_TMP
    output 3 "Restarting dnsmasq "
 		led_off
    /etc/init.d/dnsmasq restart >/dev/null  2>&1
 		[[ $? -eq 0 ]] && { okn; output "$p_name stopped.\n"; } || { failn; output "$_error_: $p_name failed to reload dnsmasq!\n"; }
 }
 process_url() {
 	local label type D_TMP R_TMP
 	[[ -n "$1" && -n "$2" && -n "$3" ]] || return 1
 	local url=$1
 	[ "$2" == "hosts" ] && label="Hosts: $(echo $1 | cut -d'/' -f3)" filter="$h_filter" || label="Domains: $(echo $1 | cut -d'/' -f3)" filter="$d_filter"
 	[ "$3" == "blocked" ] && { type='Blocked'; D_TMP="$B_TMP"; } || { type='Allowed'; D_TMP="$A_TMP"; }
 	R_TMP="/var/simple-adblock_$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c10)"
 	while [ -e "$R_TMP" ]; do R_TMP="/var/simple-adblock_$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c10)"; done
 	touch "$R_TMP"
    output 2 "[DL] $type $label "
    $dl "${url}" > "$R_TMP" && ok || fail
 	{ sed -i "$filter" "$R_TMP"; cat "$R_TMP" >> "$D_TMP"; rm -f "$R_TMP"; } &
 }
 start_adblocking () {
 		local whitelist_domains blacklist_domains whitelist_domains_urls blacklist_domains_urls blacklist_hosts_urls
    config_get whitelist_domains          'config' 'whitelist_domain'
    config_get blacklist_domains          'config' 'blacklist_domain'
    config_get whitelist_domains_urls     'config' 'whitelist_domains_url'
    config_get blacklist_domains_urls     'config' 'blacklist_domains_url'
    config_get blacklist_hosts_urls       'config' 'blacklist_hosts_url'
    local hf w_filter
 	[ ! -d ${hosts_file%/*} ] && mkdir -p ${hosts_file%/*}
 	if [[ -s $T_TMP && ! "$1" == "reload" ]]; then
 		output 3 'Found existing data file, reusing it '
 		mv $T_TMP $hosts_file && okn || failn
 	else
 		[ -f $A_TMP ] && rm -f $A_TMP; [ -f $B_TMP ] && rm -f $B_TMP; [ -f $T_TMP ] && rm -f $T_TMP; [ -f $hosts_file ] && rm -f $hosts_file
 	    touch $A_TMP; touch $B_TMP; touch $T_TMP;
 	    if [ -n "$blacklist_hosts_urls" ]; then
 		    output 1 '[DL] Blocked Hosts '
 		    for hf in ${blacklist_hosts_urls}; do process_url "$hf" 'hosts' 'blocked'; done
 				output 1 '\n'
 		fi
 	    if [ -n "$blacklist_domains_urls" ]; then
 		    output 1 '[DL] Blocked Domains '
 		    for hf in ${blacklist_domains_urls}; do process_url "$hf" 'domains' 'blocked'; done
 				output 1 '\n'
 		fi
 	    if [ -n "$whitelist_domains_urls" ]; then
 		    output 1 '[DL] Allowed Domains '
 		    for hf in ${whitelist_domains_urls}; do process_url "$hf" 'domains' 'allowed'; done
 				output 1 '\n'
 		fi
 		output 3 'Waiting for background processes '
 		wait && okn
 	    [ -n "$blacklist_domains" ] && for hf in ${blacklist_domains}; do echo "$hf" | sed "$d_filter" >> $B_TMP; done
 	    whitelist_domains="${whitelist_domains}"$'\n'"$(cat $A_TMP)"
 	    [ -n "$whitelist_domains" ] && for hf in ${whitelist_domains}; do hf=$(echo $hf | sed 's/\./\\./g'); w_filter="$w_filter/^${hf}$/d;/\\.${hf}$/d;"; done
 		if [ -s $B_TMP ]; then
 		    output 1 'Processing downloads '
 		    output 2 'Sorting merged file '; sort $B_TMP | uniq > $T_TMP && ok || fail
 		    output 2 'Whitelisting domains '; sed -i "$w_filter" $T_TMP && ok || fail
 		    output 2 'Formatting merged file '; sed "$f_filter" $T_TMP > $hosts_file && ok || fail
 		    output 1 '\n'
 		    output 3 'Removing temporary files '
 			[ -f $A_TMP ] && rm -f $A_TMP; [ -f $B_TMP ] && rm -f $B_TMP; [ -f $T_TMP ] && rm -f $T_TMP;
 		    okn
 		fi
 	fi
 	if [ -s $hosts_file ]; then
 		output 3 'Restarting dnsmasq '
 		/etc/init.d/dnsmasq restart >/dev/null  2>&1
 		if [[ $? -eq 0 ]]; then
 			led_on; okn;
 			output "$p_name blocking $(wc -l < $hosts_file) domains $_ok_\n"
 		else
 			failn; output "$_error_: $p_name failed to reload dnsmasq!\n";
 			exit 1
 		fi
 	else
 		output "$_error_: $p_name failed to create its data file!\n"
 		exit 1
 	fi
 }
 boot() { sleep 10; rc_procd start_service; rc_procd service_triggers; }
 reload_service () {
  is_enabled || return 1
 	[[ -t 1 && $bgrun -eq 1 ]] && (start_adblocking 'reload' | cat &) || start_adblocking 'reload'
 }
 start_service () {
  is_enabled || return 1
 	procd_open_instance
 	procd_set_param stdout 1
 	procd_set_param stderr 1
 	procd_close_instance
 	reset_iptables quiet
 	set_iptables
 	[[ -t 1 && $bgrun -eq 1 ]] && (start_adblocking $1 | cat &) || start_adblocking $1
 }
 stop_service () {
  is_enabled || return 1
 	reset_iptables
 	[[ -t 1 && $bgrun -eq 1 ]] && (stop_adblocking | cat &) || stop_adblocking
 }
 service_triggers () {
    procd_add_reload_trigger 'simple-adblock'
 }