LILiK
/
openwrt-packages-dist
5
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

simple-adblock: initial commit 

simple-adblock: version 1.5 introduces background processing of downloaded list while the next list is being downloaded

Signed-off-by: Stan Grishin <stangri@melmac.net>
lilik-openwrt-22.03
Stan Grishin 8 years ago
parent
285298dc5f
commit
7ac3fcd9fd
4 changed files with 395 additions and 0 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +50
    -0
      net/simple-adblock/Makefile
  2. +118
    -0
      net/simple-adblock/files/README.md
  3. +26
    -0
      net/simple-adblock/files/simple-adblock.conf
  4. +201
    -0
      net/simple-adblock/files/simple-adblock.init

+ 50
- 0
net/simple-adblock/Makefile View File

@ -0,0 +1,50 @@
# Copyright (c) 2017 Stan Grishin (stangri@melmac.net)
# This is free software, licensed under the GNU General Public License v3.
include $(TOPDIR)/rules.mk
PKG_NAME:=simple-adblock
PKG_VERSION:=1.5.6
PKG_RELEASE:=6
PKG_LICENSE:=GPL-3.0+
PKG_MAINTAINER:=Stan Grishin <stangri@melmac.net>
include $(INCLUDE_DIR)/package.mk
define Package/$(PKG_NAME)
SECTION:=net
CATEGORY:=Network
TITLE:=Simple AdBlock Service
PKGARCH:=all
endef
define Package/$(PKG_NAME)/description
This service provides dnsmasq-based ad blocking.
Please see the README for further information.
endef
define Package/$(PKG_NAME)/conffiles
/etc/config/simple-adblock
endef
define Build/Prepare
mkdir -p $(PKG_BUILD_DIR)/files/
$(CP) ./files/simple-adblock.init $(PKG_BUILD_DIR)/files/simple-adblock.init
sed -i "s|^\(PKG_VERSION\).*|\1='$(PKG_VERSION)-$(PKG_RELEASE)'|" $(PKG_BUILD_DIR)/files/simple-adblock.init
endef
define Build/Configure
endef
define Build/Compile
endef
define Package/$(PKG_NAME)/install
$(INSTALL_DIR) $(1)/etc/init.d
$(INSTALL_BIN) $(PKG_BUILD_DIR)/files/simple-adblock.init $(1)/etc/init.d/simple-adblock
$(INSTALL_DIR) $(1)/etc/config
$(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock
endef
$(eval $(call BuildPackage,$(PKG_NAME)))

+ 118
- 0
net/simple-adblock/files/README.md View File

@ -0,0 +1,118 @@
# Simple AdBlock
A simple DNSMASQ-based AdBlocking service for OpenWrt/LEDE Project. Loosely based on [bole5's](https://forum.openwrt.org/profile.php?id=45571) idea with major performance improvements, added features and Web UI (as a separate package); inspired by @dibdot's innovation.
## Features
- Supports OpenWrt Designated Driver and LEDE Project.
- Super-fast due to the nature of supported block lists and backgrounding of already downloaded data while next list is downloading.
- Supports both hosts files and domains lists for blocking (to keep it lean and fast).
- Everything is configurable from Web UI.
- Allows you to easily add your own domains to whitelist or blacklist.
- Allows you to easily add URLs to your own blocked hosts or domains lists to block/whitelist (just put whitelisted domains one per line).
- Requires no configuration for the download utility wherever you want to use wget/libopenssl or uclient-fetch/libustream-mbedtls.
- Installs dependencies automatically (DD/LEDE-default uclient-fetch libustream-mbedtls).
- Doesn't stay in memory -- creates the list of blocked domains and then uses DNSMASQ and firewall rules to serve "domain not found reply".
- As some of the default lists are using https, reliably works with either wget/libopenssl or uclient-fetch/libustream-mbedtls.
- Very lightweight and easily hackable, the whole script is just one /etc/init.d/simple-adblock file.
- Logs single entry in the system log with the number of blocked domains if verbosity is set to 0.
- Retains the downloaded/sorted adblocking list on service stop and reuses it on service start (use reload if you want to force re-download of the list).
- Blocks ads served over https.
- Proudly made in Canada, using locally-sourced electrons.
If you want a more robust AdBlocking, supporting free memory detection and complex block lists, check out [@dibdot's adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files).
## Screenshot (luci-app-simple-adblock)
![screenshot](https://raw.githubusercontent.com/stangri/screenshots/master/simple-adblock/screenshot04.png "screenshot")
## Requirements
This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` and either ```wget``` and ```libopenssl``` (for OpenWrt CC 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for OpenWrt DD trunk and all LEDE Project release and snapshot builds). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing.
To satisfy the requirements for connect to your router via ssh and run the following commands:
###### OpenWrt CC 15.05.1
```sh
opkg update; opkg install wget libopenssl coreutils-sort dnsmasq
```
###### LEDE Project and OpenWrt DD trunk
```sh
opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq
```
###### IPv6 Support
For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-nat6``` packages from Web UI or run the following in the command line:
```sh
opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6
```
###### Speed up blocklist processing with coreutils-sort
The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```.
#### Unmet dependencies
If you are running a development (trunk/snapshot) build of OpenWrt/LEDE Project on your router and your build is outdated (meaning that packages of the same revision/commit hash are no longer available and when you try to satisfy the [requirements](#requirements) you get errors), please flash either current LEDE release image or current development/snapshot image.
## How to install
Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web UI or run the following in the command line:
```sh
opkg update; opkg install simple-adblock luci-app-simple-adblock
```
If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](#add-custom-repo-to-your-router) first.
#### Add custom repo to your router
If your router is not set up with the access to repository containing these packages you will need to add custom repository to your router by connecting to your router via ssh and running the following commands:
```sh
echo -e -n 'untrusted comment: public key 7ffc7517c4cc0c56\nRWR//HUXxMwMVnx7fESOKO7x8XoW4/dRidJPjt91hAAU2L59mYvHy0Fa\n' > /tmp/stangri-repo.pub && opkg-key add /tmp/stangri-repo.pub
! grep -q 'stangri_repo' /etc/opkg/customfeeds.conf && echo 'src/gz stangri_repo https://raw.githubusercontent.com/stangri/openwrt-repo/master' >> /etc/opkg/customfeeds.conf
opkg update
```
#### Default Settings
Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1```) and selected ad/malware lists suitable for routers with 64Mb RAM. The configuration file has lists in descending order starting with biggest ones, comment out or delete the lists you don't want or your router can't handle.
## How to customize
You can use Web UI (found in Services/Simple AdBlock) to add/remove/edit links to:
- hosts files (127.0.0.1 or 0.0.0.0 followed by space and domain name per line) to be blocked.
- domains lists (one domain name per line) to be blocked.
- domains lists (one domain name per line) to be whitelisted. It is useful if you want to run simple-adblock on multiple routers and maintain one centralized whitelist which you can publish on a web-server.
Please note that these lists **have** to include either ```http://``` or ```https://``` prefix. Some of the top block lists (both hosts files and domains lists) suitable for routers with at least 8MB RAM are used in the default simple-adblock installation.
You can also use Web UI to add individual domains to be blocked or whitelisted.
If you want to use CLI to customize simple-adblock config, you can probably figure out how to do it by looking at the contents of ```/etc/config/simple-adblock``` or output of the ```uci show simple-adblock``` command.
## How does it work
This service downloads (and processes in the background, removing comments and other useless data) lists of hosts and domains to be blocked, combines those lists into one big block list, removes duplicates and sorts it and then removes your whitelisted domains from the block list before converting to to dnsmasq-compatible file and restarting dnsmasq. The result of the process is that dnsmasq returns "domain not found" for the blocked domains.
If you specify ```google.com``` as a domain to be whitelisted, you will have access to ```google.com```, ```www.google.com```, ```analytics.google.com```, but not fake domains like ```email-google.com``` or ```drive.google.com.verify.signin.normandeassociation.com``` for example. If you only want to allow ```www.google.com``` while blocking all other ```google.com``` subdomains, just specify ```www.google.com``` as domain to be whitelisted.
In general, whatever domain is specified to be whitelisted; it, along with with its subdomains will be whitelisted, but not any fake domains containing it.
## Documentation / Discussion
Please head to [OpenWrt Forum](https://forum.openwrt.org/viewtopic.php?pid=307950) or [LEDE Project Forum](https://forum.lede-project.org/t/simple-adblock-fast-lightweight-and-fully-uci-luci-configurable-ad-blocking/) for discussion of this package.
## What's New
1.5.6:
- Better handling of service start/enable from Web UI and enabled flag management.
1.5.5:
- Implemented support to set one of the router LEDs on/off based on the AdBlocking status.
- Fixed the output bug when verbosity=1.
1.5.3:
- No longer using enabled in config file, Simple AdBlocking Web UI now enables/disables service directly.
1.5.1:
- Reworked console/system log output logic and formatting.
1.5.0:
- Processes already downloaded lists in the background while downloading next list from config, dramatically increasing overall speed.
1.0.0:
- Initial release

+ 26
- 0
net/simple-adblock/files/simple-adblock.conf View File

@ -0,0 +1,26 @@
config simple-adblock 'config'
option enabled '0'
option verbosity '2'
option force_dns '1'
option run_in_background '1'
option hosts_file '/var/dnsmasq.d/simple-adblock'
list whitelist_domain 'raw.githubusercontent.com'
# list blacklist_hosts_url 'http://support.it-mate.co.uk/downloads/hosts.txt'
# list blacklist_hosts_url 'http://hostsfile.mine.nu/Hosts'
# list blacklist_hosts_url 'http://hosts-file.net/.\ad_servers.txt'
# list blacklist_hosts_url 'http://sysctl.org/cameleon/hosts.win'
list blacklist_hosts_url 'http://www.mvps.org/winhelp2002/hosts.txt'
list blacklist_hosts_url 'http://pgl.yoyo.org/as/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext'
list blacklist_hosts_url 'http://www.malwaredomainlist.com/hostslist/hosts.txt'
list blacklist_hosts_url 'https://adaway.org/hosts.txt'
list blacklist_hosts_url 'http://someonewhocares.org/hosts/hosts'
list blacklist_hosts_url 'https://zeustracker.abuse.ch/blocklist.php?download=hostfile'
list blacklist_domains_url 'http://mirror1.malwaredomains.com/files/justdomains'
list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt'
list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt'
list blacklist_domains_url 'https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt'
list blacklist_domains_url 'https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt'
list blacklist_domains_url 'https://gitlab.com/gwillem/public-snippets/snippets/28813/raw'
list blacklist_domains_url 'http://dshield.org/feeds/suspiciousdomains_High.txt'
# list blacklist_domains_url 'http://dshield.org/feeds/suspiciousdomains_Medium.txt'
# list blacklist_domains_url 'http://dshield.org/feeds/suspiciousdomains_Low.txt'

+ 201
- 0
net/simple-adblock/files/simple-adblock.init View File

@ -0,0 +1,201 @@
#!/bin/sh /etc/rc.common
PKG_VERSION=
export START=94
export USE_PROCD=1
#PROCD_DEBUG=1
readonly A_TMP='/var/hosts.allowed.tmp'
readonly B_TMP='/var/hosts.blocked.tmp'
readonly T_TMP='/var/simple-adblock.hosts'
readonly dl='wget --no-check-certificate -qO-'
readonly h_filter='/localhost/d;/^#/d;/^$/d;/^[^0-9]/d;s/^0\.0\.0\.0.//;s/^127\.0\.0\.1.//;s/[[:space:]]*#.*$//;s/[[:cntrl:]]$//;s/[[:space:]]//g;'
readonly d_filter='/localhost/d;/^#/d;/^$/d;s/[[:space:]]*#.*$//;s/[[:space:]]*$//;s/[[:cntrl:]]$//;/[[:space:]]/d;/^</d;'
readonly f_filter='s|^|local=/|;s|$|/|'
readonly _ok_='\033[0;32m\xe2\x9c\x93\033[0m'
readonly _fail_='\033[0;31m\xe2\x9c\x97\033[0m'
readonly __ok__='\033[0;32m[\xe2\x9c\x93]\033[0m'
readonly __fail__='\033[0;31m[\xe2\x9c\x97]\033[0m'
readonly _error_='\033[0;31mERROR\033[0m'
export verbosity=2 force_dns=1 bgrun=0 hosts_file='/var/dnsmasq.d/simple-adblock' led wan_if wan_gw wanphysdev
ok() { case $verbosity in 1) output "$_ok_";; 2) output "$__ok__\n";; esac; }
okn() { case $verbosity in 1) output "$_ok_\n";; 2) output "$__ok__\n";; esac; }
fail() { case $verbosity in 1) output "$_fail_";; 2) output "$__fail__\n";; esac; }
failn() { case $verbosity in 1) output "$_fail_\n";; 2) output "$__fail__\n";; esac; }
output() { [[ $# -ne 1 ]] && { [[ ! $((verbosity & $1)) -gt 0 ]] && return 0 || shift; }; local msg; msg=$(echo -n "${1/$p_name /service }" | sed 's|\\033\[[0-9]\?;\?[0-9]\?[0-9]\?m||g'); [[ -t 1 ]] && echo -e -n "$1"; [[ $(echo -e -n "$msg" | wc -l) -gt 0 ]] && logger -t "${PKG_NAME:-service} [$$]" "$(echo -e -n ${logmsg}${msg})" && logmsg='' || logmsg=${logmsg}${msg}; }
PKG_NAME="${PKG_NAME:-simple-adblock}"; p_name="${PKG_NAME} ${PKG_VERSION}"
led_on(){ [[ -n "$led" && -e "$led/trigger" ]] && echo "default-on" > "$led/trigger"; }
led_off(){ [[ -n "$led" && -e "$led/trigger" ]] && echo "none" > "$led/trigger"; }
is_enabled () {
local c=1 enabled
config_load $PKG_NAME
config_get_bool enabled 'config' 'enabled' 1
config_get_bool bgrun 'config' 'run_in_background' 0
config_get_bool force_dns 'config' 'force_dns' 1
config_get verbosity 'config' 'verbosity' '2'
config_get hosts_file 'config' 'hosts_file' '/var/dnsmasq.d/simple-adblock'
config_get led 'config' 'led'
led="${led:+/sys/class/leds/$led}"
[[ $enabled -gt 0 ]] || { output "$_error_: $p_name is not enabled.\n"; return 1; }
source /lib/functions/network.sh
while : ; do
network_find_wan wan_if; [ -n "$wan_if" ] && network_get_gateway wan_gw $wan_if;
[[ $c -ge 25 || -n "$wan_gw" ]] && break
output "$p_name waiting for wan gateway...\n"; sleep 2; network_flush_cache; let "c+=1";
done
[ -n "$wan_gw" ] && return 0 || { output "$_error_: $p_name failed to discover WAN gateway.\n"; return 1; }
}
reset_iptables() {
[[ $force_dns -eq 0 ]] && return 0
[ -z "$PKG_NAME" ] && return 1
iptables-save | grep -Fv -- "$PKG_NAME" | iptables-restore
lsmod | grep -q ip6table_nat && ip6tables-save | grep -Fv -- "$PKG_NAME" | ip6tables-restore
[ ! "$1" == "quiet" ] && output 'No longer forcing local DNS server.\n'
}
set_iptables() {
local ip ipv6 label ipv6wan brname
network_get_ipaddr ip lan; network_get_ipaddr6 ipv6 lan; network_get_device brname lan; network_get_physdev wanphysdev wan;
ipv6wan=$(ifconfig $wanphysdev | grep inet6 | awk '{print $3}')
if [[ $force_dns -ne 0 ]]; then
[ -n "$ip" ] && iptables -t nat -A prerouting_rule -i $brname -p tcp --dport 53 -j DNAT --to $ip -m comment --comment "$PKG_NAME"
[ -n "$ip" ] && iptables -t nat -A prerouting_rule -i $brname -p udp --dport 53 -j DNAT --to $ip -m comment --comment "$PKG_NAME"
if [[ -n "$ipv6" && -n "$ipv6wan" ]] && lsmod | grep -q ip6table_nat; then
ip6tables -t nat -A PREROUTING -i $brname -p tcp --dport 53 -j DNAT --to-destination [$ipv6] -m comment --comment "$PKG_NAME"
ip6tables -t nat -A PREROUTING -i $brname -p udp --dport 53 -j DNAT --to-destination [$ipv6] -m comment --comment "$PKG_NAME"
label="$ip/$ipv6"
else
label="$ip"
fi
[ -n "$label" ] && output "Forcing local DNS server: $label.\n" || output "$_error_: $p_name failed to obtain LAN IP address for DNS forcing!\n"
fi
}
stop_adblocking () {
[ -f $hosts_file ] && mv $hosts_file $T_TMP
output 3 "Restarting dnsmasq "
led_off
/etc/init.d/dnsmasq restart >/dev/null 2>&1
[[ $? -eq 0 ]] && { okn; output "$p_name stopped.\n"; } || { failn; output "$_error_: $p_name failed to reload dnsmasq!\n"; }
}
process_url() {
local label type D_TMP R_TMP
[[ -n "$1" && -n "$2" && -n "$3" ]] || return 1
local url=$1
[ "$2" == "hosts" ] && label="Hosts: $(echo $1 | cut -d'/' -f3)" filter="$h_filter" || label="Domains: $(echo $1 | cut -d'/' -f3)" filter="$d_filter"
[ "$3" == "blocked" ] && { type='Blocked'; D_TMP="$B_TMP"; } || { type='Allowed'; D_TMP="$A_TMP"; }
R_TMP="/var/simple-adblock_$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c10)"
while [ -e "$R_TMP" ]; do R_TMP="/var/simple-adblock_$(head /dev/urandom | tr -dc 'A-Za-z0-9' | head -c10)"; done
touch "$R_TMP"
output 2 "[DL] $type $label "
$dl "${url}" > "$R_TMP" && ok || fail
{ sed -i "$filter" "$R_TMP"; cat "$R_TMP" >> "$D_TMP"; rm -f "$R_TMP"; } &
}
start_adblocking () {
local whitelist_domains blacklist_domains whitelist_domains_urls blacklist_domains_urls blacklist_hosts_urls
config_get whitelist_domains 'config' 'whitelist_domain'
config_get blacklist_domains 'config' 'blacklist_domain'
config_get whitelist_domains_urls 'config' 'whitelist_domains_url'
config_get blacklist_domains_urls 'config' 'blacklist_domains_url'
config_get blacklist_hosts_urls 'config' 'blacklist_hosts_url'
local hf w_filter
[ ! -d ${hosts_file%/*} ] && mkdir -p ${hosts_file%/*}
if [[ -s $T_TMP && ! "$1" == "reload" ]]; then
output 3 'Found existing data file, reusing it '
mv $T_TMP $hosts_file && okn || failn
else
[ -f $A_TMP ] && rm -f $A_TMP; [ -f $B_TMP ] && rm -f $B_TMP; [ -f $T_TMP ] && rm -f $T_TMP; [ -f $hosts_file ] && rm -f $hosts_file
touch $A_TMP; touch $B_TMP; touch $T_TMP;
if [ -n "$blacklist_hosts_urls" ]; then
output 1 '[DL] Blocked Hosts '
for hf in ${blacklist_hosts_urls}; do process_url "$hf" 'hosts' 'blocked'; done
output 1 '\n'
fi
if [ -n "$blacklist_domains_urls" ]; then
output 1 '[DL] Blocked Domains '
for hf in ${blacklist_domains_urls}; do process_url "$hf" 'domains' 'blocked'; done
output 1 '\n'
fi
if [ -n "$whitelist_domains_urls" ]; then
output 1 '[DL] Allowed Domains '
for hf in ${whitelist_domains_urls}; do process_url "$hf" 'domains' 'allowed'; done
output 1 '\n'
fi
output 3 'Waiting for background processes '
wait && okn
[ -n "$blacklist_domains" ] && for hf in ${blacklist_domains}; do echo "$hf" | sed "$d_filter" >> $B_TMP; done
whitelist_domains="${whitelist_domains}"$'\n'"$(cat $A_TMP)"
[ -n "$whitelist_domains" ] && for hf in ${whitelist_domains}; do hf=$(echo $hf | sed 's/\./\\./g'); w_filter="$w_filter/^${hf}$/d;/\\.${hf}$/d;"; done
if [ -s $B_TMP ]; then
output 1 'Processing downloads '
output 2 'Sorting merged file '; sort $B_TMP | uniq > $T_TMP && ok || fail
output 2 'Whitelisting domains '; sed -i "$w_filter" $T_TMP && ok || fail
output 2 'Formatting merged file '; sed "$f_filter" $T_TMP > $hosts_file && ok || fail
output 1 '\n'
output 3 'Removing temporary files '
[ -f $A_TMP ] && rm -f $A_TMP; [ -f $B_TMP ] && rm -f $B_TMP; [ -f $T_TMP ] && rm -f $T_TMP;
okn
fi
fi
if [ -s $hosts_file ]; then
output 3 'Restarting dnsmasq '
/etc/init.d/dnsmasq restart >/dev/null 2>&1
if [[ $? -eq 0 ]]; then
led_on; okn;
output "$p_name blocking $(wc -l < $hosts_file) domains $_ok_\n"
else
failn; output "$_error_: $p_name failed to reload dnsmasq!\n";
exit 1
fi
else
output "$_error_: $p_name failed to create its data file!\n"
exit 1
fi
}
boot() { sleep 10; rc_procd start_service; rc_procd service_triggers; }
reload_service () {
is_enabled || return 1
[[ -t 1 && $bgrun -eq 1 ]] && (start_adblocking 'reload' | cat &) || start_adblocking 'reload'
}
start_service () {
is_enabled || return 1
procd_open_instance
procd_set_param stdout 1
procd_set_param stderr 1
procd_close_instance
reset_iptables quiet
set_iptables
[[ -t 1 && $bgrun -eq 1 ]] && (start_adblocking $1 | cat &) || start_adblocking $1
}
stop_service () {
is_enabled || return 1
reset_iptables
[[ -t 1 && $bgrun -eq 1 ]] && (stop_adblocking | cat &) || stop_adblocking
}
service_triggers () {
procd_add_reload_trigger 'simple-adblock'
}

Write Preview
Loading…
Cancel
Save