Browse Source

simple-adblock: bugfix: proper error reporting on failed downloads; lists update script

Signed-off-by: Stan Grishin <stangri@melmac.net>
lilik-openwrt-22.03
Stan Grishin 5 years ago
parent
commit
901159b8c7
5 changed files with 65 additions and 35 deletions
  1. +5
    -2
      net/simple-adblock/Makefile
  2. +29
    -22
      net/simple-adblock/files/README.md
  3. +5
    -5
      net/simple-adblock/files/simple-adblock.conf
  4. +4
    -0
      net/simple-adblock/files/simple-adblock.conf.update
  5. +22
    -6
      net/simple-adblock/files/simple-adblock.init

+ 5
- 2
net/simple-adblock/Makefile View File

@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk
PKG_NAME:=simple-adblock
PKG_VERSION:=1.8.3
PKG_RELEASE:=3
PKG_RELEASE:=9
PKG_MAINTAINER:=Stan Grishin <stangri@melmac.net>
PKG_LICENSE:=GPL-3.0-or-later
@ -21,7 +21,7 @@ endef
define Package/simple-adblock/description
This service provides DNSMASQ or Unbound based ad blocking.
Please see the project's README at github for further information.
Please see the project's README on GitHub for further information.
endef
@ -46,6 +46,8 @@ define Package/simple-adblock/install
$(INSTALL_BIN) $(PKG_BUILD_DIR)/files/simple-adblock.init $(1)/etc/init.d/simple-adblock
$(INSTALL_DIR) $(1)/etc/config
$(INSTALL_CONF) ./files/simple-adblock.conf $(1)/etc/config/simple-adblock
$(INSTALL_DIR) $(1)/tmp
$(INSTALL_CONF) ./files/simple-adblock.conf.update $(1)/tmp/simple-adblock.conf.update
$(INSTALL_DIR) $(1)/etc/hotplug.d/iface
$(INSTALL_BIN) ./files/simple-adblock.hotplug $(1)/etc/hotplug.d/iface/80-simple-adblock
endef
@ -54,6 +56,7 @@ define Package/simple-adblock/postinst
#!/bin/sh
# check if we are on real system
if [ -z "$${IPKG_INSTROOT}" ]; then
sed -f /tmp/simple-adblock.conf.update -i /etc/config/simple-adblock || true
/etc/init.d/simple-adblock enable
fi
exit 0


+ 29
- 22
net/simple-adblock/files/README.md View File

@ -19,63 +19,70 @@ A simple DNSMASQ/Unbound-based AdBlocking service for OpenWrt/LEDE Project.
- Blocks ads inside browsers with [DNS-over-HTTPS proxy](https://en.wikipedia.org/wiki/DNS_over_HTTPS) built-in, like [Mozilla Firefox](https://support.mozilla.org/en-US/kb/firefox-dns-over-https#w_about-dns-over-https) or [Google Chrome/Chromium](https://blog.chromium.org/2019/09/experimenting-with-same-provider-dns.html) -- with the ```dnsmasq.ipset``` option.
- Proudly made in Canada, using locally-sourced electrons.
If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out [net/adblock](https://github.com/openwrt/packages/tree/master/net/adblock/files).
If you want a more robust AdBlocking, supporting free memory detection and complex block lists, supporting IDN, check out ```net/adblock``` on [GitHub](https://github.com/openwrt/packages/tree/master/net/adblock/files)/[jsDelivr](https://cdn.jsdelivr.net/gh/openwrt/packages@master/net/adblock/files/README.md).
## Screenshots (luci-app-simple-adblock)
Service Status
![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-status.png "Service Status")
![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-status.png "Service Status")
Configuration - Basic Configuration
![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration")
![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-basic.png "Configuration - Basic Configuration")
Configuration - Advanced Configuration
![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration")
![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-config-advanced.png "Configuration - Advanced Configuration")
Whitelist and Blocklist Management
![screenshot](https://raw.githubusercontent.com/stangri/openwrt_packages/master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management")
![screenshot](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/screenshots/simple-adblock/screenshot08-lists.png "Whitelist and Blocklist Management")
## Requirements
This service requires the following packages to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound``` and either ```ca-certificates```, ```wget``` and ```libopenssl``` (for OpenWrt 15.05.1) or ```uclient-fetch``` and ```libustream-mbedtls``` (for LEDE Project and OpenWrt 18.06.xx or newer). Additionally installation of ```coreutils-sort``` is highly recommended as it speeds up blocklist processing.
To download the hosts/domains files this service requires one of the following packages to be installed on your device: ```wget``` or ```uclient-fetch``` or ```curl```. If you want to download lists over HTTPS (recommended), you also need to install support for HTTPS/SSL downloads on your device. For most devices starting with OpenWrt 15.05.1 and up the following commands will install the minimal requirements:
To satisfy the requirements for connect to your router via ssh and run the following commands:
```sh
opkg update; if ubus -S call system board | grep -q '15.05'; then
opkg install ca-certificates wget libopenssl; else
opkg install uclient-fetch libustream-mbedtls ca-bundle ca-certificates; fi
```
### How to use DNSMASQ ipset
### Requirements for file:// Scheme
The ```dnsmasq.ipset``` option requires you to install ```ipset``` and ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command:
If you want to include some local files as the blocklists or whitelists, you need to install ```curl```:
```sh
opkg update; opkg remove dnsmasq; opkg install dnsmasq-full ipset;
opkg update; opkg install curl;
```
### OpenWrt 15.05.1 Requirements
### Requirements for DNS Resolver
```sh
opkg update; opkg install ca-certificates wget libopenssl coreutils-sort dnsmasq
```
In order to actually block the ads, this service requires one of the following DNS resolvers to be installed on your router: ```dnsmasq``` or ```dnsmasq-full``` or ```unbound```. The ```dnsmasq``` package is usually included in the default OpenWrt installation.
### LEDE Project 17.01.x and OpenWrt 18.xx (or newer) Requirements
If you want to use the [```dnsmasq.ipset``` option](#dns-resolution-option) you need to install ```ipset``` and ```dnsmasq-full``` instead of the ```dnsmasq```. To do that, connect to your router via ssh and run the following command:
```sh
opkg update; opkg install uclient-fetch libustream-mbedtls coreutils-sort dnsmasq
opkg update; cd /tmp/; opkg download dnsmasq-full; opkg install ipset;
opkg remove dnsmasq; opkg install /tmp/dnsmasq-full*; rm -f /tmp/dnsmasq-full*;
```
### IPv6 Support
### Requirements for IPv6 Support
For IPv6 support additionally install ```ip6tables-mod-nat``` and ```kmod-ipt-nat6``` packages from Web UI or run the following in the command line:
```sh
opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6
opkg update; opkg install ip6tables-mod-nat kmod-ipt-nat6;
```
### Speed Up Blocklist Processing
### Requirements for Faster Blocklist Processing
The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run ```opkg --force-overwrite install coreutils-sort```.
The ```coreutils-sort``` is an optional, but recommended package as it speeds up sorting and removing duplicates from the merged list dramatically. If opkg complains that it can't install ```coreutils-sort``` because /usr/bin/sort is already provided by busybox, you can run the following command:
```sh
opkg --force-overwrite install coreutils-sort
```
## Unmet Dependencies
@ -89,11 +96,11 @@ Install ```simple-adblock``` and ```luci-app-simple-adblock``` packages from Web
opkg update; opkg install simple-adblock luci-app-simple-adblock
```
If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to [add a custom repo to your router](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router) first.
If ```simple-adblock``` and ```luci-app-simple-adblock``` packages are not found in the official feed/repo for your version of OpenWrt/LEDE Project, you will need to add a custom repo to your router following instructions on [GitHub](https://github.com/stangri/openwrt_packages/blob/master/README.md#on-your-router)/[jsDelivr](https://cdn.jsdelivr.net/gh/stangri/openwrt_packages@master/README.md#on-your-router) first.
## Default Settings
Default configuration has service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM.
Default configuration has the service disabled (use Web UI to enable/start service or run ```uci set simple-adblock.config.enabled=1; uci commit simple-adblock;```) and selected ad/malware lists suitable for routers with 64Mb RAM.
If your router has less then 64Mb RAM, edit the configuration file, located at ```/etc/config/simple-adblock```. The configuration file has lists in ascending order starting with smallest ones and each list has a preceding comment indicating its size, comment out or delete the lists you don't want or your router can't handle.


+ 5
- 5
net/simple-adblock/files/simple-adblock.conf View File

@ -11,7 +11,7 @@ config simple-adblock 'config'
option parallel_downloads '1'
option debug '0'
option compressed_cache '0'
list whitelist_domain 'raw.githubusercontent.com'
list whitelist_domain 'cdn.jsdelivr.net'
# Thu Oct 3 17:54:04 PDT 2019
# File size: 4.0K
@ -45,7 +45,7 @@ config simple-adblock 'config'
list blacklist_hosts_url 'https://adaway.org/hosts.txt'
# File size: 20.0K
list blacklist_hosts_url 'https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt'
list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@master/hosts.txt'
# File size: 36.0K
list blacklist_hosts_url 'https://www.malwaredomainlist.com/hostslist/hosts.txt'
@ -56,7 +56,7 @@ config simple-adblock 'config'
# File size: 388.0K
# blocklist may be too big for some routers
# blocklist may block some video-streaming content
# list blacklist_hosts_url 'https://raw.githubusercontent.com/jawz101/MobileAdTrackers/master/hosts'
# list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@master/hosts'
# File size: 424.0K
# blocklist may be too big for some routers
@ -72,7 +72,7 @@ config simple-adblock 'config'
# File size: 1.6M
# blocklist too big for most routers
# list blacklist_hosts_url 'https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts'
# list blacklist_hosts_url 'https://cdn.jsdelivr.net/gh/StevenBlack/hosts@master/hosts'
# File size: 3.1M
# blocklist too big for most routers
@ -80,7 +80,7 @@ config simple-adblock 'config'
# File size: 23.0M
# blocklist too big for most routers
# list blacklist_hosts_url 'https://dbl.oisd.nl/'
# list blacklist_hosts_url 'https://hosts.oisd.nl/'
# site was down on last check
# list blacklist_domains_url 'http://support.it-mate.co.uk/downloads/hosts.txt'

+ 4
- 0
net/simple-adblock/files/simple-adblock.conf.update View File

@ -0,0 +1,4 @@
s|dbl.oisd.nl|hosts.oisd.nl|g
s|raw.githubusercontent.com/StevenBlack/hosts/|cdn.jsdelivr.net/gh/StevenBlack/hosts@|g
s|raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/|cdn.jsdelivr.net/gh/hoshsadiq/adblock-nocoin-list@|g
s|raw.githubusercontent.com/jawz101/MobileAdTrackers/|cdn.jsdelivr.net/gh/jawz101/MobileAdTrackers@|g

+ 22
- 6
net/simple-adblock/files/simple-adblock.init View File

@ -145,6 +145,7 @@ export whitelist_domains blacklist_domains
export whitelist_domains_urls blacklist_domains_urls blacklist_hosts_urls
export wan_if wan_gw wanphysdev dl_command serviceStatus dl_flag
export outputFilter outputFilterIPv6 outputFile outputGzip outputCache ipv6Enabled
export is_ssl_supported
load_package_config() {
config_load "$packageName"
@ -231,11 +232,11 @@ load_package_config() {
fi
. /lib/functions/network.sh
. /usr/share/libubox/jshn.sh
# Prefer curl because it supports the file: scheme.
# Prefer curl because it supports the file:// scheme.
if [ -x /usr/bin/curl ]; then
dl_command="curl --insecure --retry $curlRetry --connect-timeout $dlTimeout --silent"
dl_flag="-o"
elif wget -V 2>/dev/null | grep -q "+ssl"; then
elif wget --version 2>/dev/null | grep -q "+https"; then
dl_command="wget --no-check-certificate --timeout $dlTimeout -q"
dl_flag="-O"
else
@ -243,6 +244,15 @@ load_package_config() {
dl_flag="-O"
fi
led="${led:+/sys/class/leds/$led}"
if curl --version 2>/dev/null | grep -q "https" \
|| wget --version 2>/dev/null | grep -q "+https" \
|| grep -q "libustream-mbedtls" /usr/lib/opkg/status \
|| grep -q "libustream-openssl" /usr/lib/opkg/status \
|| grep -q "libustream-wolfssl" /usr/lib/opkg/status; then
is_ssl_supported=1
else
unset is_ssl_supported
fi
}
is_enabled() {
@ -668,19 +678,25 @@ process_url() {
else
type='Allowed'; D_TMP="$A_TMP";
fi
if [ "${1:0:5}" == "https" ] && [ -z "$is_ssl_supported" ]; then
output 1 "$_FAIL_"
output 2 "[DL] $type $label $__FAIL__\\n"
echo "errorNoSSLSupport|${1}" >> "$sharedMemoryError"
return 0
fi
while [ -z "$R_TMP" ] || [ -e "$R_TMP" ]; do
R_TMP="$(mktemp -u -q -t ${packageName}_tmp.XXXXXXXX)"
done
if ! $dl_command "$1" $dl_flag "$R_TMP" 2>/dev/null || [ ! -s "$R_TMP" ]; then
output 1 "$_FAIL_"
output 2 "[DL] $type $label $__FAIL__\\n"
echo "errorDownloadingList=${1}" >> "$sharedMemoryError"
echo "errorDownloadingList|${1}" >> "$sharedMemoryError"
else
sed -i "$filter" "$R_TMP"
if [ ! -s "$R_TMP" ]; then
output 1 "$_FAIL_"
output 2 "[DL] $type $label $__FAIL__\\n"
echo "errorParsingList=${1}" >> "$sharedMemoryError"
echo "errorParsingList|${1}" >> "$sharedMemoryError"
else
cat "${R_TMP}" >> "$D_TMP"
output 1 "$_OK_"
@ -1115,8 +1131,8 @@ showstatus() {
fi
if [ -n "$error" ]; then
for c in $error; do
url="${c##*=}"
c="${c%=*}"
url="${c##*|}"
c="${c%|*}"
case "$c" in
errorDownloadingList|errorParsingList)
output "$_ERROR_: $(getErrorText "$c") $url!\\n";;


Loading…
Cancel
Save