You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

434 lines
14 KiB

  1. #!/bin/sh
  2. # ad/abuse domain blocking script for dnsmasq/openwrt
  3. # written by Dirk Brenken (openwrt@brenken.org)
  4. # This is free software, licensed under the GNU General Public License v3.
  5. # You should have received a copy of the GNU General Public License
  6. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  7. # set the C locale
  8. #
  9. LC_ALL=C
  10. # script debug switch (disabled by default)
  11. # set 'DEBUG=1' to enable script debugging
  12. #
  13. DEBUG=0
  14. if [ $((DEBUG)) -eq 0 ]
  15. then
  16. exec 2>/dev/null
  17. fi
  18. # pid handling
  19. #
  20. adb_pid="${$}"
  21. adb_pidfile="/var/run/adblock.pid"
  22. if [ -r "${adb_pidfile}" ]
  23. then
  24. rc=255
  25. /usr/bin/logger -s -t "adblock[${adb_pid}] error" "adblock service already running ($(cat ${adb_pidfile}))"
  26. exit ${rc}
  27. else
  28. printf "${adb_pid}" > "${adb_pidfile}"
  29. fi
  30. # get current directory, script- and openwrt version
  31. #
  32. adb_scriptdir="${0%/*}"
  33. adb_scriptver="1.0.0"
  34. openwrt_version="$(cat /etc/openwrt_version)"
  35. # source in adblock function library
  36. #
  37. if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
  38. then
  39. . "${adb_scriptdir}/adblock-helper.sh"
  40. else
  41. rc=254
  42. /usr/bin/logger -s -t "adblock[${adb_pid}] error" "adblock function library not found"
  43. rm -f "${adb_pidfile}"
  44. exit ${rc}
  45. fi
  46. # call trap function on error signals (HUP, INT, QUIT, BUS, SEGV, TERM)
  47. #
  48. trap "rc=250; f_log 'error signal received/trapped' '${rc}'; f_exit" 1 2 3 10 11 15
  49. # load environment
  50. #
  51. f_envload
  52. # start logging
  53. #
  54. f_log "domain adblock processing started (${adb_scriptver}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
  55. # parse environment
  56. #
  57. f_envparse
  58. # check environment
  59. #
  60. f_envcheck
  61. # start shallalist (pre-)processing
  62. #
  63. if [ -n "${adb_arc_shalla}" ]
  64. then
  65. # start shallalist processing
  66. #
  67. shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
  68. shalla_file="${adb_tmpdir}/shallalist.txt"
  69. src_name="shalla"
  70. adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
  71. if [ -r "${adb_dnsfile}" ]
  72. then
  73. list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}")"
  74. fi
  75. f_log "=> (pre-)processing adblock source '${src_name}'"
  76. # only process shallalist archive with updated timestamp,
  77. # extract and merge only domains of selected shallalist categories
  78. #
  79. shalla_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${adb_arc_shalla}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
  80. if [ -z "${shalla_time}" ]
  81. then
  82. shalla_time="$(date)"
  83. f_log " no online timestamp received, current date will be used"
  84. fi
  85. if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ]
  86. then
  87. ${adb_fetch} ${wget_parm} --output-document="${shalla_archive}" "${adb_arc_shalla}"
  88. rc=${?}
  89. if [ $((rc)) -eq 0 ]
  90. then
  91. > "${shalla_file}"
  92. for category in ${adb_cat_shalla}
  93. do
  94. tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
  95. rc=${?}
  96. if [ $((rc)) -ne 0 ]
  97. then
  98. f_log " archive extraction failed (${category})"
  99. break
  100. fi
  101. done
  102. # remove temporary files
  103. #
  104. rm -f "${shalla_archive}"
  105. rm -rf "${adb_tmpdir}/BL"
  106. if [ $((rc)) -eq 0 ]
  107. then
  108. adb_sources="${adb_sources} ${shalla_file}&ruleset=rset_shalla"
  109. f_log " source archive (pre-)processing finished"
  110. else
  111. rc=0
  112. fi
  113. else
  114. rc=0
  115. adb_errsrclist="-name ${adb_dnsprefix}.${src_name}"
  116. f_log " source archive download failed"
  117. fi
  118. else
  119. adb_srclist="! -name ${adb_dnsprefix}.${src_name}"
  120. f_log " source archive doesn't change, no update required"
  121. fi
  122. fi
  123. # add blacklist source to active adblock domain sources
  124. #
  125. if [ -s "${adb_blacklist}" ]
  126. then
  127. adb_sources="${adb_sources} ${adb_blacklist}&ruleset=rset_blacklist"
  128. fi
  129. # loop through active adblock domain sources,
  130. # download sources, prepare output and store all extracted domains in temp file
  131. #
  132. for src in ${adb_sources}
  133. do
  134. url="${src/\&ruleset=*/}"
  135. src_name="${src/*\&ruleset=rset_/}"
  136. adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
  137. if [ -r "${adb_dnsfile}" ]
  138. then
  139. list_time="$(awk '$0 ~ /^# last modified/ {printf substr($0,18)}' "${adb_dnsfile}")"
  140. fi
  141. f_log "=> processing adblock source '${src_name}'"
  142. # prepare find statement with active adblock list sources
  143. #
  144. if [ -z "${adb_srclist}" ]
  145. then
  146. adb_srclist="! -name ${adb_dnsprefix}.${src_name}"
  147. else
  148. adb_srclist="${adb_srclist} -a ! -name ${adb_dnsprefix}.${src_name}"
  149. fi
  150. # only download adblock list with newer/updated timestamp
  151. #
  152. if [ "${src_name}" = "blacklist" ]
  153. then
  154. url_time="$(date -r "${adb_blacklist}")"
  155. elif [ "${src_name}" = "shalla" ]
  156. then
  157. url_time="${shalla_time}"
  158. else
  159. url_time="$(${adb_fetch} ${wget_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
  160. fi
  161. if [ -z "${url_time}" ]
  162. then
  163. url_time="$(date)"
  164. f_log " no online timestamp received, current date will be used"
  165. fi
  166. if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
  167. then
  168. if [ "${src_name}" = "blacklist" ]
  169. then
  170. tmp_domains="$(cat "${adb_blacklist}")"
  171. rc=${?}
  172. elif [ "${src_name}" = "shalla" ]
  173. then
  174. tmp_domains="$(cat "${shalla_file}")"
  175. rc=${?}
  176. else
  177. tmp_domains="$(${adb_fetch} ${wget_parm} --output-document=- "${url}")"
  178. rc=${?}
  179. fi
  180. else
  181. f_log " source doesn't change, no update required"
  182. continue
  183. fi
  184. # check download result and prepare domain output by regex patterns
  185. #
  186. if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ]
  187. then
  188. eval "src_rset=\${rset_${src_name}}"
  189. count="$(printf "%s\n" "${tmp_domains}" | eval "${src_rset}" | tee "${adb_tmpfile}" | wc -l)"
  190. f_log " source download finished (${count} entries)"
  191. if [ "${src_name}" = "shalla" ]
  192. then
  193. rm -f "${shalla_file}"
  194. fi
  195. unset tmp_domains
  196. elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
  197. then
  198. f_log " empty source download finished"
  199. continue
  200. else
  201. rc=0
  202. if [ -z "${adb_errsrclist}" ]
  203. then
  204. adb_errsrclist="-name ${adb_dnsprefix}.${src_name}"
  205. else
  206. adb_errsrclist="${adb_errsrclist} -o -name ${adb_dnsprefix}.${src_name}"
  207. fi
  208. f_log " source download failed"
  209. continue
  210. fi
  211. # remove whitelist domains, sort domains and make them unique,
  212. # finally rewrite ad/abuse domain information to separate dnsmasq files
  213. #
  214. if [ $((count)) -gt 0 ] && [ -n "${adb_tmpfile}" ]
  215. then
  216. if [ -s "${adb_whitelist}" ]
  217. then
  218. grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
  219. rc=${?}
  220. else
  221. sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
  222. rc=${?}
  223. fi
  224. # prepare find statement with revised adblock list sources
  225. #
  226. if [ -z "${adb_revsrclist}" ]
  227. then
  228. adb_revsrclist="-name ${adb_dnsprefix}.${src_name}"
  229. else
  230. adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}"
  231. fi
  232. # write preliminary footer
  233. #
  234. if [ $((rc)) -eq 0 ]
  235. then
  236. printf "%s\n" "#---------------------------------------------" >> "${adb_dnsfile}"
  237. printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}"
  238. printf "%s\n" "##" >> "${adb_dnsfile}"
  239. f_log " domain merging finished"
  240. else
  241. f_log " domain merging failed" "${rc}"
  242. f_restore
  243. fi
  244. else
  245. f_log " empty domain input received"
  246. continue
  247. fi
  248. done
  249. # remove disabled adblock lists and their backups
  250. #
  251. if [ -n "${adb_srclist}" ]
  252. then
  253. rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -print -exec rm -f "{}" \;)"
  254. rc=${?}
  255. if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
  256. then
  257. f_log "disabled adblock lists removed"
  258. if [ "${backup_ok}" = "true" ]
  259. then
  260. rm_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srclist} \) -print -exec rm -f "{}" \;)"
  261. rc=${?}
  262. if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
  263. then
  264. f_log "disabled adblock list backups removed"
  265. elif [ $((rc)) -ne 0 ]
  266. then
  267. f_log "error during removal of disabled adblock list backups" "${rc}"
  268. f_exit
  269. fi
  270. fi
  271. elif [ $((rc)) -ne 0 ]
  272. then
  273. f_log "error during removal of disabled adblock lists" "${rc}"
  274. f_exit
  275. fi
  276. else
  277. rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec rm -f "{}" \;)"
  278. rc=${?}
  279. if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
  280. then
  281. f_log "all adblock lists removed"
  282. if [ "${backup_ok}" = "true" ]
  283. then
  284. rm_done="$(find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" -print -exec rm -f "{}" \;)"
  285. rc=${?}
  286. if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
  287. then
  288. f_log "all adblock list backups removed"
  289. elif [ $((rc)) -ne 0 ]
  290. then
  291. f_log "error during removal of all adblock list backups" "${rc}"
  292. f_exit
  293. fi
  294. fi
  295. elif [ $((rc)) -ne 0 ]
  296. then
  297. f_log "error during removal of all adblock lists" "${rc}"
  298. f_exit
  299. fi
  300. fi
  301. # partial restore of adblock lists in case of download errors
  302. #
  303. if [ "${backup_ok}" = "true" ] && [ -n "${adb_errsrclist}" ]
  304. then
  305. restore_done="$(find "${adb_backupdir}" -maxdepth 1 -type f \( ${adb_errsrclist} \) -print -exec cp -pf "{}" "${adb_dnsdir}" \;)"
  306. rc=${?}
  307. if [ $((rc)) -eq 0 ] && [ -n "${restore_done}" ]
  308. then
  309. f_log "partial restore done"
  310. elif [ $((rc)) -ne 0 ]
  311. then
  312. f_log "error during partial restore" "${rc}"
  313. f_exit
  314. fi
  315. fi
  316. # make separate adblock lists entries unique
  317. #
  318. if [ "${mem_ok}" != "false" ]
  319. then
  320. if [ -n "${adb_revsrclist}" ]
  321. then
  322. f_log "remove duplicates in separate adblock lists"
  323. # generate a temporary unique overall list
  324. #
  325. head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}."* | sort -u > "${adb_dnsdir}/tmp.overall"
  326. # loop through all separate lists, ordered by size (ascending)
  327. #
  328. for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."*)
  329. do
  330. # check original separate list vs. temporary overall list,
  331. # rewrite only duplicate entries back to temporary separate list
  332. #
  333. list="${list/*./}"
  334. sort "${adb_dnsdir}/tmp.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -d > "${adb_dnsdir}/tmp.${list}"
  335. # rewrite only unique entries back to temporary overall list
  336. #
  337. tmp_unique="$(sort "${adb_dnsdir}/tmp.overall" "${adb_dnsdir}/tmp.${list}" | uniq -u)"
  338. printf "%s\n" "${tmp_unique}" > "${adb_dnsdir}/tmp.overall"
  339. # write unique result back to original separate list (with list footer)
  340. #
  341. tail -qn 3 "${adb_dnsdir}/$adb_dnsprefix.${list}" >> "${adb_dnsdir}/tmp.${list}"
  342. mv -f "${adb_dnsdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
  343. done
  344. rm -f "${adb_dnsdir}/tmp.overall"
  345. fi
  346. fi
  347. # set separate list count & get overall count
  348. #
  349. for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."*)
  350. do
  351. list="${list/*./}"
  352. count="$(head -qn -3 "${adb_dnsdir}/${adb_dnsprefix}.${list}" | wc -l)"
  353. if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
  354. then
  355. count=$((count / 2))
  356. fi
  357. if [ "$(tail -qn 1 "${adb_dnsdir}/${adb_dnsprefix}.${list}")" = "##" ]
  358. then
  359. last_line="# ${0##*/} (${adb_scriptver}) - ${count} ad\/abuse domains blocked"
  360. sed -i "s/^##$/${last_line}/" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
  361. fi
  362. adb_count=$((adb_count + count))
  363. done
  364. # restart dnsmasq with newly generated or deleted adblock lists,
  365. # check dnsmasq startup afterwards
  366. #
  367. if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ] || [ -n "${restore_done}" ]
  368. then
  369. /etc/init.d/dnsmasq restart
  370. sleep 1
  371. rc="$(ps | grep -q "[d]nsmasq"; printf ${?})"
  372. if [ $((rc)) -eq 0 ]
  373. then
  374. f_log "adblock lists with overall ${adb_count} domains loaded"
  375. else
  376. rc=100
  377. f_log "dnsmasq restart failed, please check 'logread' output" "${rc}"
  378. f_restore
  379. fi
  380. else
  381. f_log "adblock lists with overall ${adb_count} domains are still valid, no dnsmasq restart required"
  382. fi
  383. # create adblock list backups
  384. #
  385. if [ "${backup_ok}" = "true" ] && [ -n "${adb_revsrclist}" ] && [ "$(printf "${adb_dnsdir}/${adb_dnsprefix}."*)" != "${adb_dnsdir}/${adb_dnsprefix}.*" ]
  386. then
  387. backup_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec cp -pf "{}" "${adb_backupdir}" \;)"
  388. rc=${?}
  389. if [ $((rc)) -eq 0 ] && [ -n "${backup_done}" ]
  390. then
  391. f_log "new adblock list backups generated"
  392. elif [ $((rc)) -ne 0 ]
  393. then
  394. f_log "error during backup of adblock lists" "${rc}"
  395. f_exit
  396. fi
  397. fi
  398. # remove temporary files and exit
  399. #
  400. f_exit