Merge pull request #2138 from dibdot/adblock

adblock: feature update

net/adblock/Makefile

@@ -7,7 +7,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=0.22.2
+PKG_VERSION:=0.40.0
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org>
@@ -37,8 +37,10 @@ The script supports the following domain blacklist sites:
   http://www.shallalist.de
   http://www.spam404.com
   http://winhelp2002.mvps.org
+  https://adaway.org
+  https://disconnect.me
 
-Please read README.md in /etc/adblock for further information.
+Please see README.md in /etc/adblock for further information.
 
 endef
 

net/adblock/files/README.md

@@ -9,27 +9,43 @@ When the dns server on your router receives dns requests, you’ll sort out quer
 and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
 
 ## Main Features
-* support of the following domain blacklist sites (free for private usage, for commercial use please check their individual licenses):
-    * [pgl.yoyo.org](http://pgl.yoyo.org/adservers), approx. 2.500 entries
-    * [malwaredomains.com](http://malwaredomains.com), approx. 16.000 entries
-    * [zeustracker.abuse.ch](https://zeustracker.abuse.ch), approx. 420 entries
-    * [feodotracker.abuse.ch](https://feodotracker.abuse.ch), approx. 10 entries
-    * [palevotracker.abuse.ch](https://palevotracker.abuse.ch), approx. 10 entries
-    * [dshield.org](http://dshield.org), approx. 4.500 entries
-    * [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default), approx. 32.000 entries
-    * a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html)
-    * [spam404.com](http://www.spam404.com), approx. 5.000 entries
-    * [winhelp2002.mvps.org](http://winhelp2002.mvps.org), approx. 15.000 entries
+* support of the following domain blocklist sources (free for private usage, for commercial use please check their individual licenses):
+    * [pgl.yoyo.org](http://pgl.yoyo.org/adservers)
+    * => weekly updates, approx. 2.500 entries (enabled by default)
+    * [malwaredomains.com](http://malwaredomains.com)
+    * => daily updates, approx. 16.000 entries
+    * [zeustracker.abuse.ch](https://zeustracker.abuse.ch)
+    * => daily updates, approx. 440 entries
+    * [feodotracker.abuse.ch](https://feodotracker.abuse.ch)
+    * => daily updates, approx. 0-10 entries
+    * [palevotracker.abuse.ch](https://palevotracker.abuse.ch)
+    * => daily updates, approx. 15 entries
+    * [dshield.org](http://dshield.org)
+    * => daily updates, approx. 4.500 entries
+    * [shallalist.de](http://www.shallalist.de) (categories "adv" "costtraps" "spyware" "tracker" "warez" enabled by default)
+    * => daily updates, approx. 32.000 entries (a short description of all shallalist categories can be found [online](http://www.shallalist.de/categories.html))
+    * [spam404.com](http://www.spam404.com)
+    * => infrequent updates, approx. 5.000 entries
+    * [winhelp2002.mvps.org](http://winhelp2002.mvps.org)
+    * => infrequent updates, approx. 15.000 entries
+    * [adaway.org](https://adaway.org)
+    * => infrequent updates, approx. 400 entries
+    * [disconnect.me](https://disconnect.me)
+    * => numerous updates on the same day, approx. 6.500 entries
+* each blocklist source will be updated and processed separately
+* timestamp check to download and process only updated blocklists
+* overall duplicate removal in separate blocklists (will be automatically disabled on low memory systems)
 * blocklist parsing by fast & flexible regex rulesets
 * additional white- and blacklist support for manual overrides
-* separate dynamic adblock network interface
-* separate dynamic uhttpd instance as pixel server
-* adblock quality checks after list update to ensure a reliable dnsmasq service
-* optional: powerful adblock list backup/restore handling
-* optional: adblock updates only on pre-defined wan interfaces (useful for (mobile) multiwan setups)
-* optional: domain query logging as a background service to easily identify free and already blocked domains (see example output below)
-* optional: status & error logging to separate file (req. ntp time sync)
-* optional: ntp time sync
+* use of dynamic adblock network interface
+* use of dynamic uhttpd instance as pixel server
+* use of quality checks after adblocklist updates to ensure a reliable dnsmasq service
+* optional features (disabled by default): 
+    * powerful adblock list backup/restore handling
+    * adblock updates only on pre-defined wan interfaces (useful for (mobile) multiwan setups)
+    * domain query logging as a background service to easily identify free and already blocked domains (see example output below)
+    * ntp time sync
+    * status & error logging to separate file (req. ntp time sync)
 
 ## Prerequisites
 * [openwrt](https://openwrt.org) (tested only with trunk > r47025), CC should also work
@@ -37,78 +53,87 @@ and return the local ip address of your router and the internal web server deliv
     * curl
     * wget (due to an openwrt bug still needed for certain https requests - see ticket #19621)
     * optional: busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found)
-    * optional: coreutils-sort for reliable sort results, even on low memory systems
-* recommended: add an usb stick or any other storage device to supersize your /tmp directory with a swap partition (see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab))
 * the above dependencies and requirements will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors
 
 ## Usage
 * select & install adblock package (*opkg install adblock*)
 * configure */etc/config/adblock* to your needs, see additional comments in *adblock.conf.sample*
-* at least configure the ip address of the local adblock interface/uhttpd instance, needs to be a different subnet from the normal LAN
-* optional: add additional domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), both list are located in */etc/adblock*
-* by default openwrts main uhttpd instance is bind to all ports of your router. For a working adblock setup you have to bind uhttpd to the standard LAN port only, please change listen_http accordingly
+* at least configure the ip address of the local adblock interface/uhttpd instance, it needs to be a different subnet from the normal LAN
+* recommendation: there is no need to enable all blacklist sites at once, for normal use one to three lists should be sufficient
+* recommendation: to handle all blacklists at once add an usb stick or any other storage device to supersize your /tmp directory with a swap partition
+* => see [openwrt wiki](https://wiki.openwrt.org/doc/uci/fstab) for further details
+* add additional domain white- or blacklist entries, one domain per line (wildcards & regex are not allowed!), both lists are located in */etc/adblock*
+* by default openwrts main uhttpd instance is bind to all ports of your router,
+* for a working adblock setup you have to bind uhttpd to the standard LAN port only, please change listen_http accordingly
 * start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors
 
 ## Distributed samples
 * all sample configuration files stored in */etc/adblock/samples*
 * to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*
 * to bind uhttpd to standard LAN port only, see *uhttpd.config.sample*
-* for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample*
+* for rc.local based autostart and /tmp resizing on the fly see *rc.local.sample*
 * for scheduled call of *adblock-update.sh* see *root.crontab.sample*
 * to redirect/force all dns queries to your router see *firwall.user.sample*
 * for further dnsmasq tweaks see *dnsmasq.conf.sample*
 
 ## Examples
 
-  stdout excerpt for successful adblock run:  
+  log of a full adblock run:  
     
-    adblock[11541] info : domain adblock processing started (0.22.2, r47665, 29.11.2015 14:58:11)  
-    adblock[11541] info : wan update check will be disabled  
-    adblock[11541] info : get ntp time sync (192.168.254.254), after 0 loops  
-    adblock[11541] info : shallalist (pre-)processing started ...  
-    adblock[11541] info : shallalist (pre-)processing finished (adv costtraps spyware tracker warez)  
-    adblock[11541] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2423 entries)  
-    adblock[11541] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 16016 entries)  
-    adblock[11541] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 419 entries)  
-    adblock[11541] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries)  
-    adblock[11541] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 12 entries)  
-    adblock[11541] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries)  
-    adblock[11541] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries)  
-    adblock[11541] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13635 entries)  
-    adblock[11541] info : source download finished (file:////tmp/tmp.CgbMmO/shallalist.txt, 32446 entries)  
-    adblock[11541] info : empty source download finished (file:///etc/adblock/adblock.blacklist)  
-    adblock[11541] info : domain merging finished  
-    adblock[11541] info : new adblock list with 69646 domains loaded, backup generated  
-    adblock[11541] info : domain adblock processing finished (0.22.2, r47665, 29.11.2015 14:59:23)  
+    adblock[1586] info : domain adblock processing started (0.40.0, r47670, 17.12.2015 10:50:09)  
+    adblock[1586] info : wan update check will be disabled  
+    adblock[1586] info : get ntp time sync (192.168.2.254), after 0 loops  
+    adblock[1586] info : created new dynamic/volatile network interface (adblock, 192.168.2.1)  
+    adblock[1586] info : created new dynamic/volatile uhttpd instance (adblock, 192.168.2.1)  
+    adblock[1586] info : shallalist (pre-)processing started ...  
+    adblock[1586] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2432 entries)  
+    adblock[1586] info : domain merging finished (yoyo)  
+    adblock[1586] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 17368 entries)  
+    adblock[1586] info : domain merging finished (malware)  
+    adblock[1586] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 440 entries)  
+    adblock[1586] info : domain merging finished (zeus)  
+    adblock[1586] info : no online timestamp received, current date will be used (feodo)  
+    adblock[1586] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries)  
+    adblock[1586] info : empty domain input received (feodo)  
+    adblock[1586] info : no online timestamp received, current date will be used (palevo)  
+    adblock[1586] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 16 entries)  
+    adblock[1586] info : domain merging finished (palevo)  
+    adblock[1586] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries)  
+    adblock[1586] info : domain merging finished (dshield)  
+    adblock[1586] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries)  
+    adblock[1586] info : domain merging finished (spam404)  
+    adblock[1586] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13635 entries)  
+    adblock[1586] info : domain merging finished (winhelp)  
+    adblock[1586] info : source download finished (https://adaway.org/hosts.txt, 410 entries)  
+    adblock[1586] info : domain merging finished (adaway)  
+    adblock[1586] info : source download finished (https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt, 6343 entries)  
+    adblock[1586] info : domain merging finished (disconnect)  
+    adblock[1586] info : source download finished (file:////tmp/tmp.FIhIBh/shallalist.txt, 32458 entries)  
+    adblock[1586] info : domain merging finished (shalla)  
+    adblock[1586] info : source download finished (file:///etc/adblock/adblock.blacklist, 1 entries)  
+    adblock[1586] info : domain merging finished (blacklist)  
+    adblock[1586] info : remove duplicates in separate adblocklists ...  
+    adblock[1586] info : adblocklists with overall 71552 domains loaded, new backups generated  
+    adblock[1586] info : new domain query log background process started (pid: 2416)  
+    adblock[1586] info : domain adblock processing finished (0.40.0, r47670, 17.12.2015 10:52:47)  
     
 
-  generated domain blocklist for dnsmasq:  
+  domain blocklist for dnsmasq (disconnect.me after overall duplicate removal):  
     
-    address=/0-29.com/192.168.2.1  
-    address=/0-2u.com/192.168.2.1  
-    address=/0.r.msn.com/192.168.2.1  
-    address=/00.devoid.us/192.168.2.1  
-    address=/000007.ru/192.168.2.1  
+    address=/0000mps.webpreview.dsl.net/192.168.2.1  
+    address=/0001.2waky.com/192.168.2.1  
+    address=/001wen.com/192.168.2.1  
+    address=/002it.com/192.168.2.1  
+    address=/00game.net/192.168.2.1  
     [...]  
-    address=/zzz.cn/192.168.2.1  
-    address=/zzzjsh.com/192.168.2.1  
-    ####################################################  
-    # last adblock list update: 20.11.2015 - 18:00:02  
-    # adblock-update.sh (0.21.0) - 73087 ad/abuse domains blocked  
-    # domain blacklist sources:  
-    # http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext  
-    # http://mirror1.malwaredomains.com/files/justdomains  
-    # https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist  
-    # https://feodotracker.abuse.ch/blocklist/?download=domainblocklist  
-    # https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist  
-    # http://www.dshield.org/feeds/suspiciousdomains_Low.txt  
-    # http://spam404bl.com/spam404scamlist.txt  
-    # http://winhelp2002.mvps.org/hosts.txt  
-    # file:////tmp/tmp.CLBLNF/shallalist.txt  
-    # file:///etc/adblock/adblock.blacklist  
-    #####  
-    # /etc/adblock/adblock.whitelist  
-    ####################################################  
+    address=/zzsgssxh.com/192.168.2.1  
+    address=/zzshw.net/192.168.2.1  
+    address=/zztxdown.com/192.168.2.1  
+    address=/zzxcws.com/192.168.2.1  
+    #------------------------------------------------------------------  
+    # adblock-update.sh (0.40.0) - 3710 ad/abuse domains blocked  
+    # source: https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt  
+    # last modified: Thu, 17 Dec 2015 09:21:17 GMT  
     
 
   domain query log excerpt:  

net/adblock/files/adblock-helper.sh

@@ -13,9 +13,9 @@ f_envload()
     #
     if [ -r "/lib/functions.sh" ]
     then
-        . /lib/functions.sh
+        . "/lib/functions.sh" 2>/dev/null
     else
-        rc=510
+        rc=500
         f_log "openwrt function library not found" "${rc}"
         f_deltemp
     fi
@@ -24,9 +24,9 @@ f_envload()
     #
     if [ -r "/usr/share/libubox/jshn.sh" ]
     then
-        . "/usr/share/libubox/jshn.sh"
+        . "/usr/share/libubox/jshn.sh" 2>/dev/null
     else
-        rc=515
+        rc=505
         f_log "openwrt json helpers library not found" "${rc}"
         f_deltemp
     fi
@@ -36,7 +36,7 @@ f_envload()
     pkg_list="$(opkg list-installed 2>/dev/null)"
     if [ -z "${pkg_list}" ]
     then
-        rc=520
+        rc=510
         f_log "empty openwrt package list" "${rc}"
         f_deltemp
     fi
@@ -48,16 +48,19 @@ f_envload()
 f_envparse()
 {
     # set the C locale, characters are single bytes, the charset is ASCII
-    # speeds up sort, grep etc., guarantees unique domains
+    # speeds up sort, grep etc.
     #
     LC_ALL=C
 
-    # set initial defaults (may be overwritten by adblock config options)
+    # set initial defaults (may be overwritten by setting appropriate adblock config options)
     #
     adb_if="adblock"
     adb_minspace="20000"
     adb_maxtime="60"
     adb_maxloop="5"
+    adb_unique="1"
+    adb_blacklist="/etc/adblock/adblock.blacklist"
+    adb_whitelist="/etc/adblock/adblock.whitelist"
 
     # adblock device name auto detection
     # derived from first entry in openwrt lan ifname config
@@ -90,7 +93,7 @@ f_envparse()
             {
                 local option="${1}"
                 local value="${2}"
-                local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p')"
+                local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p' 2>/dev/null)"
                 if [ -z "${opt_out}" ]
                 then
                     all_options="${all_options} ${option}"
@@ -127,7 +130,7 @@ f_envparse()
                 config_get value "${config}" "${option}"
                 if [ -n "${value}" ]
                 then
-                    local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p')"
+                    local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p' 2>/dev/null)"
                     if [ -n "${opt_src}" ]
                     then
                         adb_sources="${adb_sources} ${value}"
@@ -138,13 +141,13 @@ f_envparse()
             done
         elif [ "${config}" = "wancheck" ]
         then
-           unset adb_wandev 2>/dev/null
+           unset adb_wandev
         elif [ "${config}" = "ntpcheck" ]
         then
-           unset adb_ntpsrv 2>/dev/null
+           unset adb_ntpsrv
         elif [ "${config}" = "shalla" ]
         then
-           unset adb_cat_shalla 2>/dev/null
+           unset adb_cat_shalla
         fi
     }
 
@@ -154,25 +157,35 @@ f_envparse()
     config_foreach parse_config service
     config_foreach parse_config source
 
-    # set temp variables and counter
+    # set temp variables and defaults 
     #
     adb_tmpfile="$(mktemp -tu 2>/dev/null)"
     adb_tmpdir="$(mktemp -p /tmp -d 2>/dev/null)"
+    unset adb_srcfind
+    unset adb_revsrcfind
 
     # set adblock source ruleset definitions
     #
     rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
     rset_end="sed '/^[#/:;_\s]*$/d'"
-    rset_default="${rset_start} | ${rset_end}"
-    rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
+    rset_adaway="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
+    rset_blacklist="${rset_start} | ${rset_end}"
+    rset_disconnect="${rset_start} | ${rset_end}"
+    rset_dshield="${rset_start} | ${rset_end}"
+    rset_feodo="${rset_start} | ${rset_end}"
+    rset_malware="${rset_start} | ${rset_end}"
+    rset_palevo="${rset_start} | ${rset_end}"
     rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
     rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
     rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
+    rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
+    rset_zeus="${rset_start} | ${rset_end}"
 
-    # set adblock/dnsmasq destination file and format
+    # set dnsmasq defaults
     #
-    adb_dnsfile="/tmp/dnsmasq.d/adlist.conf"
+    adb_dnsdir="/tmp/dnsmasq.d"
     adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
+    adb_dnsprefix="adb_list"
 }
 
 #############################################
@@ -180,14 +193,25 @@ f_envparse()
 #
 f_envcheck()
 {
-    # check required config variables
+    # check adblock config file
+    #
+    check_config="$(grep -F "ruleset=rset_default" /etc/config/adblock 2>/dev/null)"
+    if [ -n "${check_config}" ]
+    then
+        rc=515
+        grep -Fv "#" "/etc/adblock/samples/adblock.conf.sample" > /etc/config/adblock
+        f_log "new default adblock config applied, please check your configuration settings in /etc/config/adblock" "${rc}"
+        f_deltemp
+    fi
+
+    # check required config options
     #
-    adb_varlist="adb_ip adb_dev adb_if adb_domain adb_minspace adb_maxloop adb_maxtime adb_blacklist adb_whitelist"
+    adb_varlist="adb_ip adb_dev adb_domain"
     for var in ${adb_varlist}
     do
         if [ -z "$(eval printf \"\$"${var}"\")" ]
         then
-            rc=525
+            rc=520
             f_log "missing adblock config option (${var})" "${rc}"
             f_deltemp
         fi
@@ -195,12 +219,12 @@ f_envcheck()
 
     # check main uhttpd configuration
     #
-    check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -Fo "0.0.0.0")"
+    check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -Fo "0.0.0.0" 2>/dev/null)"
     if [ -n "${check_uhttpd}" ]
     then
-        rc=530
+        rc=525
         lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)"
-        f_log "main uhttpd instance listens to all network interfaces, please bind uhttpd to LAN only (${lan_ip})" "${rc}"
+        f_log "please bind main uhttpd instance to LAN only (lan ip: ${lan_ip})" "${rc}"
         f_deltemp
     fi
 
@@ -208,30 +232,36 @@ f_envcheck()
     #
     if [ ! -d "/sys/class/net/${adb_dev}" ]
     then
-        rc=535
+        rc=530
         f_log "invalid adblock network device input (${adb_dev})" "${rc}"
         f_deltemp
     fi
 
     # check adblock network interface configuration
     #
-    check_if="$(printf "${adb_if}" | sed -n '/[^._0-9A-Za-z]/p')"
-    banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
+    check_if="$(printf "${adb_if}" | sed -n '/[^._0-9A-Za-z]/p' 2>/dev/null)"
+    banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p' 2>/dev/null)"
     if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
     then
-        rc=540
+        rc=535
         f_log "invalid adblock network interface input (${adb_if})" "${rc}"
         f_deltemp
     fi
 
     # check adblock ip address configuration
     #
-    check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
+    check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' 2>/dev/null)"
+    lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)"
     if [ -z "${check_ip}" ]
     then
-        rc=545
+        rc=540
         f_log "invalid adblock ip address input (${adb_ip})" "${rc}"
         f_deltemp
+    elif [ "${adb_ip}" = "${lan_ip}" ]
+    then
+        rc=545
+        f_log "adblock ip needs to be a different subnet from the normal LAN (adblock ip: ${adb_ip})" "${rc}"
+        f_deltemp
     fi
 
     # check adblock blacklist/whitelist configuration
@@ -263,7 +293,7 @@ f_envcheck()
 
     # check curl package dependency
     #
-    check="$(printf "${pkg_list}" | grep "^curl -")"
+    check="$(printf "${pkg_list}" | grep "^curl -" 2>/dev/null)"
     if [ -z "${check}" ]
     then
         rc=565
@@ -273,7 +303,7 @@ f_envcheck()
 
     # check wget package dependency
     #
-    check="$(printf "${pkg_list}" | grep "^wget -")"
+    check="$(printf "${pkg_list}" | grep "^wget -" 2>/dev/null)"
     if [ -z "${check}" ]
     then
         rc=570
@@ -281,31 +311,32 @@ f_envcheck()
         f_deltemp
     fi
 
-    # check ca-certificates package and set wget/curl parms accordingly
+    # check ca-certificates package and set wget/curl options accordingly
     #
-    check="$(printf "${pkg_list}" | grep "^ca-certificates -")"
+    check="$(printf "${pkg_list}" | grep "^ca-certificates -" 2>/dev/null)"
     if [ -z "${check}" ]
     then
-        curl_parm="-q --insecure"
-        wget_parm="--no-config --no-hsts --no-check-certificate"
+        curl_parm="-q --insecure --silent"
+        wget_parm="--no-config --no-hsts --no-check-certificate --quiet"
     else
-        curl_parm="-q"
-        wget_parm="--no-config --no-hsts"
+        curl_parm="-q --silent"
+        wget_parm="--no-config --no-hsts --quiet"
     fi
 
     # check total and swap memory
     #
-    mem_total="$(cat /proc/meminfo | grep -F "MemTotal" | grep -o "[0-9]*")"
-    mem_free="$(cat /proc/meminfo | grep -F "MemFree" | grep -o "[0-9]*")"
-    swap_total="$(cat /proc/meminfo | grep -F "SwapTotal" | grep -o "[0-9]*")"
+    mem_total="$(grep -F "MemTotal" "/proc/meminfo" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
+    mem_free="$(grep -F "MemFree" "/proc/meminfo" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
+    swap_total="$(grep -F "SwapTotal" "/proc/meminfo" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
     if [ $((mem_total)) -le 64000 ] && [ $((swap_total)) -eq 0 ]
     then
+        adb_unique=0
+        f_log "overall sort/unique processing will be disabled,"
         f_log "please consider adding an external swap device to supersize your /tmp directory (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})"
     fi
 
     # check backup configuration
     #
-    adb_backupdir="${adb_backupfile%/*}"
     if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
     then
         f_space "${adb_backupdir}"
@@ -323,7 +354,7 @@ f_envcheck()
     then
         # check find capabilities
         #
-        check="$(find --help 2>&1 | grep -F "mtime")"
+        check="$(find --help 2>&1 | grep -F "mtime" 2>/dev/null)"
         if [ -z "${check}" ]
         then
             query_ok="false"
@@ -417,15 +448,9 @@ f_envcheck()
         fi
     fi
 
-    # remove no longer used environment variables
+    # remove no longer used package list
     #
-    env_list="$(set | grep -o "CONFIG_[A-Za-z0-9_]*")"
-    for var in ${env_list}
-    do
-        unset "${var}" 2>/dev/null
-    done
-    unset env_list 2>/dev/null
-    unset pkg_list 2>/dev/null
+    unset pkg_list
 }
 
 ################################################
@@ -461,7 +486,7 @@ f_space()
     if [ -d "${mp}" ]
     then
         df "${mp}" 2>/dev/null |\
-        tail -n1 |\
+        tail -n1 2>/dev/null |\
         while read filesystem overall used available scrap
         do
             av_space="${available}"
@@ -516,14 +541,31 @@ f_remove()
         if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
         then
             kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1
-            find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
-            f_log "remove old domain query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping"
+            rc=${?}
+            if [ $((rc)) -eq 0 ]
+            then
+                find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f "{}" \; 2>/dev/null
+                rc=${?}
+            fi
+            if [ $((rc)) -eq 0 ]
+            then
+                f_log "remove old domain query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping"
+            else
+                f_log "error during domain query removal/housekeeping (pid: $(cat "${adb_querypid}"))"
+            fi
             > "${adb_querypid}"
         fi
         if [ ! -s "${adb_querypid}" ]
         then
-            (logread -f 2>/dev/null & printf ${!} > "${adb_querypid}") | grep -Eo "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" &
-            f_log "new domain query log background process started"
+            (logread -f 2>/dev/null & printf ${!} > "${adb_querypid}") | grep -Eo "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" 2>/dev/null >> "${adb_queryfile}.${query_date}" &
+            rc=${?}
+            if [ $((rc)) -eq 0 ]
+            then
+                sleep 1
+                f_log "new domain query log background process started (pid: $(cat "${adb_querypid}"))"
+            else
+                f_log "error during domain query start"
+            fi
         fi
     fi
     f_deltemp
@@ -534,13 +576,33 @@ f_remove()
 #
 f_restore()
 {
-    if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ]
+    # remove bogus adblocklists
+    #
+    if [ -n "${adb_revsrclist}" ]
     then
-        cp -f "${adb_backupfile}" "${adb_dnsfile}" >/dev/null 2>&1
-        f_log "adblocklist backup restored"
-    else
-        > "${adb_dnsfile}"
-        f_log="empty adblocklist generated"
+        find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec rm -f "{}" \; 2>/dev/null
+        if [ $((rc)) -eq 0 ]
+        then
+            f_log "bogus adblocklists removed"
+        else
+            f_log "error during removal of bogus adblocklists" "${rc}"
+            f_remove
+        fi
+    fi
+
+    # restore backups
+    #
+    if [ "${backup_ok}" = "true" ] && [ -d "${adb_backupdir}" ] && [ "$(printf "${adb_backupdir}/${adb_dnsprefix}."*)" != "${adb_backupdir}/${adb_dnsprefix}.*" ]
+    then
+        cp -f "${adb_backupdir}/${adb_dnsprefix}."* "${adb_dnsdir}" >/dev/null 2>&1
+        rc=${?}
+        if [ $((rc)) -eq 0 ]
+        then
+            f_log "all available backups restored"
+        else
+            f_log "error during restore" "${rc}"
+            f_remove
+        fi
     fi
     /etc/init.d/dnsmasq restart >/dev/null 2>&1
     f_remove
@@ -616,55 +678,3 @@ f_ntpcheck()
         f_restore
     fi
 }
-
-####################################################################
-# f_dnscheck: dnsmasq health check with newly generated adblock list
-#
-f_dnscheck()
-{
-    local dns_status
-    dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
-    rc=${?}
-    if [ -z "${dns_status}" ]
-    then
-        dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep -F "${adb_ip}")"
-        rc=${?}
-        if [ -z "${dns_status}" ]
-        then
-            if [ "${backup_ok}" = "true" ]
-            then
-                cp -f "${adb_dnsfile}" "${adb_backupfile}" >/dev/null 2>&1
-                f_log "new adblock list with ${adb_count} domains loaded, backup generated"
-            else
-                f_log "new adblock list with ${adb_count} domains loaded, no backup"
-            fi
-        else
-            f_log "nslookup probe failed" "${rc}"
-            f_restore
-        fi
-    else
-        f_log "dnsmasq probe failed" "${rc}"
-        f_restore
-    fi
-}
-
-##########################################################
-# f_footer: write footer with a few statistics to dns file
-#
-f_footer()
-{
-    local url
-    adb_count="$(wc -l < "${adb_dnsfile}")"
-    printf "%s\n" "####################################################" >> "${adb_dnsfile}"
-    printf "%s\n" "# last adblock list update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}"
-    printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}"
-    printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}"
-    for src in ${adb_sources}
-    do
-        url="${src//\&ruleset=*/}"
-        printf "%s\n" "# ${url}" >> "${adb_dnsfile}"
-    done
-    printf "%s\n" "#####" >> "${adb_dnsfile}"
-    printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}"
-    printf "%s\n" "####################################################" >> "${adb_dnsfile}"
-}

net/adblock/files/adblock-update.sh

@@ -25,7 +25,7 @@
 
 # set script version
 #
-adb_version="0.22.2"
+adb_version="0.40.0"
 
 # get current pid, script directory and openwrt version
 #
@@ -37,9 +37,9 @@ openwrt_version="$(cat /etc/openwrt_version 2>/dev/null)"
 #
 if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
 then
-    . "${adb_scriptdir}/adblock-helper.sh"
+    . "${adb_scriptdir}/adblock-helper.sh" 2>/dev/null
 else
-    rc=500
+    rc=600
     /usr/bin/logger -s -t "adblock[${pid}] error" "adblock function library not found, rc: ${rc}"
     exit ${rc}
 fi
@@ -50,7 +50,7 @@ fi
 
 # call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM)
 #
-trap "f_log 'trap error' '600'; f_restore" 1 2 3 10 11 15
+trap "f_log 'trap error' '700'; f_restore" 1 2 3 10 11 15
 
 # start logging
 #
@@ -72,116 +72,307 @@ f_envcheck
 #
 if [ -n "${adb_arc_shalla}" ]
 then
-    # download shallalist archive
+    # start shallalist processing
     #
-    f_log "shallalist (pre-)processing started ..."
     shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
     shalla_file="${adb_tmpdir}/shallalist.txt"
-    curl ${curl_parm} --max-time "${adb_maxtime}" "${adb_arc_shalla}" --output "${shalla_archive}" 2>/dev/null
-    rc=${?}
-    if [ $((rc)) -ne 0 ]
-    then
-        f_log "shallalist archive download failed (${adb_arc_shalla})" "${rc}"
-        f_restore
-    fi
+    src_name="shalla"
+    adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
+    list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)"
+    list_time="${list_time/*: /}"
 
-    # extract and merge only domains of selected shallalist categories
+    # only process shallalist archive with updated timestamp
     #
-    > "${shalla_file}"
-    for category in ${adb_cat_shalla}
-    do
-        tar -xOzf "${shalla_archive}" BL/${category}/domains 2>/dev/null >> "${shalla_file}"
+    shalla_time="$(curl ${curl_parm} --max-time 5 --head "${adb_arc_shalla}" 2>/dev/null | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
+    shalla_time="${shalla_time/*: /}"
+    if [ -z "${shalla_time}" ]
+    then
+        shalla_time="$(date)"
+        f_log "no online timestamp received, current date will be used (${src_name})"
+    fi
+    if [ -z "${list_time}" ] || [ "${list_time}" != "${shalla_time}" ]
+    then
+        f_log "shallalist (pre-)processing started ..."
+        curl ${curl_parm} --max-time "${adb_maxtime}" "${adb_arc_shalla}" --output "${shalla_archive}" 2>/dev/null
         rc=${?}
         if [ $((rc)) -ne 0 ]
         then
-            f_log "shallalist archive extraction failed (${category})" "${rc}"
+            f_log "source download failed (${src_name})" "${rc}"
             f_restore
         fi
-    done
 
-    # finish shallalist (pre-)processing
-    #
-    rm -f "${shalla_archive}" >/dev/null 2>&1
-    rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1 
+        # extract and merge only domains of selected shallalist categories
+        #
+        > "${shalla_file}"
+        for category in ${adb_cat_shalla}
+        do
+            tar -xOzf "${shalla_archive}" BL/${category}/domains 2>/dev/null >> "${shalla_file}"
+            rc=${?}
+            if [ $((rc)) -ne 0 ]
+            then
+                f_log "source archive extraction failed (${category})" "${rc}"
+                f_restore
+            fi
+        done
+
+        # remove temporary files
+        #
+        rm -f "${shalla_archive}" >/dev/null 2>&1
+        rm -rf "${adb_tmpdir}/BL" >/dev/null 2>&1 
+    fi
     adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla"
-    f_log "shallalist (pre-)processing finished (${adb_cat_shalla# })"
+fi
+
+# add blacklist source to active adblock domain sources
+#
+if [ -n "${adb_sources}" ] && [ -s "${adb_blacklist}" ]
+then
+    adb_sources="${adb_sources} file://${adb_blacklist}&ruleset=rset_blacklist"
 fi
 
 # loop through active adblock domain sources,
 # prepare output and store all extracted domains in temp file
 #
-adb_sources="${adb_sources} file://${adb_blacklist}&ruleset=rset_default"
 for src in ${adb_sources}
 do
     # download selected adblock sources
     #
-    url="${src//\&ruleset=*/}"
-    check_url="$(printf "${url}" | sed -n '/^https:/p')"
+    url="${src/\&ruleset=*/}"
+    check_url="$(printf "${url}" | sed -n '/^https:/p' 2>/dev/null)"
+    src_name="${src/*\&ruleset=rset_/}"
+    adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
+    list_time="$(grep -F "# last modified: " "${adb_dnsfile}" 2>/dev/null)"
+    list_time="${list_time/*: /}"
+
+    # prepare find statement for all active adblocklist sources
+    #
+    if [ -z "${adb_srcfind}" ]
+    then
+        adb_srcfind="! -name ${adb_dnsprefix}.${src_name}"
+    else
+        adb_srcfind="${adb_srcfind} -a ! -name ${adb_dnsprefix}.${src_name}"
+    fi
+
+    # wget/curl switch
+    # only download blocklist with newer/updated timestamp
+    #
     if [ -n "${check_url}" ]
     then
-        tmp_var="$(wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
-        rc=${?}
+        url_time="$(wget ${wget_parm} --timeout=5 --server-response --spider "${url}" 2>&1 | grep -F "Last-Modified: " 2>/dev/null | tr -d '\r' 2>/dev/null)"
+        url_time="${url_time/*: /}"
+        if [ -z "${url_time}" ]
+        then
+            url_time="$(date)"
+            f_log "no online timestamp received, current date will be used (${src_name})"
+        fi
+        if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
+        then
+            tmp_domains="$(wget ${wget_parm} --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
+            rc=${?}
+        else
+            f_log "source doesn't change, no update required (${src_name})"
+            continue
+        fi
     else
-        tmp_var="$(curl ${curl_parm} --max-time "${adb_maxtime}" "${url}" 2>/dev/null)"
-        rc=${?}
+        if [ "${src_name}" = "shalla" ]
+        then
+            url_time="${shalla_time}"
+        else
+            url_time="$(curl ${curl_parm} --max-time 5 --head "${url}" 2>/dev/null | grep -F "Last-Modified: " | tr -d '\r')"
+            url_time="${url_time/*: /}"
+        fi
+        if [ -z "${url_time}" ]
+        then
+            url_time="$(date)"
+            f_log "no online timestamp received, current date will be used (${src_name})"
+        fi
+        if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ]
+        then
+            tmp_domains="$(curl ${curl_parm} --max-time "${adb_maxtime}" "${url}" 2>/dev/null)"
+            rc=${?}
+        else
+            f_log "source doesn't change, no update required (${src_name})"
+            continue
+        fi
     fi
 
     # check download result and prepare domain output by regex patterns
     #
-    if [ $((rc)) -eq 0 ] && [ -n "${tmp_var}" ]
+    if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ]
     then
         eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')"
-        tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[A-Z]' '[a-z]')"
-        count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)"
+        count="$(printf "%s\n" "${tmp_domains}" | tr '[A-Z]' '[a-z]' | eval "${ruleset}" | tee "${adb_tmpfile}" | wc -l)"
         f_log "source download finished (${url}, ${count} entries)"
-        if [ "${url}" = "file:///${shalla_file}" ]
+        if [ "${src_name}" = "shalla" ]
         then
             rm -f "${shalla_file}" >/dev/null 2>&1
         fi
-        unset tmp_var 2>/dev/null
-    elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ]
+        unset tmp_domains
+    elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
     then
-        f_log "empty source download finished (${url})"
+        f_log "empty source download finished (${src_name})"
+        continue
     else
-        f_log "source download failed (${url})" "${rc}"
+        f_log "source download failed (${src_name})" "${rc}"
         f_restore
     fi
+
+    # remove whitelist domains, sort domains and make them unique,
+    # finally rewrite ad/abuse domain information to separate dnsmasq files
+    #
+    if [ $((count)) -gt 0 ] && [ -n "${adb_tmpfile}" ]
+    then
+        if [ -s "${adb_whitelist}" ]
+        then
+            grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort 2>/dev/null | uniq -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
+            rc=${?}
+        else
+            sort "${adb_tmpfile}" 2>/dev/null | uniq -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
+            rc=${?}
+        fi
+
+        # prepare find statement for revised adblocklist sources
+        #
+        if [ -z "${adb_revsrcfind}" ]
+        then
+            adb_revsrcfind="-name ${adb_dnsprefix}.${src_name}"
+        else
+            adb_revsrcfind="${adb_revsrcfind} -o -name ${adb_dnsprefix}.${src_name}"
+        fi
+
+        # write preliminary adblocklist footer
+        #
+        if [ $((rc)) -eq 0 ]
+        then
+            count="$(wc -l < "${adb_dnsdir}/${adb_dnsprefix}.${src_name}")"
+            printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsfile}"
+            printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsfile}"
+            printf "%s\n" "# source: ${url}" >> "${adb_dnsfile}"
+            printf "%s\n" "# last modified: ${url_time}" >> "${adb_dnsfile}"
+            f_log "domain merging finished (${src_name})"
+        else
+            f_log "domain merging failed (${src_name})" "${rc}"
+            f_restore
+        fi
+    else
+        f_log "empty domain input received (${src_name})"
+        continue
+    fi
 done
 
-# remove whitelist domains, sort domains and make them unique
-# and finally rewrite ad/abuse domain information to dnsmasq file
+# remove old adblocklists and their backups
 #
-if [ -s "${adb_whitelist}" ]
+if [ -n "${adb_srcfind}" ]
 then
-    grep -Fvxf "${adb_whitelist}" "${adb_tmpfile}" 2>/dev/null | sort -u 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
-    rc=${?}
+    find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srcfind} \) -exec rm -f "{}" \; 2>/dev/null
+    if [ $((rc)) -ne 0 ]
+    then
+        f_log "error during removal of old adblocklists" "${rc}"
+        f_remove
+    fi
+    if [ "${backup_ok}" = "true" ]
+    then
+        find "${adb_backupdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}.*" \( ${adb_srcfind} \) -exec rm -f "{}" \; 2>/dev/null
+        if [ $((rc)) -ne 0 ]
+        then
+            f_log "error during removal of old backups" "${rc}"
+            f_remove
+        fi
+    fi
 else
-    sort -u "${adb_tmpfile}" 2>/dev/null | eval "${adb_dnsformat}" 2>/dev/null > "${adb_dnsfile}"
-    rc=${?}
+    rm -f "${adb_dnsdir}/${adb_dnsprefix}."* >/dev/null 2>&1
+    if [ "${backup_ok}" = "true" ]
+    then
+        rm -f "${adb_backupdir}/${adb_dnsprefix}."* >/dev/null 2>&1
+        f_log "all available adblocklists and backups removed"
+    else
+        f_log "all available adblocklists removed"
+    fi
 fi
 
-if [ $((rc)) -eq 0 ]
+# make separate adblocklists unique
+#
+if [ $((adb_unique)) -eq 1 ]
 then
-    rm -f "${adb_tmpfile}" >/dev/null 2>&1
-    f_log "domain merging finished"
-else
-    f_log "domain merging failed" "${rc}"
-    f_restore
-fi
+    if [ -n "${adb_revsrcfind}" ]
+    then
+        f_log "remove duplicates in separate adblocklists ..."
 
-# write dns file footer
-#
-f_footer
+        # generate a temporary, unique overall list
+        #
+        head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | sort -u 2>/dev/null > "${adb_dnsdir}/tmp.overall"
+
+        # loop through all separate lists, ordered by size (ascending)
+        #
+        for list in $(ls -Sr "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null)
+        do
+            # check separate lists vs. overall list,
+            # rewrite only duplicate entries back to separate lists
+            #
+            list="${list/*./}"
+            sort "${adb_dnsdir}/tmp.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" 2>/dev/null | uniq -d 2>/dev/null > "${adb_dnsdir}/tmp.${list}"
 
-# restart dnsmasq with newly generated block list
+            # remove these entries from overall list,
+            # rewrite only unique entries back to overall list
+            #
+            tmp_unique="$(sort "${adb_dnsdir}/tmp.overall" "${adb_dnsdir}/tmp.${list}" 2>/dev/null | uniq -u 2>/dev/null)"
+            printf "%s\n" "${tmp_unique}" > "${adb_dnsdir}/tmp.overall"
+
+            # write final adblocklist footer
+            #
+            count="$(wc -l < "${adb_dnsdir}/tmp.${list}")"
+            printf "%s\n" "#------------------------------------------------------------------" >> "${adb_dnsdir}/tmp.${list}"
+            printf "%s\n" "# ${0##*/} (${adb_version}) - ${count} ad/abuse domains blocked" >> "${adb_dnsdir}/tmp.${list}"
+            tail -qn -2 "${adb_dnsdir}/$adb_dnsprefix.${list}" 2>/dev/null >> "${adb_dnsdir}/tmp.${list}"
+            mv -f "${adb_dnsdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}" >/dev/null 2>&1
+        done
+        rm -f "${adb_dnsdir}/tmp.overall" >/dev/null 2>&1
+    fi
+fi
+
+# restart dnsmasq with newly generated block lists
 #
 /etc/init.d/dnsmasq restart >/dev/null 2>&1
-sleep 2
+sleep 3
 
 # dnsmasq health check
 #
-f_dnscheck
+dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up" 2>/dev/null)"
+if [ -z "${dns_status}" ]
+then
+    dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep -F "${adb_ip}" 2>/dev/null)"
+    if [ -z "${dns_status}" ]
+    then
+        adb_count="$(head -qn -4 "${adb_dnsdir}/${adb_dnsprefix}."* 2>/dev/null | wc -l)"
+        if [ "${backup_ok}" = "true" ]
+        then
+            if [ -n "${adb_revsrcfind}" ]
+            then
+                find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec cp -f "{}" "${adb_backupdir}" \; 2>/dev/null
+                rc=${?}
+                if [ $((rc)) -ne 0 ]
+                then
+                    f_log "error during backup of adblocklists" "${rc}"
+                    f_remove
+                fi
+                f_log "adblocklists with overall ${adb_count} domains loaded, new backups generated"
+            else
+                f_log "adblocklists with overall ${adb_count} domains loaded, no new backups"
+            fi
+        else
+            f_log "adblocklists with overall ${adb_count} domains loaded, backups disabled"
+        fi
+    else
+        rc=605
+        f_log "nslookup probe failed" "${rc}"
+        f_restore
+    fi
+else
+    rc=610
+    f_log "dnsmasq probe failed" "${rc}"
+    f_restore
+fi
 
-# remove files and exit
+# remove temporary files and exit
 #
 f_remove

net/adblock/files/adblock.conf

@@ -1,4 +1,6 @@
-# adblock configuration
+# adblock configuration, for further information
+# please read /etc/adblock/samples/adblock.conf.sample
+# and /etc/adblock/README.md
 #
 config adblock "global"
 	option adb_ip "192.168.2.1"
@@ -15,7 +17,7 @@ config service "ntpcheck"
 
 config service "backup"
 	option enabled "0"
-	option adb_backupfile "/tmp/adlist.backup"
+	option adb_backupdir "/tmp"
 
 config service "debuglog"
 	option enabled "0"
@@ -32,23 +34,23 @@ config source "yoyo"
 
 config source "malware"
 	option enabled "0"
-	option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default"
+	option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
 
 config source "zeus"
 	option enabled "0"
-	option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default"
+	option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus"
 
 config source "feodo"
 	option enabled "0"
-	option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default"
+	option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo"
 
 config source "palevo"
 	option enabled "0"
-	option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default"
+	option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo"
 
 config source "dshield"
 	option enabled "0"
-	option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default"
+	option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield"
 
 config source "spam404"
 	option enabled "0"
@@ -58,6 +60,14 @@ config source "winhelp"
 	option enabled "0"
 	option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
 
+config source "adaway"
+	option enabled "0"
+	option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
+
+config source "disconnect"
+	option enabled "0"
+	option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
+
 config source "shalla"
 	option enabled "0"
 	option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"

net/adblock/files/samples/adblock.conf.sample

@@ -1,4 +1,6 @@
-# adblock configuration sample
+# adblock configuration, for further information
+# please read /etc/adblock/samples/adblock.conf.sample
+# and /etc/adblock/README.md
 #
 
 # generic options (always required)
@@ -7,15 +9,17 @@ config adblock "global"
 	# ip address of the local adblock interface/uhttpd instance,
 	# needs to be a different subnet from the normal LAN
 	option adb_ip "192.168.2.1"
-	
+
 	# name of an "always accessible" domain,
 	# this domain will be used for the final nslookup check
 	option adb_domain "heise.de"
-	
+
 	# full path to static domain blacklist file (one domain per line)
+	# wildcards or regex expressions are not allowed
 	option adb_blacklist "/etc/adblock/adblock.blacklist"
 
 	# full path to static domain whitelist file (one domain per line)
+	# wildcards or regex expressions are not allowed
 	option adb_whitelist "/etc/adblock/adblock.whitelist"
 
 # list of wan devices that are allowed for adblock updates (check /sys/class/net/<dev>),
@@ -32,11 +36,11 @@ config service "wancheck"
 config service "ntpcheck"
 	option enabled "0"
 
-# full path to backup file for adlist backups
+# full path to backup directory for adlist backups
 # disabled by default
 config service "backup"
 	option enabled "0"
-	option adb_backupfile "/tmp/adlist.backup"
+	option adb_backupdir "/tmp"
 
 # full path to debug logfile
 # by default adblock logs to syslog and stdout only
@@ -65,23 +69,23 @@ config source "yoyo"
 
 config source "malware"
 	option enabled "0"
-	option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_default"
+	option adb_src_malware "http://mirror1.malwaredomains.com/files/justdomains&ruleset=rset_malware"
 
 config source "zeus"
 	option enabled "0"
-	option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_default"
+	option adb_src_zeus "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist&ruleset=rset_zeus"
 
 config source "feodo"
 	option enabled "0"
-	option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_default"
+	option adb_src_feodo "https://feodotracker.abuse.ch/blocklist/?download=domainblocklist&ruleset=rset_feodo"
 
 config source "palevo"
 	option enabled "0"
-	option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_default"
+	option adb_src_palevo "https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist&ruleset=rset_palevo"
 
 config source "dshield"
 	option enabled "0"
-	option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_default"
+	option adb_src_dshield "http://www.dshield.org/feeds/suspiciousdomains_Low.txt&ruleset=rset_dshield"
 
 config source "spam404"
 	option enabled "0"
@@ -91,12 +95,19 @@ config source "winhelp"
 	option enabled "0"
 	option adb_src_winhelp "http://winhelp2002.mvps.org/hosts.txt&ruleset=rset_winhelp"
 
+config source "adaway"
+	option enabled "0"
+	option adb_src_adaway "https://adaway.org/hosts.txt&ruleset=rset_adaway"
+
+config source "disconnect"
+	option enabled "0"
+	option adb_src_disconnect "https://s3.amazonaws.com/lists.disconnect.me/simple_malvertising.txt&ruleset=rset_disconnect"
+
 config source "shalla"
 	option enabled "0"
 	option adb_arc_shalla "http://www.shallalist.de/Downloads/shallalist.tar.gz"
 	list adb_catlist "adv"
 	list adb_catlist "costtraps"
-	list adb_catlist "downloads"
 	list adb_catlist "spyware"
 	list adb_catlist "tracker"
 	list adb_catlist "warez"