adblock: logging & documentation update

- centralized logging via separate function to stdout, syslog and file
- remove dependencies between helper functions
- add two new options "adb_maxtime" and "adb_maxloop"
- add description to every adblock config option (see
adblock.conf.sample)
- update README.md

Signed-off-by: Dirk Brenken <dirk@brenken.org>

net/adblock/Makefile

@@ -7,7 +7,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=adblock
-PKG_VERSION:=0.20.3
+PKG_VERSION:=0.21.0
 PKG_RELEASE:=1
 PKG_LICENSE:=GPL-3.0+
 PKG_MAINTAINER:=Dirk Brenken <dirk@brenken.org>
@@ -24,8 +24,9 @@ endef
 
 define Package/$(PKG_NAME)/description
 powerful adblock script to block ad/abuse domains
-When the dns server on your router receives dns requests, we will sort out queries that ask for the [A] resource records of ad servers
-and return the local ip address of your router and the internal web server delivers a transparent pixel instead.
+When the dns server on your router receives dns requests, we will sort out queries that ask for the [A]
+resource records of ad servers and return the local ip address of your router and the internal web server
+delivers a transparent pixel instead.
 The script supports the following domain blacklist sites:
   http://pgl.yoyo.org/adservers
   http://malwaredomains.com

net/adblock/files/README.md

@@ -24,7 +24,7 @@ and return the local ip address of your router and the internal web server deliv
 * separate dynamic adblock network interface
 * separate dynamic uhttpd instance as pixel server
 * optional: quality checks and a powerful backup/restore handling to ensure a reliable dnsmasq service
-* optional: adblock updates only on pre-defined wan interfaces
+* optional: adblock updates only on pre-defined wan interfaces (useful for (mobile) multiwan setups)
 * optional: domain query logging as a background service to easily identify free and already blocked domains
 * optional: ntp time sync
 * optional: status & error logging (req. ntp time sync)
@@ -36,20 +36,87 @@ and return the local ip address of your router and the internal web server deliv
     * wget (due to an openwrt bug still needed for certain https requests - see ticket #19621)
     * busybox find with *-mtime* support for logfile housekeeping (enabled by default with r47362, will be disabled if not found)
 * optional: mounted usb stick or any other storage device to overcome limited memory resources on embedded router devices
-* the above dependencies will be checked during package installation & script runtime, please check *logread -e "adblock"* for errors
+* the above dependencies will be checked during package installation & script startup, please check console output or *logread -e "adblock"* for errors
 
 ## Usage
 * select & install adblock package (*opkg install adblock*)
-* configure /etc/config/adblock to your needs
-* start /usr/bin/adblock-update.sh and check *logread -e "adblock"* for errors
+* configure /etc/config/adblock to your needs, see additional comments in *adblock.conf.sample*
+* by default openwrt uhttpd config is bind to 0.0.0.0 (to all ports of your router). For a working adblock configuration you have to bind uHTTPd to the standard LAN port only, please change *listen_http* and *listen_https* accordingly
+* start /usr/bin/adblock-update.sh and check console output or *logread -e "adblock"* for errors
 
 ## Distributed samples
 * all sample configuration files stored in */etc/adblock/samples*.
 * to enable/disable additional domain query logging set the dnsmasq option *logqueries* accordingly, see *dhcp.config.sample*.
+* to bind uhttpd to standard LAN port only, see *uhttpd.config.sample*
 * for script autostart by rc.local and /tmp resizing on the fly see *rc.local.sample*.
 * for scheduled call of *adblock-update.sh* see *root.crontab.sample*.
 * to redirect/force all dns queries to your router see *firwall.user.sample*.
 * for further dnsmasq tweaks see *dnsmasq.conf.sample*.
 
+## Examples
+
+  stdout excerpt for successful adblock run:  
+    
+    root@pi2wrt:~# /usr/bin/adblock-update.sh  
+    adblock[17771] info : domain adblock processing started (0.21.0)  
+    adblock[17771] info : get wan/update interface (wlan1), after 0 loops  
+    adblock[17771] info : get ntp time sync (0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org 3.pool.ntp.org), after 0 loops  
+    adblock[17771] info : shallalist archive download finished  
+    adblock[17771] info : shallalist archive extraction finished  
+    adblock[17771] info : shallalist (pre-)processing finished (adv costtraps downloads spyware tracker warez)  
+    adblock[17771] info : source download finished (http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext, 2426 entries)  
+    adblock[17771] info : source download finished (http://mirror1.malwaredomains.com/files/justdomains, 15275 entries)  
+    adblock[17771] info : source download finished (https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist, 3 entries)  
+    adblock[17771] info : source download finished (https://feodotracker.abuse.ch/blocklist/?download=domainblocklist, 0 entries)  
+    adblock[17771] info : source download finished (https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist, 11 entries)  
+    adblock[17771] info : source download finished (http://www.dshield.org/feeds/suspiciousdomains_Low.txt, 4542 entries)  
+    adblock[17771] info : source download finished (http://spam404bl.com/spam404scamlist.txt, 5193 entries)  
+    adblock[17771] info : source download finished (http://winhelp2002.mvps.org/hosts.txt, 13852 entries)  
+    adblock[17771] info : source download finished (file:////tmp/tmp.emlDeH/shallalist.txt, 36961 entries)  
+    adblock[17771] info : source download finished (file:///etc/adblock/adblock.blacklist, 1 entries)  
+    adblock[17771] info : new adblock list with 73090 domains loaded, backup generated  
+    adblock[17771] info : domain adblock processing finished (0.21.0)  
+    
+
+  generated domain blocklist for dnsmasq:  
+    
+    address=/0-29.com/192.168.2.1  
+    address=/0-2u.com/192.168.2.1  
+    address=/0.r.msn.com/192.168.2.1  
+    address=/00.devoid.us/192.168.2.1  
+    address=/000007.ru/192.168.2.1  
+    [...]  
+    address=/zzz.cn/192.168.2.1  
+    address=/zzzjsh.com/192.168.2.1  
+    ####################################################  
+    # last adblock list update: 20.11.2015 - 18:00:02  
+    # adblock-update.sh (0.21.0) - 73087 ad/abuse domains blocked  
+    # domain blacklist sources:  
+    # http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext  
+    # http://mirror1.malwaredomains.com/files/justdomains  
+    # https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist  
+    # https://feodotracker.abuse.ch/blocklist/?download=domainblocklist  
+    # https://palevotracker.abuse.ch/blocklists.php?download=domainblocklist  
+    # http://www.dshield.org/feeds/suspiciousdomains_Low.txt  
+    # http://spam404bl.com/spam404scamlist.txt  
+    # http://winhelp2002.mvps.org/hosts.txt  
+    # file:////tmp/tmp.CLBLNF/shallalist.txt  
+    # file:///etc/adblock/adblock.blacklist  
+    #####  
+    # /etc/adblock/adblock.whitelist  
+    ####################################################  
+    
+
+  domain query log excerpt:  
+    
+    query[A] www.seenby.de from fe80::6257:18ff:fe6b:4667  
+    query[A] tarifrechner.heise.de from 192.168.1.131  
+    query[A] www.mittelstandswiki.de from fe80::6257:18ff:fe6b:4667  
+    query[A] ad.doubleclick.net from 192.168.1.131  
+    ad.doubleclick.net is 192.168.2.1  
+    
+
+The first three queries are OK (not blocked), the last one has been blocked and answered by local dnsmasq instance.
+
 Have fun!  
 Dirk  

net/adblock/files/adblock-helper.sh

@@ -15,9 +15,9 @@ f_envload()
     then
         . /lib/functions.sh
     else
-        /usr/bin/logger -t "adblock[${pid}]" "error: openwrt function library not found"
+        rc=510
+        f_log "openwrt function library not found" "${rc}"
         f_deltemp
-        exit 10
     fi
 
     # source in openwrt json helpers library
@@ -26,9 +26,9 @@ f_envload()
     then
         . "/usr/share/libubox/jshn.sh"
     else
-        /usr/bin/logger -t "adblock[${pid}]" "error: openwrt json helpers library not found"
+        rc=515
+        f_log "openwrt json helpers library not found" "${rc}"
         f_deltemp
-        exit 15
     fi
 
     # get list with all installed openwrt packages
@@ -36,9 +36,9 @@ f_envload()
     pkg_list="$(opkg list-installed 2>/dev/null)"
     if [ -z "${pkg_list}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: empty openwrt package list"
+        rc=520
+        f_log "empty openwrt package list" "${rc}"
         f_deltemp
-        exit 20
     fi
 }
 
@@ -133,11 +133,8 @@ f_envparse()
 
     # set temp variables and counter
     #
-    adb_tmpfile="$(mktemp -tu)"
-    adb_tmpdir="$(mktemp -d)"
-    cnt=0
-    max_cnt=30
-    max_time=60
+    adb_tmpfile="$(mktemp -tu 2>/dev/null)"
+    adb_tmpdir="$(mktemp -d 2>/dev/null)"
 
     # set adblock source ruleset definitions
     #
@@ -160,13 +157,26 @@ f_envparse()
 #
 f_envcheck()
 {
+    # check required config variables
+    #
+    adb_varlist="adb_ip adb_dev adb_if adb_domain adb_minspace adb_maxloop adb_maxtime adb_blacklist adb_whitelist"
+    for var in ${adb_varlist}
+    do
+        if [ -z "$(eval printf \"\$"${var}"\")" ]
+        then
+            rc=525
+            f_log "missing adblock config option (${var})" "${rc}"
+            f_deltemp
+        fi
+    done
+
     # check adblock network device configuration
     #
     if [ ! -d "/sys/class/net/${adb_dev}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network device input (${adb_dev})"
+        rc=530
+        f_log "invalid adblock network device input (${adb_dev})" "${rc}"
         f_deltemp
-        exit 25
     fi
 
     # check adblock network interface configuration
@@ -175,9 +185,9 @@ f_envcheck()
     banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
     if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network interface input (${adb_if})"
+        rc=535
+        f_log "invalid adblock network interface input (${adb_if})" "${rc}"
         f_deltemp
-        exit 30
     fi
 
     # check adblock ip address configuration
@@ -185,43 +195,56 @@ f_envcheck()
     check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
     if [ -z "${check_ip}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock ip address input (${adb_ip})"
+        rc=540
+        f_log "invalid adblock ip address input (${adb_ip})" "${rc}"
         f_deltemp
-        exit 35
     fi
 
     # check adblock blacklist/whitelist configuration
     #
     if [ ! -r "${adb_blacklist}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: adblock blacklist not found"
+        rc=545
+        f_log "adblock blacklist not found" "${rc}"
         f_deltemp
-        exit 40
     elif [ ! -r "${adb_whitelist}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: adblock whitelist not found"
+        rc=550
+        f_log "adblock whitelist not found" "${rc}"
         f_deltemp
-        exit 45
     fi
 
-    # check wan update configuration
+    # check adblock temp directory
     #
-    if [ -n "${adb_wandev}" ]
+    if [ -n "${adb_tmpdir}" ] && [ -d "${adb_tmpdir}" ]
     then
-        wan_ok="true"
+        f_space "${adb_tmpdir}"
+        tmp_ok="true"
     else
-        wan_ok="false"
-        /usr/bin/logger -t "adblock[${pid}]" "info: wan update check will be disabled"
+        rc=555
+        tmp_ok="false"
+        f_log "temp directory not found" "${rc}"
+        f_deltemp
     fi
 
-    # check ntp sync configuration
+    # check curl package dependency
     #
-    if [ -n "${adb_ntpsrv}" ]
+    check="$(printf "${pkg_list}" | grep "^curl")"
+    if [ -z "${check}" ]
     then
-        ntp_ok="true"
-    else
-        ntp_ok="false"
-        /usr/bin/logger -t "adblock[${pid}]" "info: ntp time sync will be disabled"
+        rc=560
+        f_log "curl package not found" "${rc}"
+        f_deltemp
+    fi
+
+    # check wget package dependency
+    #
+    check="$(printf "${pkg_list}" | grep "^wget")"
+    if [ -z "${check}" ]
+    then
+        rc=565
+        f_log "wget package not found" "${rc}"
+        f_deltemp
     fi
 
     # check backup configuration
@@ -229,34 +252,17 @@ f_envcheck()
     adb_backupdir="${adb_backupfile%/*}"
     if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
     then
+        f_space "${adb_backupdir}"
         backup_ok="true"
-        adb_mounts="${adb_backupdir} ${adb_tmpdir}"
     else
         backup_ok="false"
-        /usr/bin/logger -t "adblock[${pid}]" "info: backup/restore will be disabled"
-    fi
-
-    # check error log configuration
-    #
-    adb_logdir="${adb_logfile%/*}"
-    if [ -n "${adb_logfile}" ] && [ "${adb_logfile}" = "/dev/stdout" ]
-    then
-        log_ok="true"
-        adb_logfile="/proc/self/fd/1"
-    elif [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] && [ "${ntp_ok}" = "true" ]
-    then
-        log_ok="true"
-        adb_mounts="${adb_mounts} ${adb_logdir}"
-    else
-        log_ok="false"
-        adb_logfile="/dev/null"
-        /usr/bin/logger -t "adblock[${pid}]" "info: error logging will be disabled"
+        f_log "backup/restore will be disabled"
     fi
 
     # check dns query log configuration
     #
     adb_querydir="${adb_queryfile%/*}"
-    query_pid="/var/run/adb_query.pid"
+    adb_querypid="/var/run/adb_query.pid"
     if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
     then
         # check find capabilities
@@ -265,77 +271,59 @@ f_envcheck()
         if [ -z "${check}" ]
         then
             query_ok="false"
-            /usr/bin/logger -t "adblock[${pid}]" "info: busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
+            f_log "busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
         else
+            f_space "${adb_querydir}"
             query_ok="true"
             query_name="${adb_queryfile##*/}"
             query_ip="${adb_ip//./\\.}"
-            adb_mounts="${adb_mounts} ${adb_querydir}"
         fi
     else
         query_ok="false"
-        if [ -s "${query_pid}" ]
+        f_log "dns query logging will be disabled"
+        if [ -s "${adb_querypid}" ]
         then
-            kill -9 $(cat "${query_pid}") 2>/dev/null
-            > "${query_pid}"
-            /usr/bin/logger -t "adblock[${pid}]" "info: remove old dns query log background process"
+            kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1
+            f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}"))"
+            > "${adb_querypid}"
         fi
-        /usr/bin/logger -t "adblock[${pid}]" "info: dns query logging will be disabled"
     fi
 
-    # check mount points & space requirements
+    # check debug log configuration
     #
-    adb_mounts="${adb_mounts} ${adb_tmpdir}"
-    for mp in ${adb_mounts}
-    do
-        df "${mp}" 2>/dev/null |\
-        tail -n1 |\
-        while read filesystem overall used available scrap
-        do
-            av_space="${available}"
-            if [ $((av_space)) -eq 0 ]
-            then
-                /usr/bin/logger -t "adblock[${pid}]" "error: no space left on device/not mounted (${mp})"
-                exit 50
-            elif [ $((av_space)) -lt $((adb_minspace)) ]
-            then
-                /usr/bin/logger -t "adblock[${pid}]" "error: not enough space left on device (${mp})"
-                exit 55
-            fi
-        done
-        # subshell return code handling
-        #
-        rc=$?
-        if [ $((rc)) -ne 0 ]
-        then
-            f_deltemp
-            exit ${rc}
-        fi
-    done
+    adb_logdir="${adb_logfile%/*}"
+    if [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ]
+    then
+        f_space "${adb_logdir}"
+        log_ok="true"
+    else
+        log_ok="false"
+        f_log "debug logging will be disabled"
+    fi
 
-    # check curl package dependency
+    # check wan update configuration
     #
-    check="$(printf "${pkg_list}" | grep "^curl")"
-    if [ -z "${check}" ]
+    if [ -n "${adb_wandev}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: curl package not found"
-        f_deltemp
-        exit 60
+        f_wancheck "${adb_maxloop}"
+    else
+        wan_ok="false"
+        f_log "wan update check will be disabled"
     fi
 
-    # check wget package dependency
+    # check ntp sync configuration
     #
-    check="$(printf "${pkg_list}" | grep "^wget")"
-    if [ -z "${check}" ]
+    if [ -n "${adb_ntpsrv}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "error: wget package not found"
-        f_deltemp
-        exit 65
+        f_ntpcheck "${adb_maxloop}"
+    else
+        ntp_ok="false"
+        f_log "ntp time sync will be disabled"
     fi
 
     # check dynamic/volatile adblock network interface configuration
     #
-    rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)"
+    rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf ${?})"
     if [ $((rc)) -ne 0 ]
     then
         json_init
@@ -347,97 +335,148 @@ f_envcheck()
         json_close_array
         json_close_object
         ubus call network add_dynamic "$(json_dump)"
-        /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
+        rc=${?}
+        if [ $((rc)) -eq 0 ]
+        then
+            f_log "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
+        else
+            f_log "failed to initialize new dynamic/volatile network interface (${adb_if}, ${adb_ip})" "${rc}"
+            f_deltemp
+        fi
     fi
 
     # check dynamic/volatile adblock uhttpd instance configuration
     #
-    rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf $?)"
+    rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf ${?})"
     if [ $((rc)) -ne 0 ]
     then
-        uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80"
-        /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
+        uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" >/dev/null 2>&1
+        rc=${?}
+        if [ $((rc)) -eq 0 ]
+        then
+            f_log "created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
+        else
+            f_log "failed to initialize new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" "${rc}"
+            f_deltemp
+        fi
+    fi
+}
+
+################################################
+# f_log: log messages to stdout, syslog, logfile
+#
+f_log()
+{
+    local log_msg="${1}"
+    local log_rc="${2}"
+    local class="info "
+    if [ -n "${log_msg}" ]
+    then
+        if [ $((log_rc)) -ne 0 ]
+        then
+            class="error"
+            log_rc=", rc: ${log_rc}"
+        fi
+        /usr/bin/logger -s -t "adblock[${pid}] ${class}" "${log_msg}${log_rc}"
+        if [ "${log_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
+        then
+            printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") adblock[${pid}] ${class}: ${log_msg}${log_rc}" >> "${adb_logfile}"
+        fi
+    fi
+}
+
+################################################
+# f_space: check mount points/space requirements
+#
+f_space()
+{
+    local mp="${1}"
+    if [ -d "${mp}" ]
+    then
+        df "${mp}" 2>/dev/null |\
+        tail -n1 |\
+        while read filesystem overall used available scrap
+        do
+            av_space="${available}"
+            if [ $((av_space)) -eq 0 ]
+            then
+                rc=570
+                f_log "no space left on device/not mounted (${mp})" "${rc}"
+                exit ${rc}
+            elif [ $((av_space)) -lt $((adb_minspace)) ]
+            then
+                rc=575
+                f_log "not enough space left on device (${mp})" "${rc}"
+                exit ${rc}
+            fi
+        done
+        rc=${?}
+        if [ $((rc)) -eq 0 ]
+        then
+            space_ok="true"
+        else
+            space_ok="false"
+            f_deltemp
+        fi
     fi
 }
 
-###################################################
-# f_deltemp: delete temporary files and directories
+####################################################
+# f_deltemp: delete temp files, directories and exit
 #
 f_deltemp()
 {
     if [ -f "${adb_tmpfile}" ]
     then
-       rm -f "${adb_tmpfile}" 2>/dev/null
+       rm -f "${adb_tmpfile}" >/dev/null 2>&1
     fi
     if [ -d "${adb_tmpdir}" ]
     then
-       rm -rf "${adb_tmpdir}" 2>/dev/null
+       rm -rf "${adb_tmpdir}" >/dev/null 2>&1
     fi
+    f_log "domain adblock processing finished (${adb_version})"
+    exit ${rc}
 }
 
-################################################################
-# f_remove: remove temporary files, start and maintain query log
+####################################################
+# f_remove: maintain and (re-)start domain query log
 #
 f_remove()
 {
-    # delete temporary files and directories
-    #
-    f_deltemp
-
-    # remove existing domain query log background process,
-    # do housekeeping and start a new process on daily basis
-    #
     if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
     then
         query_date="$(date "+%Y%m%d")"
-        if [ -s "${query_pid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
+        if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
         then
-            kill -9 $(cat "${query_pid}") 2>/dev/null
-            > "${query_pid}"
-            find "${adb_backupdir}" -maxdepth 1 -type f -mtime +${adb_queryhistory} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
-            /usr/bin/logger -t "adblock[${pid}]" "info: remove old dns query log background process and do logfile housekeeping"
+            kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1
+            find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
+            f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}")) and do logfile housekeeping"
+            > "${adb_querypid}"
         fi
-        if [ ! -s "${query_pid}" ]
+        if [ ! -s "${adb_querypid}" ]
         then
-            ( logread -f 2>/dev/null & printf "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" &
-            /usr/bin/logger -t "adblock[${pid}]" "info: start new domain query log background process"
+            ( logread -f 2>/dev/null & printf ${!} > "${adb_querypid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" &
+            f_log "new domain query log background process started (pid: $(cat "${adb_querypid}"))"
         fi
     fi
-
-    # final log entry
-    #
-    /usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing finished (${adb_version})"
+    f_deltemp
 }
 
-#####################################################
-# f_restore: if available, restore last adlist backup
+################################################################
+# f_restore: restore last adblocklist backup and restart dnsmasq
 #
 f_restore()
 {
-    if [ -z "${restore_msg}" ]
-    then
-        restore_msg="unknown"
-    fi
-
     if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ]
     then
-        cp -f "${adb_backupfile}" "${adb_dnsfile}" 2>/dev/null
-        /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, adlist backup restored"
-        printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${adb_logfile}"
+        cp -f "${adb_backupfile}" "${adb_dnsfile}" >/dev/null 2>&1
+        f_log "adblocklist backup restored"
     else
         > "${adb_dnsfile}"
-        /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, empty adlist generated"
-        printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${adb_logfile}"
+        f_log="empty adblocklist generated"
     fi
-
-    # restart dnsmasq
-    #
     /etc/init.d/dnsmasq restart >/dev/null 2>&1
-
-    # remove files and exit
-    #
     f_remove
-    exit 100
 }
 
 #######################################################
@@ -445,35 +484,35 @@ f_restore()
 #
 f_wancheck()
 {
-    if [ "${wan_ok}" = "true" ]
-    then
-        # wait for wan update interface(s)
-        #
-        while [ $((cnt)) -le $((max_cnt)) ]
+    local cnt=0
+    local cnt_max="${1}"
+    local dev
+    local dev_out
+    while [ $((cnt)) -le $((cnt_max)) ]
+    do
+        for dev in ${adb_wandev}
         do
-            for dev in ${adb_wandev}
-            do
-                if [ -d "/sys/class/net/${dev}" ]
-                then
-                    dev_out="$(cat /sys/class/net/${dev}/operstate 2>/dev/null)"
-                    if [ "${dev_out}" = "up" ]
-                    then
-                        /usr/bin/logger -t "adblock[${pid}]" "info: get wan/update interface: ${dev}, after ${cnt} loops"
-                        break 2
-                    fi
-                fi
-                if [ $((cnt)) -eq $((max_cnt)) ]
+            if [ -d "/sys/class/net/${dev}" ]
+            then
+                dev_out="$(cat /sys/class/net/${dev}/operstate 2>/dev/null)"
+                rc=${?}
+                if [ "${dev_out}" = "up" ]
                 then
-                    wan_ok="false"
-                    /usr/bin/logger -t "adblock[${pid}]" "error: no wan/update interface(s) found (${adb_wandev})"
-                    printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${adb_wandev})" >> "${adb_logfile}"
-                    restore_msg="no wan/update interface(s)"
-                    f_restore
+                    wan_ok="true"
+                    f_log "get wan/update interface (${dev}), after ${cnt} loops"
+                    break 2
                 fi
-            done
-            sleep 1
-            cnt=$((cnt + 1))
+            fi
         done
+        sleep 1
+        cnt=$((cnt + 1))
+    done
+    if [ -z "${wan_ok}" ]
+    then
+        rc=580
+        wan_ok="false"
+        f_log "no wan/update interface(s) found (${adb_wandev# })" "${rc}"
+        f_restore
     fi
 }
 
@@ -482,72 +521,63 @@ f_wancheck()
 #
 f_ntpcheck()
 {
-    if [ "${ntp_ok}" = "true" ]
+    local cnt=0
+    local cnt_max="${1}"
+    local ntp_pool
+    for srv in ${adb_ntpsrv}
+    do
+        ntp_pool="${ntp_pool} -p ${srv}"
+    done
+    while [ $((cnt)) -le $((cnt_max)) ]
+    do
+        /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
+        rc=${?}
+        if [ $((rc)) -eq 0 ]
+        then
+            ntp_ok="true"
+            f_log "get ntp time sync (${adb_ntpsrv# }), after ${cnt} loops"
+            break
+        fi
+        sleep 1
+        cnt=$((cnt + 1))
+    done
+    if [ -z "${ntp_ok}" ]
     then
-        # prepare ntp server pool
-        #
-        unset ntp_pool
-        for srv in ${adb_ntpsrv}
-        do
-            ntp_pool="${ntp_pool} -p ${srv}"
-        done
-
-        # wait for ntp time sync
-        #
-        while [ $((cnt)) -le $((max_cnt)) ]
-        do
-            /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
-            rc=$?
-            if [ $((rc)) -eq 0 ]
-            then
-                /usr/bin/logger -t "adblock[${pid}]" "info: get ntp time sync (${adb_ntpsrv}), after ${cnt} loops"
-                break
-            fi
-            if [ $((cnt)) -eq $((max_cnt)) ]
-            then
-                ntp_ok="false"
-                /usr/bin/logger -t "adblock[${pid}]" "error: ntp time sync failed (${adb_ntpsrv})"
-                printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${adb_ntpsrv})" >> "${adb_logfile}"
-                restore_msg="time sync failed"
-                f_restore
-            fi
-            sleep 1
-            cnt=$((cnt + 1))
-        done
+        rc=585
+        ntp_ok="false"
+        f_log "ntp time sync failed (${adb_ntpsrv# })" "${rc}"
+        f_restore
     fi
 }
 
-#################################################################
-# f_dnscheck: dnsmasq health check with newly generated blocklist
+####################################################################
+# f_dnscheck: dnsmasq health check with newly generated adblock list
 #
 f_dnscheck()
 {
-    # check 1: dnsmasq startup
-    #
+    local dns_status
     dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
+    rc=${?}
     if [ -z "${dns_status}" ]
     then
-        # check 2: nslookup probe
-        #
         dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")"
+        rc=${?}
         if [ -z "${dns_status}" ]
         then
-            # create backup of new block list only, if both checks are OK and backup enabled
-            #
             if [ "${backup_ok}" = "true" ]
             then
-                cp -f "${adb_dnsfile}" "${adb_backupfile}" 2>/dev/null
-                /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, backup generated"
+                cp -f "${adb_dnsfile}" "${adb_backupfile}" >/dev/null 2>&1
+                f_log "new adblock list with ${adb_count} domains loaded, backup generated"
             else
-                /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, no backup"
+                f_log "new adblock list with ${adb_count} domains loaded, no backup"
             fi
         else
-            restore_msg="nslookup probe failed"
+            f_log "nslookup probe failed" "${rc}"
             f_restore
         fi
     else
-            restore_msg="dnsmasq probe failed"
-            f_restore
+        f_log "dnsmasq probe failed" "${rc}"
+        f_restore
     fi
 }
 
@@ -556,9 +586,10 @@ f_dnscheck()
 #
 f_footer()
 {
+    local url
     adb_count="$(wc -l < "${adb_dnsfile}")"
-    printf "%s\n" "###################################################" >> "${adb_dnsfile}"
-    printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}"
+    printf "%s\n" "####################################################" >> "${adb_dnsfile}"
+    printf "%s\n" "# last adblock list update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}"
     printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}"
     printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}"
     for src in ${adb_sources}
@@ -566,8 +597,7 @@ f_footer()
         url="${src//\&ruleset=*/}"
         printf "%s\n" "# ${url}" >> "${adb_dnsfile}"
     done
-    printf "%s\n" "###################################################" >> "${adb_dnsfile}"
-    printf "%s\n" "# domain whitelist source:" >> "${adb_dnsfile}"
+    printf "%s\n" "#####" >> "${adb_dnsfile}"
     printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}"
-    printf "%s\n" "###################################################" >> "${adb_dnsfile}"
+    printf "%s\n" "####################################################" >> "${adb_dnsfile}"
 }

net/adblock/files/adblock-update.sh

@@ -25,11 +25,11 @@
 
 # set script version
 #
-adb_version="0.20.3"
+adb_version="0.21.0"
 
 # get current pid and script directory
 #
-pid=$$
+pid=${$}
 adb_scriptdir="${0%/*}"
 
 # source in adblock function library
@@ -38,8 +38,9 @@ if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
 then
     . "${adb_scriptdir}/adblock-helper.sh"
 else
-    /usr/bin/logger -t "adblock[${pid}]" "error: adblock function library not found"
-    exit 200
+    rc=500
+    /usr/bin/logger -s -t "adblock[${pid}] error" "adblock function library not found, rc: ${rc}"
+    exit ${rc}
 fi
 
 ################
@@ -48,11 +49,11 @@ fi
 
 # call restore function on trap signals (HUP, INT, QUIT, BUS, SEGV, TERM)
 #
-trap "restore_msg='trap error'; f_restore" 1 2 3 10 11 15
+trap "f_log 'trap error' '600'; f_restore" 1 2 3 10 11 15
 
 # start logging
 #
-/usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing started (${adb_version})"
+f_log "domain adblock processing started (${adb_version})"
 
 # load environment
 #
@@ -66,14 +67,6 @@ f_envparse
 #
 f_envcheck
 
-# check ntp time sync
-#
-f_ntpcheck
-
-# check wan update interface(s)
-#
-f_wancheck
-
 # check/start shallalist (pre-)processing
 #
 if [ -n "${adb_arc_shalla}" ]
@@ -82,29 +75,25 @@ then
     #
     shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
     shalla_file="${adb_tmpdir}/shallalist.txt"
-    curl --insecure --max-time "${max_time}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null
-    rc=$?
+    curl --insecure --max-time "${adb_maxtime}" "${adb_arc_shalla}" -o "${shalla_archive}" 2>/dev/null
+    rc=${?}
     if [ $((rc)) -eq 0 ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive download finished"
+        f_log "shallalist archive download finished"
     else
-        /usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive download failed (${adb_arc_shalla})"
-        printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive download failed (${adb_arc_shalla})" >> "${adb_logfile}"
-        restore_msg="archive download failed"
+        f_log "shallalist archive download failed (${adb_arc_shalla})" "${rc}"
         f_restore
     fi
 
     # extract shallalist archive
     #
     tar -xzf "${shalla_archive}" -C "${adb_tmpdir}" 2>/dev/null
-    rc=$?
+    rc=${?}
     if [ $((rc)) -eq 0 ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "info: shallalist archive extraction finished"
+        f_log "shallalist archive extraction finished"
     else
-        /usr/bin/logger -t "adblock[${pid}]" "error: shallalist archive extraction failed"
-        printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist archive extraction failed" >> "${adb_logfile}"
-        restore_msg="archive extraction failed"
+        f_log "shallalist archive extraction failed" "${rc}"
         f_restore
     fi
 
@@ -115,10 +104,10 @@ then
     do
         if [ -f "${adb_tmpdir}/BL/${category}/domains" ]
         then
-            cat "${adb_tmpdir}/BL/${category}/domains" >> "${shalla_file}" 2>/dev/null
-            rc=$?
+            cat "${adb_tmpdir}/BL/${category}/domains" 2>/dev/null >> "${shalla_file}"
+            rc=${?}
         else
-            rc=220
+            rc=505
         fi
         if [ $((rc)) -ne 0 ]
         then
@@ -131,11 +120,9 @@ then
     if [ $((rc)) -eq 0 ]
     then
         adb_sources="${adb_sources} file:///${shalla_file}&ruleset=rset_shalla"
-        /usr/bin/logger -t "adblock[${pid}]" "info: shallalist (pre-)processing finished (${adb_cat_shalla})"
+        f_log "shallalist (pre-)processing finished (${adb_cat_shalla# })"
     else
-        /usr/bin/logger -t "adblock[${pid}]" "error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})"
-        printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: shallalist (pre-)processing failed (${rc}, ${adb_cat_shalla})" >> "${adb_logfile}"
-        restore_msg="shallalist merge failed"
+        f_log "shallalist (pre-)processing failed (${adb_cat_shalla# })" "${rc}"
         f_restore
     fi
 fi
@@ -152,11 +139,11 @@ do
     check_url="$(printf "${url}" | sed -n '/^https:/p')"
     if [ -n "${check_url}" ]
     then
-        tmp_var="$(wget --timeout="${max_time}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
-        rc=$?
+        tmp_var="$(wget --timeout="${adb_maxtime}" --tries=1 --output-document=- "${url}" 2>/dev/null)"
+        rc=${?}
     else
-        tmp_var="$(curl --insecure --max-time "${max_time}" "${url}" 2>/dev/null)"
-        rc=$?
+        tmp_var="$(curl --insecure --max-time "${adb_maxtime}" "${url}" 2>/dev/null)"
+        rc=${?}
     fi
 
     # check download result and prepare domain output by regex patterns
@@ -165,15 +152,13 @@ do
     then
         eval "$(printf "${src}" | sed 's/\(.*\&ruleset=\)/ruleset=\$/g')"
         tmp_var="$(printf "%s\n" "${tmp_var}" | tr '[A-Z]' '[a-z]')"
-        adb_count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)"
-        /usr/bin/logger -t "adblock[${pid}]" "info: source download finished (${url}, ${adb_count} entries)"
+        count="$(printf "%s\n" "${tmp_var}" | eval "${ruleset}" | tee -a "${adb_tmpfile}" | wc -l)"
+        f_log "source download finished (${url}, ${count} entries)"
     elif [ $((rc)) -eq 0 ] && [ -z "${tmp_var}" ]
     then
-        /usr/bin/logger -t "adblock[${pid}]" "info: empty source download finished (${url})"
+        f_log "empty source download finished (${url})"
     else
-        /usr/bin/logger -t "adblock[${pid}]" "error: source download failed (${url})"
-        printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: source download failed (${url})" >> "${adb_logfile}"
-        restore_msg="download failed"
+        f_log "source download failed (${url})" "${rc}"
         f_restore
     fi
 done
@@ -203,4 +188,3 @@ f_dnscheck
 # remove files and exit
 #
 f_remove
-exit 0

net/adblock/files/adblock.conf

@@ -5,7 +5,9 @@ config adblock "global"
 	option adb_dev "eth0"
 	option adb_if "adblock"
 	option adb_domain "heise.de"
-	option adb_minspace "100000"
+	option adb_minspace "20000"
+	option adb_maxloop "5"
+	option adb_maxtime "60"
 	option adb_blacklist "/etc/adblock/adblock.blacklist"
 	option adb_whitelist "/etc/adblock/adblock.whitelist"
 
@@ -24,13 +26,13 @@ config service "backup"
 	option enabled "0"
 	option adb_backupfile "/tmp/adlist.backup"
 
-config service "errorlog"
+config service "debuglog"
 	option enabled "0"
-	option adb_logfile "/tmp/error.log"
+	option adb_logfile "/tmp/adb_debug.log"
 
 config service "querylog"
 	option enabled "0"
-	option adb_queryfile "/tmp/query.log"
+	option adb_queryfile "/tmp/adb_query.log"
 	option adb_queryhistory "1"
 
 config source "yoyo"

net/adblock/files/samples/adblock.conf.sample

@@ -1,18 +1,52 @@
-# adblock configuration
+# adblock configuration sample
+#
+
+# generic options (always required)
 #
 config adblock "global"
+	# ip address of the local adblock interface/uhttpd instance,
+	# needs to be a different subnet from the normal LAN
 	option adb_ip "192.168.2.1"
+	
+	# name of the physical adblock network device (check /sys/class/net/<dev>),
+	# should point to the default lan interface
 	option adb_dev "eth0"
+	
+	# name of the adblock network interface and uhttpd instance
 	option adb_if "adblock"
+	
+	# name of an "always accessible" domain,
+	# this domain will be used for the final nslookup check
 	option adb_domain "heise.de"
-	option adb_minspace "100000"
+	
+	# minimum required space for adlist, backups & logfiles (in kbyte)
+	# if you don't use all adblock sources, you can reduce this value accordingly
+	option adb_minspace "20000"
+	
+	# number of retries for wancheck and ntpcheck (see below)
+	option adb_maxloop "5"
+	
+	# download timeout for every adblock source (in seconds)
+	option adb_maxtime "60"
+	
+	# full path to static domain blacklist file (one domain per line)
 	option adb_blacklist "/etc/adblock/adblock.blacklist"
+
+	# full path to static domain whitelist file (one domain per line)
 	option adb_whitelist "/etc/adblock/adblock.whitelist"
 
+# list of devices that are allowed for adblock updates (check /sys/class/net/<dev>),
+# if no one found the last adlist backup will be used,
+# useful for (mobile) multiwan setups
+# disabled by default
 config service "wancheck"
 	option enabled "0"
 	list adb_wanlist "wan"
 
+# list of ntp time server pools,
+# check that ntp has adjusted the system time on this device,
+# will be used for logfile writing and logfile housekeeping
+# disabled by default
 config service "ntpcheck"
 	option enabled "0"
 	list adb_ntplist "0.pool.ntp.org"
@@ -20,19 +54,33 @@ config service "ntpcheck"
 	list adb_ntplist "2.pool.ntp.org"
 	list adb_ntplist "3.pool.ntp.org"
 
+# full path to backup file for adlist backups
+# disabled by default
 config service "backup"
 	option enabled "0"
 	option adb_backupfile "/tmp/adlist.backup"
 
-config service "errorlog"
+# full path to debug logfile
+# by default adblock logs to syslog and stdout only
+# disabled by default
+config service "debuglog"
 	option enabled "0"
-	option adb_logfile "/tmp/error.log"
+	option adb_logfile "/tmp/adb_debug.log"
 
+# full path to domain query logfile
+# a background task will trace every dns request to file, to easily identify free and already blocked domains,
+# for this to work, you've to enable the dnsmasq option "logqueries" too.
+# the "queryhistory" option deletes query logfiles older than n days (req. busybox find with mtime support)
+# disabled by default
 config service "querylog"
 	option enabled "0"
-	option adb_queryfile "/tmp/query.log"
+	option adb_queryfile "/tmp/adb_query.log"
 	option adb_queryhistory "1"
 
+# different adblock list sources
+# please do not change the urls listed below, 
+# enable/disable sources as needed
+# for shallalist you can also enable/disable different ad categories
 config source "yoyo"
 	option enabled "1"
 	option adb_src_yoyo "http://pgl.yoyo.org/adservers/serverlist.php?hostformat=one-line&showintro=0&mimetype=plaintext&ruleset=rset_yoyo"

net/adblock/files/samples/uhttpd.config.sample

@@ -0,0 +1,3 @@
+    config uhttpd 'main'
+            list listen_http '192.168.1.1:80'
+            list listen_https '192.168.1.1:443'