Merge pull request #14225 from nickberry17/update_watchcat_util

watchcat: add restart_iface and ping size features + refactor

utils/watchcat/Makefile

@@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=watchcat
 PKG_VERSION:=1
-PKG_RELEASE:=8
+PKG_RELEASE:=11
 
 PKG_MAINTAINER:=Roger D <rogerdammit@gmail.com>
 PKG_LICENSE:=GPL-2.0

utils/watchcat/files/initd_watchcat

@@ -5,123 +5,120 @@ START=97
 PIDFILE="/tmp/run/watchcat"
 
 append_string() {
-	local varname="$1"; local add="$2"; local separator="${3:- }"; local actual
+	varname="$1"
+	add="$2"
+	separator="${3:- }"
+	actual
 	eval "actual=\$$varname"
 
 	new="${actual:+$actual$separator}$add"
 	eval "$varname=\$new"
 }
 
-timetoseconds() {
-	local time=$1
-	unset seconds
+time_to_seconds() {
+	time=$1
+
+	{ [ "$time" -ge 1 ] 2>/dev/null && seconds="$time"; } ||
+		{ [ "${time%s}" -ge 1 ] 2>/dev/null && seconds="${time%s}"; } ||
+		{ [ "${time%m}" -ge 1 ] 2>/dev/null && seconds=$((${time%m} * 60)); } ||
+		{ [ "${time%h}" -ge 1 ] 2>/dev/null && seconds=$((${time%h} * 3600)); } ||
+		{ [ "${time%d}" -ge 1 ] 2>/dev/null && seconds=$((${time%d} * 86400)); }
 
-	{ [ "$time" -ge 1 ] 2> /dev/null && seconds="$time"; } || \
-	{ [ "${time%s}" -ge 1 ] 2> /dev/null && seconds="${time%s}"; } || \
-	{ [ "${time%m}" -ge 1 ] 2> /dev/null && seconds=$((${time%m}*60)); } || \
-	{ [ "${time%h}" -ge 1 ] 2> /dev/null && seconds=$((${time%h}*3600)); } || \
-	{ [ "${time%d}" -ge 1 ] 2> /dev/null && seconds=$((${time%d}*86400)); }
+	echo $seconds
+	unset seconds
+	unset time
 }
 
 load_watchcat() {
-	config_get period	$1 period
-	config_get mode		$1 mode
-	config_get pinghosts	$1 pinghosts
-	config_get pingperiod	$1 pingperiod
-	config_get nopingtime	$1 nopingtime
-	config_get forcedelay	$1 forcedelay
-	
-	local nopingtime_dflt="900"
-	local forcedelay_dflt="60"
+	config_get period "$1" period "120"
+	config_get mode "$1" mode "restart_iface"
+	config_get pinghosts "$1" pinghosts "8.8.8.8"
+	config_get pingperiod "$1" pingperiod "60"
+	config_get forcedelay "$1" forcedelay "60"
+	config_get pingsize "$1" pingsize "standard"
+	config_get interface "$1" interface
+	config_get mmifacename "$1" mmifacename
+	config_get unlockbands "$1" unlockbands "0"
+
+	# Fix potential typo in mode and provide backward compatibility.
+	[ "$mode" = "allways" ] && mode="periodic_reboot"
+	[ "$mode" = "always" ] && mode="periodic_reboot"
+	[ "$mode" = "ping" ] && mode="ping_reboot"
 	
-	# Fix potential typo in mode (backward compatibility).
-	[ "$mode" = "allways" ] && mode="always"
-
 	error=""
 	warn=""
-	
-	if [ -z "$period" ] 
-	then
-		append_string "error" "period is not set! Use time value(ex: '30'; '4m'; '6h'; '2d')." "; "	
-	else
-		timetoseconds "$period";period="$seconds"	
-		[ "$period" -ge 1 ] \
-			|| append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; "	
-	fi
-	
-	[ "$mode" = "always" -o "$mode" = "ping" ] \
-		|| append_string "error" "mode must be 'always' or 'ping'" "; "	
-	
-	if [ -z "$forcedelay" ]
-	then
-		forcedelay="$forcedelay_dflt"		
-		append_string "warn" "forcedelay is not configured! Defaulted to $forcedelay seconds" "; "		
-	else
-		[ "$forcedelay" -ge 0 ] || {
-			forcedelay="$forcedelay_dflt"		
-			append_string "warn" "forcedelay is invalid! Defaulted to $forcedelay seconds" "; "		
-		}
+
+	# Checks for settings common to all operation modes
+	if [ "$mode" != "periodic_reboot" ] && [ "$mode" != "ping_reboot" ] && [ "$mode" != "restart_iface" ]; then
+		append_string "error" "mode must be 'periodic_reboot' or 'ping_reboot' or 'restart_iface'" "; "
 	fi
-	
-	[ -z "$error" -a "$mode" = "ping" ] && {			
-		[ -z "$pinghosts" ] \
-			&& append_string "error" "pinghosts must be set in 'ping' mode! Use space separated address list (ex: '8.8.8.8 9.9.9.9')" "; "
-	
-		if [ -z "$nopingtime" ]
-		then
-			nopingtime="$nopingtime_dflt"
-			append_string "warn" "nopingtime is not configured! Defaulted to $nopingtime seconds" "; "			
-		else
-			timetoseconds "$nopingtime";nopingtime="$seconds"
-			[ "$nopingtime" -ge 0 ] || {
-				nopingtime="$nopingtime_dflt"
-				append_string "warn" "nopingtime invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $nopingtime seconds" "; "
-			}
-		fi
-		
-		local pingperiod_dflt="$((period/5))"
-				
-		if [ -z "$pingperiod" ] 
-		then
-			pingperiod="$pingperiod_dflt"
-			append_string "warn" "pingperiod is not configured! Defaulted to $pingperiod seconds(1/5 of period)" "; "
-		else
-			timetoseconds "$pingperiod";pingperiod="$seconds"
-			[ "$pingperiod" -ge 0 -a "$pingperiod" -ge "$period" ] && {
-				pingperiod="$pingperiod_dflt"
-				append_string "warn" "pingperiod is invalid value(greater than period)! Defaulted to $pingperiod seconds(1/5 of period)" "; "
-			}
-			[ "$pingperiod" -ge 0 ] || {
-				pingperiod="$pingperiod_dflt"				
-				append_string "warn" "pingperiod has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $pingperiod seconds(1/5 of period)" "; "
-			}			
+
+	period="$(time_to_seconds "$period")"
+	[ "$period" -ge 1 ] ||
+		append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; "
+
+	# ping_reboot mode and restart_iface mode specific checks
+	if [ "$mode" = "ping_reboot" ] || [ "$mode" = "restart_iface" ]; then
+
+		if [ -z "$error" ]; then
+
+			pingperiod_default="$((period / 5))"
+
+			pingperiod="$(time_to_seconds "$pingperiod")"
+			if [ "$pingperiod" -ge 0 ] && [ "$pingperiod" -ge "$period" ]; then
+				pingperiod="$(time_to_seconds "$pingperiod_default")"
+				append_string "warn" "pingperiod cannot be greater than $period. Defaulted to $pingperiod_default seconds (1/5 of period)" "; "
+			fi
+
+			if [ "$pingperiod" -lt 0 ]; then
+				append_string "warn" "pingperiod cannot be a negative value." "; "
+			fi
+
+			if [ "$mmifacename" != "" ] && [ "$period" -lt 30 ]; then
+				append_string "error" "Check interval is less than 30s. For robust operation with ModemManager modem interfaces it is recommended to set the period to at least 30s."
+			fi
 		fi
-	}
-	
-	[ -n "$warn" ] && logger -p user.warn -t "watchcat" "$1: $warn"
-	[ -n "$error" ] && { logger -p user.err -t "watchcat" "reboot program $1 not started - $error"; return; }
+	fi
 
-	if [ "$mode" = "always" ]
-	then
-		/usr/bin/watchcat.sh "always" "$period" "$forcedelay" &
-		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;forcedelay=$forcedelay)"
-	else
-		/usr/bin/watchcat.sh "ping" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$nopingtime" &
-		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;nopingtime=$nopingtime)"
+	# ping_reboot mode and periodic_reboot mode specific checks
+	if [ "$mode" = "ping_reboot" ] || [ "$mode" = "periodic_reboot" ]; then
+		forcedelay="$(time_to_seconds "$forcedelay")"
 	fi
 
-	echo $! >> "${PIDFILE}.pids"
+	[ -n "$warn" ] && logger -p user.warn -t "watchcat" "$1: $warn"
+	[ -n "$error" ] && {
+		logger -p user.err -t "watchcat" "reboot program $1 not started - $error"
+		return
+	}
+
+	case "$mode" in
+	periodic_reboot)
+		/usr/bin/watchcat.sh "periodic_reboot" "$period" "$forcedelay" &
+		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;forcedelay=$forcedelay)"
+		;;
+	ping_reboot)
+		/usr/bin/watchcat.sh "ping_reboot" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$pingsize" &
+		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;pingsize=$pingsize)"
+		;;
+	restart_iface)
+		/usr/bin/watchcat.sh "restart_iface" "$period" "$pinghosts" "$pingperiod" "$pingsize" "$interface" "$mmifacename" &
+		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;pingsize=$pingsize;interface=$interface;mmifacename=$mmifacename;unlockbands=$unlockbands)"
+		;;
+	*)
+		echo "Error starting Watchcat service. Invalid mode selection: $mode"
+		;;
+	esac
+
+	echo $! >>"${PIDFILE}.pids"
 }
 
 stop() {
-	if [ -f "${PIDFILE}.pids" ]
-	then
+	if [ -f "${PIDFILE}.pids" ]; then
 		logger -p user.info -t "watchcat" "stopping all tasks"
 
-		while read pid
-		do
+		while read pid; do
 			kill -KILL "$pid"
-		done < "${PIDFILE}.pids"
+		done <"${PIDFILE}.pids"
 
 		rm "${PIDFILE}.pids"
 
@@ -135,8 +132,7 @@ start() {
 	[ -f "${PIDFILE}.pids" ] && stop
 
 	config_load system
-	if [ -n "$(uci show system.@watchcat[0])" ] # at least one watchcat section exists
-	then
+	if [ -n "$(uci show system.@watchcat[0])" ]; then # at least one watchcat section exists
 		logger -p user.info -t "watchcat" "starting all tasks"
 		config_foreach load_watchcat watchcat
 		logger -p user.info -t "watchcat" "all tasks started"

utils/watchcat/files/uci_defaults_watchcat

@@ -3,7 +3,7 @@
 uci -q show system.@watchcat[0] || {
 	uci add system watchcat
 	uci set system.@watchcat[0].period=6h
-	uci set system.@watchcat[0].mode=ping
+	uci set system.@watchcat[0].mode=ping_reboot
 	uci set system.@watchcat[0].pinghosts=8.8.8.8
 	uci set system.@watchcat[0].forcedelay=30
 	uci commit

utils/watchcat/files/watchcat.sh

@@ -1,65 +1,223 @@
 #!/bin/sh
 #
 # Copyright (C) 2010 segal.di.ubi.pt
+# Copyright (C) 2020 nbembedded.com
 #
 # This is free software, licensed under the GNU General Public License v2.
 #
 
+get_ping_size() {
+    ps=$1
+    case "$ps" in
+    small)
+        ps="1"
+        ;;
+    windows)
+        ps="32"
+        ;;
+    standard)
+        ps="56"
+        ;;
+    big)
+        ps="248"
+        ;;
+    huge)
+        ps="1492"
+        ;;
+    jumbo)
+        ps="9000"
+        ;;
+    *)
+        echo "Error: invalid ping_size. ping_size should be either: small, windows, standard, big, huge or jumbo"
+        echo "Cooresponding ping packet sizes (bytes): small=1, windows=32, standard=56, big=248, huge=1492, jumbo=9000"
+        ;;
+    esac
+    echo $ps
+}
+
 reboot_now() {
-	reboot &
+    reboot &
 
-	[ "$1" -ge 1 ] && {
-		sleep "$1"
-		echo 1 > /proc/sys/kernel/sysrq
-		echo b > /proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks.
-	}
+    [ "$1" -ge 1 ] && {
+        sleep "$1"
+        echo 1 >/proc/sys/kernel/sysrq
+        echo b >/proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks.
+    }
 }
 
-watchcat_always() {
-	local period="$1"; local forcedelay="$2"
+watchcat_periodic() {
+    failure_period="$1"
+    force_reboot_delay="$2"
 
-	sleep "$period" && reboot_now "$forcedelay"
+    sleep "$failure_period" && reboot_now "$force_reboot_delay"
+}
+
+watchcat_restart_modemmanager_iface() {
+    [ "$2" -gt 0 ] && {
+        logger -t INFO "Resetting current-bands to 'any' on modem: \"$1\" now."
+        /usr/bin/mmcli -m any --set-current-bands=any
+    }
+    logger -t INFO "Reconnecting modem: \"$1\" now."
+    /etc/init.d/modemmanager restart
+    ifup "$1"
+}
+
+watchcat_restart_network_iface() {
+    logger -t INFO "Restarting network interface: \"$1\"."
+    ip link set "$1" down
+    ip link set "$1" up
+}
+
+watchcat_restart_all_network() {
+    logger -t INFO "Restarting networking now by running: /etc/init.d/network restart"
+    /etc/init.d/network restart
+}
+
+watchcat_monitor_network() {
+    failure_period="$1"
+    ping_hosts="$2"
+    ping_frequency_interval="$3"
+    ping_size="$4"
+    iface="$5"
+    mm_iface_name="$6"
+    mm_iface_unlock_bands="$7"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+
+    [ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+    time_lastcheck="$time_now"
+    time_lastcheck_withinternet="$time_now"
+
+    ping_size="$(get_ping_size "$ping_size")"
+
+    while true; do
+        # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_diff="$((time_now - time_lastcheck))"
+
+        [ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))"
+
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_lastcheck="$time_now"
+
+        for host in $ping_hosts; do
+            if [ "$iface" != "" ]; then
+                ping_result="$(
+                    ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            else
+                ping_result="$(
+                    ping -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            fi
+
+            if [ "$ping_result" -eq 0 ]; then
+                time_lastcheck_withinternet="$time_now"
+            else
+                if [ "$iface" != "" ]; then
+                    logger -p daemon.info -t "watchcat[$$]" "Could not reach $host via \"$iface\" for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting \"$iface\" after reaching \"$failure_period\" seconds"
+                else
+                    logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting networking after reaching \"$failure_period\" seconds"
+                fi
+            fi
+        done
+
+        [ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && {
+            if [ "$mm_iface_name" != "" ]; then
+                watchcat_restart_modemmanager_iface "$mm_iface_name" "$mm_iface_unlock_bands"
+            fi
+            if [ "$iface" != "" ]; then
+                watchcat_restart_network_iface "$iface"
+            else
+                watchcat_restart_all_network
+            fi
+            /etc/init.d/watchcat start
+        }
+
+    done
 }
 
 watchcat_ping() {
-	local period="$1"; local forcedelay="$2"; local pinghosts="$3"; local pingperiod="$4"; local nopingtime="$5"
-
-	local time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}"
-
-	[ "$time_now" -lt "$nopingtime" ] && sleep "$((nopingtime-time_now))"
-
-	time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}"
-	local time_lastcheck="$time_now"
-	local time_lastcheck_withinternet="$time_now"
-
-	while true
-	do
-		# account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
-		time_now="$(cat /proc/uptime)"; time_now="${time_now%%.*}"
-		local time_diff="$((time_now-time_lastcheck))"
-
-		[ "$time_diff" -lt "$pingperiod" ] && sleep "$((pingperiod-time_diff))"
-
-		time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}"
-		time_lastcheck="$time_now"
-
-		for host in $pinghosts
-		do
-			if ping -c 1 "$host" &> /dev/null
-			then
-				time_lastcheck_withinternet="$time_now"
-			else								
-				logger -p daemon.info -t "watchcat[$$]" "no internet connectivity for $((time_now-time_lastcheck_withinternet)). Reseting when reaching $period"
-			fi
-		done
-		
-		[ "$((time_now-time_lastcheck_withinternet))" -ge "$period" ] && reboot_now "$forcedelay"
-	done
+    failure_period="$1"
+    force_reboot_delay="$2"
+    ping_hosts="$3"
+    ping_frequency_interval="$4"
+    ping_size="$5"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+
+    [ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+    time_lastcheck="$time_now"
+    time_lastcheck_withinternet="$time_now"
+
+    ping_size="$(get_ping_size "$ping_size")"
+
+    while true; do
+        # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_diff="$((time_now - time_lastcheck))"
+
+        [ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))"
+
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_lastcheck="$time_now"
+
+        for host in $ping_hosts; do
+            if [ "$iface" != "" ]; then
+                ping_result="$(
+                    ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            else
+                ping_result="$(
+                    ping -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            fi
+
+            if [ "$ping_result" -eq 0 ]; then
+                time_lastcheck_withinternet="$time_now"
+            else
+                logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for $((time_now - time_lastcheck_withinternet)). Rebooting after reaching $failure_period"
+            fi
+        done
+
+        [ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && reboot_now "$force_reboot_delay"
+    done
 }
 
-if [ "$1" = "always" ]
-then
-	watchcat_always "$2" "$3"
-else
-	watchcat_ping "$2" "$3" "$4" "$5" "$6"
-fi
+mode="$1"
+
+# Fix potential typo in mode and provide backward compatibility.
+[ "$mode" = "allways" ] && mode="periodic_reboot"
+[ "$mode" = "always" ] && mode="periodic_reboot"
+[ "$mode" = "ping" ] && mode="ping_reboot"
+
+case "$mode" in
+periodic_reboot)
+    watchcat_periodic "$2" "$3"
+    ;;
+ping_reboot)
+    watchcat_ping "$2" "$3" "$4" "$5" "$6"
+    ;;
+restart_iface)
+    watchcat_monitor_network "$2" "$3" "$4" "$5" "$6" "$7"
+    ;;
+*)
+    echo "Error: invalid mode selected: $mode"
+    ;;
+esac