watchcat: add ability to restart interface + refactor

Signed-off-by: Nicholas Smith <nicholas@nbembedded.com>

utils/watchcat/Makefile

@@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=watchcat
 PKG_VERSION:=1
-PKG_RELEASE:=10
+PKG_RELEASE:=11
 
 PKG_MAINTAINER:=Roger D <rogerdammit@gmail.com>
 PKG_LICENSE:=GPL-2.0

utils/watchcat/files/initd_watchcat

@@ -5,10 +5,10 @@ START=97
 PIDFILE="/tmp/run/watchcat"
 
 append_string() {
-	local varname="$1"
-	local add="$2"
-	local separator="${3:- }"
-	local actual
+	varname="$1"
+	add="$2"
+	separator="${3:- }"
+	actual
 	eval "actual=\$$varname"
 
 	new="${actual:+$actual$separator}$add"
@@ -16,29 +16,31 @@ append_string() {
 }
 
 time_to_seconds() {
-	local time=$1
-	unset seconds
+	time=$1
 
 	{ [ "$time" -ge 1 ] 2>/dev/null && seconds="$time"; } ||
 		{ [ "${time%s}" -ge 1 ] 2>/dev/null && seconds="${time%s}"; } ||
 		{ [ "${time%m}" -ge 1 ] 2>/dev/null && seconds=$((${time%m} * 60)); } ||
 		{ [ "${time%h}" -ge 1 ] 2>/dev/null && seconds=$((${time%h} * 3600)); } ||
 		{ [ "${time%d}" -ge 1 ] 2>/dev/null && seconds=$((${time%d} * 86400)); }
+
+	echo $seconds
+	unset seconds
+	unset time
 }
 
 load_watchcat() {
-	config_get period $1 period
-	config_get mode $1 mode
-	config_get pinghosts $1 pinghosts
-	config_get ping_period $1 ping_period
-	config_get no_ping_time $1 no_ping_time
-	config_get force_delay $1 force_delay
-	config_get ping_size $1 ping_size "standard"
-
-	local nopingtime_dflt="900"
-	local force_delay_default="60"
-
-	# Fix potential typo in mode (backward compatibility).
+	config_get period "$1" period "120"
+	config_get mode "$1" mode "restart_iface"
+	config_get pinghosts "$1" pinghosts "8.8.8.8"
+	config_get pingperiod "$1" pingperiod "60"
+	config_get forcedelay "$1" forcedelay "60"
+	config_get pingsize "$1" pingsize "standard"
+	config_get interface "$1" interface
+	config_get mmifacename "$1" mmifacename
+	config_get unlockbands "$1" unlockbands "0"
+
+	# Fix potential typo in mode and provide backward compatibility.
 	[ "$mode" = "allways" ] && mode="periodic_reboot"
 	[ "$mode" = "always" ] && mode="periodic_reboot"
 	[ "$mode" = "ping" ] && mode="ping_reboot"
@@ -46,62 +48,42 @@ load_watchcat() {
 	error=""
 	warn=""
 
-	if [ -z "$period" ]; then
-		append_string "error" "period is not set! Use time value(ex: '30'; '4m'; '6h'; '2d')." "; "
-	else
-		time_to_seconds "$period"
-		period="$seconds"
-		[ "$period" -ge 1 ] ||
-			append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; "
+	# Checks for settings common to all operation modes
+	if [ "$mode" != "periodic_reboot" ] && [ "$mode" != "ping_reboot" ] && [ "$mode" != "restart_iface" ]; then
+		append_string "error" "mode must be 'periodic_reboot' or 'ping_reboot' or 'restart_iface'" "; "
 	fi
 
-	[ "$mode" = "periodic_reboot" -o "$mode" = "ping_reboot" ] ||
-		append_string "error" "mode must be 'periodic_reboot' or 'ping_reboot'" "; "
+	period="$(time_to_seconds "$period")"
+	[ "$period" -ge 1 ] ||
+		append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; "
 
-	if [ -z "$force_delay" ]; then
-		force_delay="$force_delay_default"
-		append_string "warn" "force_delay is not configured! Defaulted to $force_delay seconds" "; "
-	else
-		[ "$force_delay" -ge 0 ] || {
-			force_delay="$force_delay_default"
-			append_string "warn" "force_delay is invalid! Defaulted to $force_delay seconds" "; "
-		}
-	fi
+	# ping_reboot mode and restart_iface mode specific checks
+	if [ "$mode" = "ping_reboot" ] || [ "$mode" = "restart_iface" ]; then
 
-	[ -z "$error" -a "$mode" = "ping_reboot" ] && {
-		[ -z "$pinghosts" ] &&
-			append_string "error" "pinghosts must be set in 'ping_reboot' mode! Use space separated address list (ex: '8.8.8.8 9.9.9.9')" "; "
-
-		if [ -z "$no_ping_time" ]; then
-			no_ping_time="$nopingtime_dflt"
-			append_string "warn" "no_ping_time is not configured! Defaulted to $no_ping_time seconds" "; "
-		else
-			time_to_seconds "$no_ping_time"
-			no_ping_time="$seconds"
-			[ "$no_ping_time" -ge 0 ] || {
-				no_ping_time="$nopingtime_dflt"
-				append_string "warn" "no_ping_time invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $no_ping_time seconds" "; "
-			}
-		fi
+		if [ -z "$error" ]; then
+
+			pingperiod_default="$((period / 5))"
+
+			pingperiod="$(time_to_seconds "$pingperiod")"
+			if [ "$pingperiod" -ge 0 ] && [ "$pingperiod" -ge "$period" ]; then
+				pingperiod="$(time_to_seconds "$pingperiod_default")"
+				append_string "warn" "pingperiod cannot be greater than $period. Defaulted to $pingperiod_default seconds (1/5 of period)" "; "
+			fi
 
-		local ping_period_default="$((period / 5))"
-
-		if [ -z "$ping_period" ]; then
-			ping_period="$ping_period_default"
-			append_string "warn" "ping_period is not configured! Defaulted to $ping_period seconds(1/5 of period)" "; "
-		else
-			time_to_seconds "$ping_period"
-			ping_period="$seconds"
-			[ "$ping_period" -ge 0 -a "$ping_period" -ge "$period" ] && {
-				ping_period="$ping_period_default"
-				append_string "warn" "ping_period is invalid value(greater than period)! Defaulted to $ping_period seconds(1/5 of period)" "; "
-			}
-			[ "$ping_period" -ge 0 ] || {
-				ping_period="$ping_period_default"
-				append_string "warn" "ping_period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $ping_period seconds(1/5 of period)" "; "
-			}
+			if [ "$pingperiod" -lt 0 ]; then
+				append_string "warn" "pingperiod cannot be a negative value." "; "
+			fi
+
+			if [ "$mmifacename" != "" ] && [ "$period" -lt 30 ]; then
+				append_string "error" "Check interval is less than 30s. For robust operation with ModemManager modem interfaces it is recommended to set the period to at least 30s."
+			fi
 		fi
-	}
+	fi
+
+	# ping_reboot mode and periodic_reboot mode specific checks
+	if [ "$mode" = "ping_reboot" ] || [ "$mode" = "periodic_reboot" ]; then
+		forcedelay="$(time_to_seconds "$forcedelay")"
+	fi
 
 	[ -n "$warn" ] && logger -p user.warn -t "watchcat" "$1: $warn"
 	[ -n "$error" ] && {
@@ -111,18 +93,22 @@ load_watchcat() {
 
 	case "$mode" in
 	periodic_reboot)
-		/usr/bin/watchcat.sh "periodic_reboot" "$period" "$force_delay" &
-		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;force_delay=$force_delay)"
+		/usr/bin/watchcat.sh "periodic_reboot" "$period" "$forcedelay" &
+		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;forcedelay=$forcedelay)"
 		;;
 	ping_reboot)
-		/usr/bin/watchcat.sh "ping_reboot" "$period" "$force_delay" "$ping_hosts" "$ping_period" "$no_ping_time" "$ping_size" &
-		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;ping_hosts=$ping_hosts;ping_period=$ping_period;force_delay=$force_delay;no_ping_time=$no_ping_time;ping_size=$ping_size)"
+		/usr/bin/watchcat.sh "ping_reboot" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$pingsize" &
+		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;pingsize=$pingsize)"
+		;;
+	restart_iface)
+		/usr/bin/watchcat.sh "restart_iface" "$period" "$pinghosts" "$pingperiod" "$pingsize" "$interface" "$mmifacename" &
+		logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;pingsize=$pingsize;interface=$interface;mmifacename=$mmifacename;unlockbands=$unlockbands)"
 		;;
 	*)
 		echo "Error starting Watchcat service. Invalid mode selection: $mode"
 		;;
 	esac
-	
+
 	echo $! >>"${PIDFILE}.pids"
 }
 

utils/watchcat/files/uci_defaults_watchcat

@@ -3,7 +3,7 @@
 uci -q show system.@watchcat[0] || {
 	uci add system watchcat
 	uci set system.@watchcat[0].period=6h
-	uci set system.@watchcat[0].mode=ping
+	uci set system.@watchcat[0].mode=ping_reboot
 	uci set system.@watchcat[0].pinghosts=8.8.8.8
 	uci set system.@watchcat[0].forcedelay=30
 	uci commit

utils/watchcat/files/watchcat.sh

@@ -1,107 +1,223 @@
 #!/bin/sh
 #
 # Copyright (C) 2010 segal.di.ubi.pt
+# Copyright (C) 2020 nbembedded.com
 #
 # This is free software, licensed under the GNU General Public License v2.
 #
 
 get_ping_size() {
-	ps=$1
-	case "$ps" in
-	small)
-		ps="1"
-		;;
-	windows)
-		ps="32"
-		;;
-	standard)
-		ps="56"
-		;;
-	big)
-		ps="248"
-		;;
-	huge)
-		ps="1492"
-		;;
-	jumbo)
-		ps="9000"
-		;;
-	*)
-		echo "Error: invalid ping_size. ping_size should be either: small, windows, standard, big, huge or jumbo"
-		echo "Cooresponding ping packet sizes (bytes): small=1, windows=32, standard=56, big=248, huge=1492, jumbo=9000"
-		;;
-	esac
-	echo $ps
+    ps=$1
+    case "$ps" in
+    small)
+        ps="1"
+        ;;
+    windows)
+        ps="32"
+        ;;
+    standard)
+        ps="56"
+        ;;
+    big)
+        ps="248"
+        ;;
+    huge)
+        ps="1492"
+        ;;
+    jumbo)
+        ps="9000"
+        ;;
+    *)
+        echo "Error: invalid ping_size. ping_size should be either: small, windows, standard, big, huge or jumbo"
+        echo "Cooresponding ping packet sizes (bytes): small=1, windows=32, standard=56, big=248, huge=1492, jumbo=9000"
+        ;;
+    esac
+    echo $ps
 }
 
 reboot_now() {
-	reboot &
+    reboot &
 
-	[ "$1" -ge 1 ] && {
-		sleep "$1"
-		echo 1 >/proc/sys/kernel/sysrq
-		echo b >/proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks.
-	}
+    [ "$1" -ge 1 ] && {
+        sleep "$1"
+        echo 1 >/proc/sys/kernel/sysrq
+        echo b >/proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks.
+    }
 }
 
 watchcat_periodic() {
-	local period="$1"
-	local force_delay="$2"
+    failure_period="$1"
+    force_reboot_delay="$2"
 
-	sleep "$period" && reboot_now "$force_delay"
+    sleep "$failure_period" && reboot_now "$force_reboot_delay"
+}
+
+watchcat_restart_modemmanager_iface() {
+    [ "$2" -gt 0 ] && {
+        logger -t INFO "Resetting current-bands to 'any' on modem: \"$1\" now."
+        /usr/bin/mmcli -m any --set-current-bands=any
+    }
+    logger -t INFO "Reconnecting modem: \"$1\" now."
+    /etc/init.d/modemmanager restart
+    ifup "$1"
+}
+
+watchcat_restart_network_iface() {
+    logger -t INFO "Restarting network interface: \"$1\"."
+    ip link set "$1" down
+    ip link set "$1" up
+}
+
+watchcat_restart_all_network() {
+    logger -t INFO "Restarting networking now by running: /etc/init.d/network restart"
+    /etc/init.d/network restart
+}
+
+watchcat_monitor_network() {
+    failure_period="$1"
+    ping_hosts="$2"
+    ping_frequency_interval="$3"
+    ping_size="$4"
+    iface="$5"
+    mm_iface_name="$6"
+    mm_iface_unlock_bands="$7"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+
+    [ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+    time_lastcheck="$time_now"
+    time_lastcheck_withinternet="$time_now"
+
+    ping_size="$(get_ping_size "$ping_size")"
+
+    while true; do
+        # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_diff="$((time_now - time_lastcheck))"
+
+        [ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))"
+
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_lastcheck="$time_now"
+
+        for host in $ping_hosts; do
+            if [ "$iface" != "" ]; then
+                ping_result="$(
+                    ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            else
+                ping_result="$(
+                    ping -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            fi
+
+            if [ "$ping_result" -eq 0 ]; then
+                time_lastcheck_withinternet="$time_now"
+            else
+                if [ "$iface" != "" ]; then
+                    logger -p daemon.info -t "watchcat[$$]" "Could not reach $host via \"$iface\" for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting \"$iface\" after reaching \"$failure_period\" seconds"
+                else
+                    logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting networking after reaching \"$failure_period\" seconds"
+                fi
+            fi
+        done
+
+        [ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && {
+            if [ "$mm_iface_name" != "" ]; then
+                watchcat_restart_modemmanager_iface "$mm_iface_name" "$mm_iface_unlock_bands"
+            fi
+            if [ "$iface" != "" ]; then
+                watchcat_restart_network_iface "$iface"
+            else
+                watchcat_restart_all_network
+            fi
+            /etc/init.d/watchcat start
+        }
+
+    done
 }
 
 watchcat_ping() {
-	local period="$1"
-	local force_delay="$2"
-	local ping_hosts="$3"
-	local ping_period="$4"
-	local no_ping_time="$5"
-	local ping_size="$6"
-
-	local time_now="$(cat /proc/uptime)"
-	time_now="${time_now%%.*}"
-
-	[ "$time_now" -lt "$no_ping_time" ] && sleep "$((no_ping_time - time_now))"
-
-	time_now="$(cat /proc/uptime)"
-	time_now="${time_now%%.*}"
-	local time_lastcheck="$time_now"
-	local time_lastcheck_withinternet="$time_now"
-	local ping_size="$(get_ping_size "$ping_size")"
-
-	while true; do
-		# account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
-		time_now="$(cat /proc/uptime)"
-		time_now="${time_now%%.*}"
-		local time_diff="$((time_now - time_lastcheck))"
-
-		[ "$time_diff" -lt "$ping_period" ] && sleep "$((ping_period - time_diff))"
-
-		time_now="$(cat /proc/uptime)"
-		time_now="${time_now%%.*}"
-		time_lastcheck="$time_now"
-
-		for host in $ping_hosts; do
-			if ping -s "$ping_size" -c 1 "$host" &>/dev/null; then
-				time_lastcheck_withinternet="$time_now"
-			else
-				logger -p daemon.info -t "watchcat[$$]" "no internet connectivity for $((time_now - time_lastcheck_withinternet)). Reseting when reaching $period"
-			fi
-		done
-
-		[ "$((time_now - time_lastcheck_withinternet))" -ge "$period" ] && reboot_now "$force_delay"
-	done
+    failure_period="$1"
+    force_reboot_delay="$2"
+    ping_hosts="$3"
+    ping_frequency_interval="$4"
+    ping_size="$5"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+
+    [ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))"
+
+    time_now="$(cat /proc/uptime)"
+    time_now="${time_now%%.*}"
+    time_lastcheck="$time_now"
+    time_lastcheck_withinternet="$time_now"
+
+    ping_size="$(get_ping_size "$ping_size")"
+
+    while true; do
+        # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay.
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_diff="$((time_now - time_lastcheck))"
+
+        [ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))"
+
+        time_now="$(cat /proc/uptime)"
+        time_now="${time_now%%.*}"
+        time_lastcheck="$time_now"
+
+        for host in $ping_hosts; do
+            if [ "$iface" != "" ]; then
+                ping_result="$(
+                    ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            else
+                ping_result="$(
+                    ping -s "$ping_size" -c 1 "$host" &>/dev/null
+                    echo $?
+                )"
+            fi
+
+            if [ "$ping_result" -eq 0 ]; then
+                time_lastcheck_withinternet="$time_now"
+            else
+                logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for $((time_now - time_lastcheck_withinternet)). Rebooting after reaching $failure_period"
+            fi
+        done
+
+        [ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && reboot_now "$force_reboot_delay"
+    done
 }
 
+mode="$1"
+
+# Fix potential typo in mode and provide backward compatibility.
+[ "$mode" = "allways" ] && mode="periodic_reboot"
+[ "$mode" = "always" ] && mode="periodic_reboot"
+[ "$mode" = "ping" ] && mode="ping_reboot"
+
 case "$mode" in
 periodic_reboot)
-	watchcat_periodic "$2" "$3"
-	;;
+    watchcat_periodic "$2" "$3"
+    ;;
 ping_reboot)
-	watchcat_ping "$2" "$3" "$4" "$5" "$6" "$7"
-	;;
+    watchcat_ping "$2" "$3" "$4" "$5" "$6"
+    ;;
+restart_iface)
+    watchcat_monitor_network "$2" "$3" "$4" "$5" "$6" "$7"
+    ;;
 *)
-	echo "Error: invalid mode selected: $mode"
-	;;
+    echo "Error: invalid mode selected: $mode"
+    ;;
 esac