diff --git a/utils/watchcat/Makefile b/utils/watchcat/Makefile index ec6e0895e..0cbf11dda 100644 --- a/utils/watchcat/Makefile +++ b/utils/watchcat/Makefile @@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=watchcat PKG_VERSION:=1 -PKG_RELEASE:=8 +PKG_RELEASE:=11 PKG_MAINTAINER:=Roger D PKG_LICENSE:=GPL-2.0 diff --git a/utils/watchcat/files/initd_watchcat b/utils/watchcat/files/initd_watchcat index b34b50eda..95f59271f 100644 --- a/utils/watchcat/files/initd_watchcat +++ b/utils/watchcat/files/initd_watchcat @@ -5,123 +5,120 @@ START=97 PIDFILE="/tmp/run/watchcat" append_string() { - local varname="$1"; local add="$2"; local separator="${3:- }"; local actual + varname="$1" + add="$2" + separator="${3:- }" + actual eval "actual=\$$varname" new="${actual:+$actual$separator}$add" eval "$varname=\$new" } -timetoseconds() { - local time=$1 - unset seconds +time_to_seconds() { + time=$1 + + { [ "$time" -ge 1 ] 2>/dev/null && seconds="$time"; } || + { [ "${time%s}" -ge 1 ] 2>/dev/null && seconds="${time%s}"; } || + { [ "${time%m}" -ge 1 ] 2>/dev/null && seconds=$((${time%m} * 60)); } || + { [ "${time%h}" -ge 1 ] 2>/dev/null && seconds=$((${time%h} * 3600)); } || + { [ "${time%d}" -ge 1 ] 2>/dev/null && seconds=$((${time%d} * 86400)); } - { [ "$time" -ge 1 ] 2> /dev/null && seconds="$time"; } || \ - { [ "${time%s}" -ge 1 ] 2> /dev/null && seconds="${time%s}"; } || \ - { [ "${time%m}" -ge 1 ] 2> /dev/null && seconds=$((${time%m}*60)); } || \ - { [ "${time%h}" -ge 1 ] 2> /dev/null && seconds=$((${time%h}*3600)); } || \ - { [ "${time%d}" -ge 1 ] 2> /dev/null && seconds=$((${time%d}*86400)); } + echo $seconds + unset seconds + unset time } load_watchcat() { - config_get period $1 period - config_get mode $1 mode - config_get pinghosts $1 pinghosts - config_get pingperiod $1 pingperiod - config_get nopingtime $1 nopingtime - config_get forcedelay $1 forcedelay - - local nopingtime_dflt="900" - local forcedelay_dflt="60" + config_get period "$1" period "120" + config_get mode "$1" mode "restart_iface" + config_get pinghosts "$1" pinghosts "8.8.8.8" + config_get pingperiod "$1" pingperiod "60" + config_get forcedelay "$1" forcedelay "60" + config_get pingsize "$1" pingsize "standard" + config_get interface "$1" interface + config_get mmifacename "$1" mmifacename + config_get unlockbands "$1" unlockbands "0" + + # Fix potential typo in mode and provide backward compatibility. + [ "$mode" = "allways" ] && mode="periodic_reboot" + [ "$mode" = "always" ] && mode="periodic_reboot" + [ "$mode" = "ping" ] && mode="ping_reboot" - # Fix potential typo in mode (backward compatibility). - [ "$mode" = "allways" ] && mode="always" - error="" warn="" - - if [ -z "$period" ] - then - append_string "error" "period is not set! Use time value(ex: '30'; '4m'; '6h'; '2d')." "; " - else - timetoseconds "$period";period="$seconds" - [ "$period" -ge 1 ] \ - || append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; " - fi - - [ "$mode" = "always" -o "$mode" = "ping" ] \ - || append_string "error" "mode must be 'always' or 'ping'" "; " - - if [ -z "$forcedelay" ] - then - forcedelay="$forcedelay_dflt" - append_string "warn" "forcedelay is not configured! Defaulted to $forcedelay seconds" "; " - else - [ "$forcedelay" -ge 0 ] || { - forcedelay="$forcedelay_dflt" - append_string "warn" "forcedelay is invalid! Defaulted to $forcedelay seconds" "; " - } + + # Checks for settings common to all operation modes + if [ "$mode" != "periodic_reboot" ] && [ "$mode" != "ping_reboot" ] && [ "$mode" != "restart_iface" ]; then + append_string "error" "mode must be 'periodic_reboot' or 'ping_reboot' or 'restart_iface'" "; " fi - - [ -z "$error" -a "$mode" = "ping" ] && { - [ -z "$pinghosts" ] \ - && append_string "error" "pinghosts must be set in 'ping' mode! Use space separated address list (ex: '8.8.8.8 9.9.9.9')" "; " - - if [ -z "$nopingtime" ] - then - nopingtime="$nopingtime_dflt" - append_string "warn" "nopingtime is not configured! Defaulted to $nopingtime seconds" "; " - else - timetoseconds "$nopingtime";nopingtime="$seconds" - [ "$nopingtime" -ge 0 ] || { - nopingtime="$nopingtime_dflt" - append_string "warn" "nopingtime invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $nopingtime seconds" "; " - } - fi - - local pingperiod_dflt="$((period/5))" - - if [ -z "$pingperiod" ] - then - pingperiod="$pingperiod_dflt" - append_string "warn" "pingperiod is not configured! Defaulted to $pingperiod seconds(1/5 of period)" "; " - else - timetoseconds "$pingperiod";pingperiod="$seconds" - [ "$pingperiod" -ge 0 -a "$pingperiod" -ge "$period" ] && { - pingperiod="$pingperiod_dflt" - append_string "warn" "pingperiod is invalid value(greater than period)! Defaulted to $pingperiod seconds(1/5 of period)" "; " - } - [ "$pingperiod" -ge 0 ] || { - pingperiod="$pingperiod_dflt" - append_string "warn" "pingperiod has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d'). Defaulted to $pingperiod seconds(1/5 of period)" "; " - } + + period="$(time_to_seconds "$period")" + [ "$period" -ge 1 ] || + append_string "error" "period has invalid format! Use time value(ex: '30'; '4m'; '6h'; '2d')" "; " + + # ping_reboot mode and restart_iface mode specific checks + if [ "$mode" = "ping_reboot" ] || [ "$mode" = "restart_iface" ]; then + + if [ -z "$error" ]; then + + pingperiod_default="$((period / 5))" + + pingperiod="$(time_to_seconds "$pingperiod")" + if [ "$pingperiod" -ge 0 ] && [ "$pingperiod" -ge "$period" ]; then + pingperiod="$(time_to_seconds "$pingperiod_default")" + append_string "warn" "pingperiod cannot be greater than $period. Defaulted to $pingperiod_default seconds (1/5 of period)" "; " + fi + + if [ "$pingperiod" -lt 0 ]; then + append_string "warn" "pingperiod cannot be a negative value." "; " + fi + + if [ "$mmifacename" != "" ] && [ "$period" -lt 30 ]; then + append_string "error" "Check interval is less than 30s. For robust operation with ModemManager modem interfaces it is recommended to set the period to at least 30s." + fi fi - } - - [ -n "$warn" ] && logger -p user.warn -t "watchcat" "$1: $warn" - [ -n "$error" ] && { logger -p user.err -t "watchcat" "reboot program $1 not started - $error"; return; } + fi - if [ "$mode" = "always" ] - then - /usr/bin/watchcat.sh "always" "$period" "$forcedelay" & - logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;forcedelay=$forcedelay)" - else - /usr/bin/watchcat.sh "ping" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$nopingtime" & - logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;nopingtime=$nopingtime)" + # ping_reboot mode and periodic_reboot mode specific checks + if [ "$mode" = "ping_reboot" ] || [ "$mode" = "periodic_reboot" ]; then + forcedelay="$(time_to_seconds "$forcedelay")" fi - echo $! >> "${PIDFILE}.pids" + [ -n "$warn" ] && logger -p user.warn -t "watchcat" "$1: $warn" + [ -n "$error" ] && { + logger -p user.err -t "watchcat" "reboot program $1 not started - $error" + return + } + + case "$mode" in + periodic_reboot) + /usr/bin/watchcat.sh "periodic_reboot" "$period" "$forcedelay" & + logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;forcedelay=$forcedelay)" + ;; + ping_reboot) + /usr/bin/watchcat.sh "ping_reboot" "$period" "$forcedelay" "$pinghosts" "$pingperiod" "$pingsize" & + logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;forcedelay=$forcedelay;pingsize=$pingsize)" + ;; + restart_iface) + /usr/bin/watchcat.sh "restart_iface" "$period" "$pinghosts" "$pingperiod" "$pingsize" "$interface" "$mmifacename" & + logger -p user.info -t "watchcat" "started task (mode=$mode;period=$period;pinghosts=$pinghosts;pingperiod=$pingperiod;pingsize=$pingsize;interface=$interface;mmifacename=$mmifacename;unlockbands=$unlockbands)" + ;; + *) + echo "Error starting Watchcat service. Invalid mode selection: $mode" + ;; + esac + + echo $! >>"${PIDFILE}.pids" } stop() { - if [ -f "${PIDFILE}.pids" ] - then + if [ -f "${PIDFILE}.pids" ]; then logger -p user.info -t "watchcat" "stopping all tasks" - while read pid - do + while read pid; do kill -KILL "$pid" - done < "${PIDFILE}.pids" + done <"${PIDFILE}.pids" rm "${PIDFILE}.pids" @@ -135,8 +132,7 @@ start() { [ -f "${PIDFILE}.pids" ] && stop config_load system - if [ -n "$(uci show system.@watchcat[0])" ] # at least one watchcat section exists - then + if [ -n "$(uci show system.@watchcat[0])" ]; then # at least one watchcat section exists logger -p user.info -t "watchcat" "starting all tasks" config_foreach load_watchcat watchcat logger -p user.info -t "watchcat" "all tasks started" diff --git a/utils/watchcat/files/uci_defaults_watchcat b/utils/watchcat/files/uci_defaults_watchcat index 49def81eb..da4993230 100644 --- a/utils/watchcat/files/uci_defaults_watchcat +++ b/utils/watchcat/files/uci_defaults_watchcat @@ -3,7 +3,7 @@ uci -q show system.@watchcat[0] || { uci add system watchcat uci set system.@watchcat[0].period=6h - uci set system.@watchcat[0].mode=ping + uci set system.@watchcat[0].mode=ping_reboot uci set system.@watchcat[0].pinghosts=8.8.8.8 uci set system.@watchcat[0].forcedelay=30 uci commit diff --git a/utils/watchcat/files/watchcat.sh b/utils/watchcat/files/watchcat.sh index ca660bb13..cbddffc39 100644 --- a/utils/watchcat/files/watchcat.sh +++ b/utils/watchcat/files/watchcat.sh @@ -1,65 +1,223 @@ #!/bin/sh # # Copyright (C) 2010 segal.di.ubi.pt +# Copyright (C) 2020 nbembedded.com # # This is free software, licensed under the GNU General Public License v2. # +get_ping_size() { + ps=$1 + case "$ps" in + small) + ps="1" + ;; + windows) + ps="32" + ;; + standard) + ps="56" + ;; + big) + ps="248" + ;; + huge) + ps="1492" + ;; + jumbo) + ps="9000" + ;; + *) + echo "Error: invalid ping_size. ping_size should be either: small, windows, standard, big, huge or jumbo" + echo "Cooresponding ping packet sizes (bytes): small=1, windows=32, standard=56, big=248, huge=1492, jumbo=9000" + ;; + esac + echo $ps +} + reboot_now() { - reboot & + reboot & - [ "$1" -ge 1 ] && { - sleep "$1" - echo 1 > /proc/sys/kernel/sysrq - echo b > /proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks. - } + [ "$1" -ge 1 ] && { + sleep "$1" + echo 1 >/proc/sys/kernel/sysrq + echo b >/proc/sysrq-trigger # Will immediately reboot the system without syncing or unmounting your disks. + } } -watchcat_always() { - local period="$1"; local forcedelay="$2" +watchcat_periodic() { + failure_period="$1" + force_reboot_delay="$2" - sleep "$period" && reboot_now "$forcedelay" + sleep "$failure_period" && reboot_now "$force_reboot_delay" +} + +watchcat_restart_modemmanager_iface() { + [ "$2" -gt 0 ] && { + logger -t INFO "Resetting current-bands to 'any' on modem: \"$1\" now." + /usr/bin/mmcli -m any --set-current-bands=any + } + logger -t INFO "Reconnecting modem: \"$1\" now." + /etc/init.d/modemmanager restart + ifup "$1" +} + +watchcat_restart_network_iface() { + logger -t INFO "Restarting network interface: \"$1\"." + ip link set "$1" down + ip link set "$1" up +} + +watchcat_restart_all_network() { + logger -t INFO "Restarting networking now by running: /etc/init.d/network restart" + /etc/init.d/network restart +} + +watchcat_monitor_network() { + failure_period="$1" + ping_hosts="$2" + ping_frequency_interval="$3" + ping_size="$4" + iface="$5" + mm_iface_name="$6" + mm_iface_unlock_bands="$7" + + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + + [ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))" + + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + time_lastcheck="$time_now" + time_lastcheck_withinternet="$time_now" + + ping_size="$(get_ping_size "$ping_size")" + + while true; do + # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay. + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + time_diff="$((time_now - time_lastcheck))" + + [ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))" + + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + time_lastcheck="$time_now" + + for host in $ping_hosts; do + if [ "$iface" != "" ]; then + ping_result="$( + ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null + echo $? + )" + else + ping_result="$( + ping -s "$ping_size" -c 1 "$host" &>/dev/null + echo $? + )" + fi + + if [ "$ping_result" -eq 0 ]; then + time_lastcheck_withinternet="$time_now" + else + if [ "$iface" != "" ]; then + logger -p daemon.info -t "watchcat[$$]" "Could not reach $host via \"$iface\" for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting \"$iface\" after reaching \"$failure_period\" seconds" + else + logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for \"$((time_now - time_lastcheck_withinternet))\" seconds. Restarting networking after reaching \"$failure_period\" seconds" + fi + fi + done + + [ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && { + if [ "$mm_iface_name" != "" ]; then + watchcat_restart_modemmanager_iface "$mm_iface_name" "$mm_iface_unlock_bands" + fi + if [ "$iface" != "" ]; then + watchcat_restart_network_iface "$iface" + else + watchcat_restart_all_network + fi + /etc/init.d/watchcat start + } + + done } watchcat_ping() { - local period="$1"; local forcedelay="$2"; local pinghosts="$3"; local pingperiod="$4"; local nopingtime="$5" - - local time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}" - - [ "$time_now" -lt "$nopingtime" ] && sleep "$((nopingtime-time_now))" - - time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}" - local time_lastcheck="$time_now" - local time_lastcheck_withinternet="$time_now" - - while true - do - # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay. - time_now="$(cat /proc/uptime)"; time_now="${time_now%%.*}" - local time_diff="$((time_now-time_lastcheck))" - - [ "$time_diff" -lt "$pingperiod" ] && sleep "$((pingperiod-time_diff))" - - time_now="$(cat /proc/uptime)";time_now="${time_now%%.*}" - time_lastcheck="$time_now" - - for host in $pinghosts - do - if ping -c 1 "$host" &> /dev/null - then - time_lastcheck_withinternet="$time_now" - else - logger -p daemon.info -t "watchcat[$$]" "no internet connectivity for $((time_now-time_lastcheck_withinternet)). Reseting when reaching $period" - fi - done - - [ "$((time_now-time_lastcheck_withinternet))" -ge "$period" ] && reboot_now "$forcedelay" - done + failure_period="$1" + force_reboot_delay="$2" + ping_hosts="$3" + ping_frequency_interval="$4" + ping_size="$5" + + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + + [ "$time_now" -lt "$failure_period" ] && sleep "$((failure_period - time_now))" + + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + time_lastcheck="$time_now" + time_lastcheck_withinternet="$time_now" + + ping_size="$(get_ping_size "$ping_size")" + + while true; do + # account for the time ping took to return. With a ping time of 5s, ping might take more than that, so it is important to avoid even more delay. + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + time_diff="$((time_now - time_lastcheck))" + + [ "$time_diff" -lt "$ping_frequency_interval" ] && sleep "$((ping_frequency_interval - time_diff))" + + time_now="$(cat /proc/uptime)" + time_now="${time_now%%.*}" + time_lastcheck="$time_now" + + for host in $ping_hosts; do + if [ "$iface" != "" ]; then + ping_result="$( + ping -I "$iface" -s "$ping_size" -c 1 "$host" &>/dev/null + echo $? + )" + else + ping_result="$( + ping -s "$ping_size" -c 1 "$host" &>/dev/null + echo $? + )" + fi + + if [ "$ping_result" -eq 0 ]; then + time_lastcheck_withinternet="$time_now" + else + logger -p daemon.info -t "watchcat[$$]" "Could not reach $host for $((time_now - time_lastcheck_withinternet)). Rebooting after reaching $failure_period" + fi + done + + [ "$((time_now - time_lastcheck_withinternet))" -ge "$failure_period" ] && reboot_now "$force_reboot_delay" + done } -if [ "$1" = "always" ] -then - watchcat_always "$2" "$3" -else - watchcat_ping "$2" "$3" "$4" "$5" "$6" -fi +mode="$1" + +# Fix potential typo in mode and provide backward compatibility. +[ "$mode" = "allways" ] && mode="periodic_reboot" +[ "$mode" = "always" ] && mode="periodic_reboot" +[ "$mode" = "ping" ] && mode="ping_reboot" + +case "$mode" in +periodic_reboot) + watchcat_periodic "$2" "$3" + ;; +ping_reboot) + watchcat_ping "$2" "$3" "$4" "$5" "$6" + ;; +restart_iface) + watchcat_monitor_network "$2" "$3" "$4" "$5" "$6" "$7" + ;; +*) + echo "Error: invalid mode selected: $mode" + ;; +esac