mwan3: fix interface-bound traffic when interface is offline

This commit fixed what 6d99b602 was supposed to fix without affecting
interface-bound traffic.

Before 6d99b602 interface-bound traffic was working normally as long
as at least one interface was online. However when the last interface
went offline, it was impossible to ping and such state was
unrecoverable.

Commit 6d99b602 fixed unrecoverable offline state problem (it was
possible to ping -I iface) but messed inteface-bound traffic. Traffic
with interface source address was not working if the interface was in
"offline" state, even if another interface was online.
The problem was caused by an inconsistent "offline" interface state:
iptables-related rules were kept while routing table and policy were
deleted.

The idea behind this commit is to:
 1. Keep all the rules for each interface (iptables, routing table,
    policy) regardless of its state. This ensures consistency,
 2. Make interface state hotplug events affect only iptables'
    mwan3_policy_* rules. Interface-related iptables, routing table
    and policy is removed only when mwan3 is manually stopped.

To make such changes possible, it's necessary to change the way
mwan3_policy_* rule generator keeps track of interface state hotplug
events.
Until now, it checked for the existence of custom interface-related
routing table (table id 1, 2, 3, ...). Clearly we can no longer rely
on that so each interface state is stored explicitly in file.

Signed-off-by: Marcin Jurkowski <marcin1j@gmail.com>

net/mwan3/Makefile

@@ -8,7 +8,7 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=mwan3
-PKG_VERSION:=2.6.4
+PKG_VERSION:=2.6.5
 PKG_RELEASE:=1
 PKG_MAINTAINER:=Florian Eckert <fe@dev.tdt.de>
 PKG_LICENSE:=GPLv2

net/mwan3/files/etc/hotplug.d/iface/15-mwan3

@@ -65,21 +65,23 @@ case "$ACTION" in
 		mwan3_set_general_rules
 		mwan3_set_general_iptables
 		mwan3_create_iface_iptables $INTERFACE $DEVICE
+		mwan3_create_iface_rules $INTERFACE $DEVICE
+		mwan3_create_iface_route $INTERFACE $DEVICE
 		if [ ${running} -eq 1 -a "${status}" = "online" ]; then
-			mwan3_create_iface_rules $INTERFACE $DEVICE
-			mwan3_create_iface_route $INTERFACE $DEVICE
+			$LOG notice "Starting tracker on interface $INTERFACE (${DEVICE:-unknown})"
+			mwan3_set_iface_hotplug_state $INTERFACE "online"
 			mwan3_track $INTERFACE $DEVICE "online" "$src_ip"
 			mwan3_set_policies_iptables
 			mwan3_set_user_rules
 			mwan3_flush_conntrack $INTERFACE $DEVICE "ifup"
 		else
 			$LOG notice "Starting tracker on interface $INTERFACE (${DEVICE:-unknown})"
+			mwan3_set_iface_hotplug_state $INTERFACE "offline"
 			mwan3_track $INTERFACE $DEVICE "offline" "$src_ip"
 		fi
 	;;
 	ifdown)
-		mwan3_delete_iface_rules $INTERFACE
-		mwan3_delete_iface_route $INTERFACE
+		mwan3_set_iface_hotplug_state $INTERFACE "offline"
 		mwan3_delete_iface_ipset_entries $INTERFACE
 		mwan3_track_signal $INTERFACE $DEVICE
 		mwan3_set_policies_iptables

net/mwan3/files/lib/mwan3/mwan3.sh

@@ -8,15 +8,16 @@ IPT6="ip6tables -t mangle -w"
 LOG="logger -t mwan3[$$] -p"
 CONNTRACK_FILE="/proc/net/nf_conntrack"
 
-MWAN3_STATUS_DIR="/var/run/mwan3track"
+MWAN3_STATUS_DIR="/var/run/mwan3"
+MWAN3TRACK_STATUS_DIR="/var/run/mwan3track"
 
+[ -d $MWAN3_STATUS_DIR ] || mkdir -p $MWAN3_STATUS_DIR/iface_state
 # mwan3's MARKing mask (at least 3 bits should be set)
 if [ -e "${MWAN3_STATUS_DIR}/mmx_mask" ]; then
 	MMX_MASK=$(cat "${MWAN3_STATUS_DIR}/mmx_mask")
 else
 	config_load mwan3
 	config_get MMX_MASK globals mmx_mask '0xff00'
-	mkdir -p "${MWAN3_STATUS_DIR}"
 	echo "$MMX_MASK" > "${MWAN3_STATUS_DIR}/mmx_mask"
 	$LOG notice "Using firewall mask ${MMX_MASK}"
 fi
@@ -499,7 +500,7 @@ mwan3_set_policy()
 
 	if [ "$family" == "ipv4" ]; then
 
-		if [ -n "$($IP4 route list table $id)" ]; then
+		if [ "$(mwan3_get_iface_hotplug_state $iface)" = "online" ]; then
 			if [ "$metric" -lt "$lowest_metric_v4" ]; then
 
 				total_weight_v4=$weight
@@ -532,7 +533,7 @@ mwan3_set_policy()
 
 	if [ "$family" == "ipv6" ]; then
 
-		if [ -n "$($IP6 route list table $id)" ]; then
+		if [ "$(mwan3_get_iface_hotplug_state $iface)" = "online" ]; then
 			if [ "$metric" -lt "$lowest_metric_v6" ]; then
 
 				total_weight_v6=$weight
@@ -763,6 +764,19 @@ mwan3_set_user_rules()
 	config_foreach mwan3_set_user_iptables_rule rule
 }
 
+mwan3_set_iface_hotplug_state() {
+	local iface=$1
+	local state=$2
+
+	echo -n $state > $MWAN3_STATUS_DIR/iface_state/$iface
+}
+
+mwan3_get_iface_hotplug_state() {
+	local iface=$1
+
+	cat $MWAN3_STATUS_DIR/iface_state/$iface 2>/dev/null || echo "unknown"
+}
+
 mwan3_report_iface_status()
 {
 	local device result track_ips tracking IP IPT
@@ -784,16 +798,14 @@ mwan3_report_iface_status()
 
 	if [ -z "$id" -o -z "$device" ]; then
 		result="unknown"
-	elif [ -n "$($IP rule | awk '$1 == "'$(($id+1000)):'"')"i -a -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" -a -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" -a -n "$($IPT -S mwan3_iface_out_$1 2> /dev/null)" -a -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
-		result="online"
+	elif [ -n "$($IP rule | awk '$1 == "'$(($id+1000)):'"')" -a -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" -a -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" -a -n "$($IPT -S mwan3_iface_out_$1 2> /dev/null)" -a -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
+		result="$(mwan3_get_iface_hotplug_state $1)"
 	elif [ -n "$($IP rule | awk '$1 == "'$(($id+1000)):'"')" -o -n "$($IP rule | awk '$1 == "'$(($id+2000)):'"')" -o -n "$($IPT -S mwan3_iface_in_$1 2> /dev/null)" -o -n "$($IPT -S mwan3_iface_out_$1 2> /dev/null)" -o -n "$($IP route list table $id default dev $device 2> /dev/null)" ]; then
 		result="error"
+	elif [ "$enabled" == "1" ]; then
+		result="offline"
 	else
-		if [ "$enabled" == "1" ]; then
-			result="offline"
-		else
-			result="disabled"
-		fi
+		result="disabled"
 	fi
 
 	mwan3_list_track_ips()

net/mwan3/files/usr/libexec/rpcd/mwan3

@@ -4,7 +4,7 @@
 . /lib/functions/network.sh
 . /usr/share/libubox/jshn.sh
 
-MWAN3_STATUS_DIR="/var/run/mwan3track"
+MWAN3TRACK_STATUS_DIR="/var/run/mwan3track"
 
 IPS="ipset"
 IPT4="iptables -t mangle -w"
@@ -45,7 +45,7 @@ get_mwan3_status() {
 			running="1"
 		fi
 
-		time_p="$(cat "$MWAN3_STATUS_DIR/${iface}/TIME")"
+		time_p="$(cat "$MWAN3TRACK_STATUS_DIR/${iface}/TIME")"
 		[ -z "${time_p}" ] || {
 			time_n="$(date +'%s')"
 			let age=time_n-time_p
@@ -53,13 +53,13 @@ get_mwan3_status() {
 
 		json_add_object "${iface}"
 		json_add_int age "$age"
-		json_add_int "score" "$(cat "$MWAN3_STATUS_DIR/${iface}/SCORE")"
-		json_add_int "lost" "$(cat "$MWAN3_STATUS_DIR/${iface}/LOST")"
-		json_add_int "turn" "$(cat "$MWAN3_STATUS_DIR/${iface}/TURN")"
-		json_add_string "status" "$(cat "$MWAN3_STATUS_DIR/${iface}/STATUS")"
+		json_add_int "score" "$(cat "$MWAN3TRACK_STATUS_DIR/${iface}/SCORE")"
+		json_add_int "lost" "$(cat "$MWAN3TRACK_STATUS_DIR/${iface}/LOST")"
+		json_add_int "turn" "$(cat "$MWAN3TRACK_STATUS_DIR/${iface}/TURN")"
+		json_add_string "status" "$(cat "$MWAN3TRACK_STATUS_DIR/${iface}/STATUS")"
 		json_add_boolean "running" "${running}"
 		json_add_array "track_ip"
-		for file in $MWAN3_STATUS_DIR/${iface}/*; do
+		for file in $MWAN3TRACK_STATUS_DIR/${iface}/*; do
 			track="${file#*/TRACK_}"
 			if [ "${track}" != "${file}" ]; then
 				json_add_object

net/mwan3/files/usr/sbin/mwan3

@@ -37,7 +37,6 @@ ifdown()
 	ACTION=ifdown INTERFACE=$1 /sbin/hotplug-call iface
 
 	kill $(pgrep -f "mwan3track $1 $2") &> /dev/null
-	mwan3_delete_iface_iptables $1
 	mwan3_track_clean $1
 }
 
@@ -160,7 +159,7 @@ stop()
 	done
 
 	mwan3_lock_clean
-	rm -rf "${MWAN3_STATUS_DIR}/mmx_mask"
+	rm -rf $MWAN3_STATUS_DIR $MWAN3TRACK_STATUS_DIR
 }
 
 restart() {