From bbbc6127abf132a92cc73dd23fa88409dc2b9abd Mon Sep 17 00:00:00 2001 From: Aaron Goodman Date: Sun, 11 Oct 2020 18:37:25 -0400 Subject: [PATCH] mwan3: use helper library for mwan3track Rather than using a special mwan3 user to manage mwan3track's tracking packets, this commit implements a small helper library to bind to device and to set a fwmark so that the tracking packets can be routed out of the correct interface. This provides a consistent method for binding to a device rather than relying on various packages potentially buggy implementations. For example: #8139 and #12836 This helper issue also allows for more tracking methods to be added even if they do not have a command line option to bind to device, such as iperf3 (eg #13050). Signed-off-by: Aaron Goodman --- net/mwan3/Makefile | 10 +- net/mwan3/files/lib/mwan3/common.sh | 27 ++- net/mwan3/files/lib/mwan3/mwan3.sh | 93 +--------- net/mwan3/files/usr/sbin/mwan3track | 48 +++--- net/mwan3/src/sockopt_wrap.c | 255 ++++++++++++++++++++++++++++ 5 files changed, 312 insertions(+), 121 deletions(-) create mode 100644 net/mwan3/src/sockopt_wrap.c diff --git a/net/mwan3/Makefile b/net/mwan3/Makefile index 65e2925c3..4850bfae4 100644 --- a/net/mwan3/Makefile +++ b/net/mwan3/Makefile @@ -8,10 +8,11 @@ include $(TOPDIR)/rules.mk PKG_NAME:=mwan3 -PKG_VERSION:=2.9.0 +PKG_VERSION:=2.10.0 PKG_RELEASE:=1 PKG_MAINTAINER:=Florian Eckert PKG_LICENSE:=GPL-2.0 +PKG_CONFIG_DEPENDS:=CONFIG_IPV6 include $(INCLUDE_DIR)/package.mk @@ -61,8 +62,13 @@ fi exit 0 endef +define Build/Compile + $(TARGET_CC) $(CFLAGS) $(LDFLAGS) $(FPIC) -shared -o $(PKG_BUILD_DIR)/libwrap_mwan3_sockopt.so.1.0 $(if $(CONFIG_IPV6),-DCONFIG_IPV6) $(PKG_BUILD_DIR)/sockopt_wrap.c -ldl +endef + define Package/mwan3/install -$(CP) ./files/* $(1) + $(CP) ./files/* $(1) + $(CP) $(PKG_BUILD_DIR)/libwrap_mwan3_sockopt.so.1.0 $(1)/lib/mwan3/ endef $(eval $(call BuildPackage,mwan3)) diff --git a/net/mwan3/files/lib/mwan3/common.sh b/net/mwan3/files/lib/mwan3/common.sh index 53557a6ef..4deb9bfeb 100644 --- a/net/mwan3/files/lib/mwan3/common.sh +++ b/net/mwan3/files/lib/mwan3/common.sh @@ -8,8 +8,19 @@ get_uptime() { IP4="ip -4" IP6="ip -6" SCRIPTNAME="$(basename "$0")" + +MWAN3_STATUS_DIR="/var/run/mwan3" MWAN3TRACK_STATUS_DIR="/var/run/mwan3track" +MWAN3_INTERFACE_MAX="" + +MMX_MASK="" +MMX_DEFAULT="" +MMX_BLACKHOLE="" +MM_BLACKHOLE="" + +MMX_UNREACHABLE="" +MM_UNREACHABLE="" MAX_SLEEP=$(((1<<31)-1)) LOG() @@ -21,6 +32,21 @@ LOG() [ "$facility" = "debug" ] && return logger -t "${SCRIPTNAME}[$$]" -p $facility "$*" } + +mwan3_get_true_iface() +{ + local family V + _true_iface=$2 + config_get family "$2" family ipv4 + if [ "$family" = "ipv4" ]; then + V=4 + elif [ "$family" = "ipv6" ]; then + V=6 + fi + ubus call "network.interface.${2}_${V}" status &>/dev/null && _true_iface="${2}_${V}" + export "$1=$_true_iface" +} + mwan3_get_src_ip() { local family _src_ip true_iface device addr_cmd default_ip IP sed_str @@ -149,4 +175,3 @@ mwan3_count_one_bits() done echo $count } ->>>>>>> 2a4e0dc6d... review comments diff --git a/net/mwan3/files/lib/mwan3/mwan3.sh b/net/mwan3/files/lib/mwan3/mwan3.sh index c3113007a..2a689c0c8 100644 --- a/net/mwan3/files/lib/mwan3/mwan3.sh +++ b/net/mwan3/files/lib/mwan3/mwan3.sh @@ -22,16 +22,7 @@ IPv6_REGEX="${IPv6_REGEX}::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0- IPv6_REGEX="${IPv6_REGEX}([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])" IPv4_REGEX="((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" -MWAN3_STATUS_DIR="/var/run/mwan3" -MWAN3_INTERFACE_MAX="" DEFAULT_LOWEST_METRIC=256 -MMX_MASK="" -MMX_DEFAULT="" -MMX_BLACKHOLE="" -MM_BLACKHOLE="" - -MMX_UNREACHABLE="" -MM_UNREACHABLE="" command -v ip6tables > /dev/null NO_IPV6=$? @@ -80,20 +71,6 @@ mwan3_update_iface_to_table() config_foreach update_table interface } -mwan3_get_true_iface() -{ - local family V - _true_iface=$2 - config_get family "$iface" family ipv4 - if [ "$family" = "ipv4" ]; then - V=4 - elif [ "$family" = "ipv6" ]; then - V=6 - fi - ubus call "network.interface.${iface}_${V}" status &>/dev/null && _true_iface="${iface}_${V}" - export "$1=$_true_iface" -} - mwan3_route_line_dev() { # must have mwan3 config already loaded @@ -129,63 +106,6 @@ mwan3_count_one_bits() echo $count } -# maps the 1st parameter so it only uses the bits allowed by the bitmask (2nd parameter) -# which means spreading the bits of the 1st parameter to only use the bits that are set to 1 in the 2nd parameter -# 0 0 0 0 0 1 0 1 (0x05) 1st parameter -# 1 0 1 0 1 0 1 0 (0xAA) 2nd parameter -# 1 0 1 result -mwan3_id2mask() -{ - local bit_msk bit_val result - bit_val=0 - result=0 - for bit_msk in $(seq 0 31); do - if [ $((($2>>bit_msk)&1)) = "1" ]; then - if [ $((($1>>bit_val)&1)) = "1" ]; then - result=$((result|(1< "${MWAN3_STATUS_DIR}/mmx_mask" - LOG debug "Using firewall mask ${MMX_MASK}" - - bitcnt=$(mwan3_count_one_bits MMX_MASK) - mmdefault=$(((1<&1) || LOG error "set_connected_ipv6: $error" } @@ -384,15 +300,10 @@ mwan3_set_general_iptables() -p ipv6-icmp \ -m icmp6 --icmpv6-type 137 \ -j RETURN - # do not mangle outgoing echo request - mwan3_push_update -A mwan3_hook \ - -m set --match-set mwan3_source_v6 src \ - -p ipv6-icmp \ - -m icmp6 --icmpv6-type 128 \ - -j RETURN fi mwan3_push_update -A mwan3_hook \ + -m mark --mark 0x0/$MMX_MASK \ -j CONNMARK --restore-mark --nfmask "$MMX_MASK" --ctmask "$MMX_MASK" mwan3_push_update -A mwan3_hook \ -m mark --mark 0x0/$MMX_MASK \ diff --git a/net/mwan3/files/usr/sbin/mwan3track b/net/mwan3/files/usr/sbin/mwan3track index 32b741ee2..e1f185200 100755 --- a/net/mwan3/files/usr/sbin/mwan3track +++ b/net/mwan3/files/usr/sbin/mwan3track @@ -19,6 +19,11 @@ stop_subprocs() { [ -n "$TRACK_PID" ] && kill "$TRACK_PID" && unset TRACK_PID } +WRAP() { + # shellcheck disable=SC2048 + FAMILY=$FAMILY DEVICE=$DEVICE SRCIP=$SRC_IP FWMARK=$MMX_DEFAULT LD_PRELOAD=/lib/mwan3/libwrap_mwan3_sockopt.so.1.0 $* +} + clean_up() { LOG notice "Stopping mwan3track for interface \"${INTERFACE}\". Status was \"${STATUS}\"" stop_subprocs @@ -58,10 +63,6 @@ validate_track_method() { LOG warn "Missing httping. Please install httping package." return 1 } - [ -n "$2" ] && { [ "$2" = "0.0.0.0" ] || [ "$2" = "::" ]; } && { - LOG warn "Cannot determine source IP for the interface which is required by httping." - return 1 - } ;; nping-*) command -v nping 1>/dev/null 2>&1 || { @@ -76,6 +77,12 @@ validate_track_method() { esac } +validate_wrap() { + [ -x /lib/mwan3/libwrap_mwan3_sockopt.so.1.0 ] && return + LOG error "Missing libwrap_mwan3_sockopt. Please reinstall mwan3." && + exit 1 +} + disconnected() { STATUS='offline' echo "offline" > $MWAN3TRACK_STATUS_DIR/$INTERFACE/STATUS @@ -124,17 +131,6 @@ firstconnect() { mwan3_get_src_ip SRC_IP $true_iface - # pinging IPv6 hosts with an interface is troublesome - # https://bugs.openwrt.org/index.php?do=details&task_id=2897 - # https://bugs.openwrt.org/index.php?do=details&task_id=2167 - # https://forum.openwrt.org/t/ping-and-traceroute-failing-for-eth0-3-on-ipv6/44680/11 - # so use the IP address of the interface - if [ "$family" = "ipv6" ]; then - SOURCE="$SRC_IP" - else - SOURCE="$DEVICE" - fi - LOG debug "firstconnect: called on $INTERFACE/$true_iface ($DEVICE). Status is $STATUS. SRC_IP is $SRC_IP" STARTED=1 @@ -159,7 +155,8 @@ main() { local recovery_interval down up size local keep_failure_interval check_quality failure_latency local recovery_latency failure_loss recovery_loss - local max_ttl httping_ssl family track_ips + + local max_ttl httping_ssl track_ips INTERFACE=$1 STATUS="" @@ -171,9 +168,10 @@ main() { trap if_up USR2 config_load mwan3 + config_get FAMILY $INTERFACE family ipv4 config_get track_method $INTERFACE track_method ping config_get_bool httping_ssl $INTERFACE httping_ssl 0 - validate_track_method $track_method $SRC_IP || { + validate_track_method $track_method || { track_method=ping if validate_track_method $track_method; then LOG warn "Using ping to track interface $INTERFACE avaliability" @@ -219,17 +217,13 @@ main() { if [ $host_up_count -lt $reliability ]; then case "$track_method" in ping) - # pinging IPv6 hosts with an interface is troublesome - # https://bugs.openwrt.org/index.php?do=details&task_id=2897 - # so get the IP address of the interface and use that instead - if [ $check_quality -eq 0 ]; then - $PING -${family#ipv} -I ${SOURCE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip &> /dev/null & + WRAP $PING -${FAMILY#ipv} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip &> /dev/null & TRACK_PID=$! wait $TRACK_PID result=$? else - $PING -${family#ipv} -I ${SOURCE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip 2>/dev/null > $TRACK_OUTPUT & + WRAP $PING -${family#ipv} -I ${SOURCE} -c $count -W $timeout -s $size -t $max_ttl -q $track_ip 2>/dev/null > $TRACK_OUTPUT & TRACK_PID=$! wait $TRACK_PID ping_status=$? @@ -243,23 +237,23 @@ main() { fi ;; arping) - arping -I $DEVICE -c $count -w $timeout -q $track_ip &> /dev/null & + WRAP arping -I $DEVICE -c $count -w $timeout -q $track_ip &> /dev/null & TRACK_PID=$! wait $TRACK_PID result=$? ;; httping) if [ "$httping_ssl" -eq 1 ]; then - httping -y $SRC_IP -c $count -t $timeout -q "https://$track_ip" &> /dev/null & + WRAP httping -c $count -t $timeout -q "https://$track_ip" &> /dev/null & else - httping -y $SRC_IP -c $count -t $timeout -q "http://$track_ip" &> /dev/null & + WRAP httping -c $count -t $timeout -q "http://$track_ip" &> /dev/null & fi TRACK_PID=$! wait $TRACK_PID result=$? ;; nping-*) - nping -c $count $track_ip --${FAMILY#nping-} > $TRACK_OUTPUT & + WRAP nping -c $count $track_ip --${FAMILY#nping-} > $TRACK_OUTPUT & TRACK_PID=$! wait $TRACK_PID result=$(grep $TRACK_OUTPUT Lost | awk '{print $12}') diff --git a/net/mwan3/src/sockopt_wrap.c b/net/mwan3/src/sockopt_wrap.c new file mode 100644 index 000000000..695e57502 --- /dev/null +++ b/net/mwan3/src/sockopt_wrap.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020 Aaron Goodman . All Rights Reserved. + */ + +/* + * sockopt_wrap.c provides a shared library that intercepts syscalls to various + * networking functions to bind the sockets a source IP address and network device + * and to set the firewall mark on otugoing packets. Parameters are set using the + * DEVICE, SRCIP, FWMARK environment variables. + * + * Additionally the FAMILY environment variable can be set to either 'ipv4' or + * 'ipv6' to cause sockets opened with ipv6 or ipv4 to fail, respectively. + * + * Each environment variable is optional, and if not set, the library will not + * enforce the particular parameter. + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static int (*next_socket)(int domain, int type, int protocol); +static int (*next_setsockopt)(int sockfd, int level, int optname, + const void *optval, socklen_t optlen); +static int (*next_bind)(int sockfd, const struct sockaddr *addr, socklen_t addrlen); +static int (*next_close)(int fd); +static ssize_t (*next_send)(int sockfd, const void *buf, size_t len, int flags); +static ssize_t (*next_sendto)(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen); +static ssize_t (*next_sendmsg)(int sockfd, const struct msghdr *msg, int flags); +static int (*next_connect)(int sockfd, const struct sockaddr *addr, + socklen_t addrlen); +static int device=0; +static struct sockaddr_in source4 = {0}; +#ifdef CONFIG_IPV6 +static struct sockaddr_in6 source6 = {0}; +#endif +static struct sockaddr * source = 0; +static int sockaddr_size = 0; +static int is_bound [1024] = {0}; + +#define next_func(x)\ +void set_next_##x(){\ + if (next_##x) return;\ + next_##x = dlsym(RTLD_NEXT, #x);\ + dlerror_handle();\ + return;\ +} + +void dlerror_handle() +{ + char *msg; + if ((msg = dlerror()) != NULL) { + fprintf(stderr, "socket: dlopen failed : %s\n", msg); + fflush(stderr); + exit(EXIT_FAILURE); + } +} + +next_func(bind); +next_func(close); +next_func(setsockopt); +next_func(socket); +next_func(send); +next_func(sendto); +next_func(sendmsg); +next_func(connect); + +void dobind(int sockfd) +{ + if (source && sockfd < 1024 && !is_bound[sockfd]) { + set_next_bind(); + if (next_bind(sockfd, source, sockaddr_size)) { + perror("failed to bind to ip address"); + next_close(sockfd); + exit(EXIT_FAILURE); + } + is_bound[sockfd] = 1; + } +} + +int connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) +{ + set_next_connect(); + dobind(sockfd); + return next_connect(sockfd, addr, addrlen); +} + +ssize_t send(int sockfd, const void *buf, size_t len, int flags) +{ + set_next_send(); + dobind(sockfd); + return next_send(sockfd, buf, len, flags); +} + +ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + set_next_sendto(); + dobind(sockfd); + return next_sendto(sockfd, buf, len, flags, dest_addr, addrlen); +} + +ssize_t sendmsg(int sockfd, const struct msghdr *msg, int flags) +{ + set_next_sendmsg(); + dobind(sockfd); + return next_sendmsg(sockfd, msg, flags); +} + +int bind (int sockfd, const struct sockaddr *addr, socklen_t addrlen) +{ + set_next_bind(); + if (device && addr->sa_family == AF_PACKET) { + ((struct sockaddr_ll*)addr)->sll_ifindex=device; + } + else if (source && addr->sa_family == AF_INET) { + ((struct sockaddr_in*)addr)->sin_addr = source4.sin_addr; + } +#ifdef CONFIG_IPV6 + else if (source && addr->sa_family == AF_INET6) { + ((struct sockaddr_in6*)addr)->sin6_addr = source6.sin6_addr; + } +#endif + if (sockfd < 1024) + is_bound[sockfd] = 1; + return next_bind(sockfd, addr, addrlen); +} + +int close (int sockfd) +{ + set_next_close(); + if (sockfd < 1024) + is_bound[sockfd]=0; + return next_close(sockfd); +} + +int setsockopt(int sockfd, int level, int optname, const void *optval, socklen_t optlen) +{ + set_next_setsockopt(); + if (level == SOL_SOCKET && (optname == SO_MARK || optname == SO_BINDTODEVICE)) + return 0; + return next_setsockopt(sockfd, level, optname, optval, optlen); +} + +int socket(int domain, int type, int protocol) +{ + int handle; + + const char *socket_str = getenv("DEVICE"); + const char *srcip_str = getenv("SRCIP"); + const char *fwmark_str = getenv("FWMARK"); + const char *family_str = getenv("FAMILY"); + const int iface_len = socket_str ? strnlen(socket_str, IFNAMSIZ) : 0; + int has_family = family_str && *family_str != 0; + int has_srcip = srcip_str && *srcip_str != 0; + const int fwmark = fwmark_str ? (int)strtol(fwmark_str, NULL, 0) : 0; + + set_next_close(); + set_next_socket(); + set_next_send(); + set_next_setsockopt(); + set_next_sendmsg(); + set_next_sendto(); + set_next_connect(); + if(has_family) { +#ifdef CONFIG_IPV6 + if(domain == AF_INET && strncmp(family_str,"ipv6",4) == 0) + return -1; +#endif + if(domain == AF_INET6 && strncmp(family_str,"ipv4",4) == 0) + return -1; + } + + if (domain != AF_INET +#ifdef CONFIG_IPV6 + && domain != AF_INET6 +#endif + ) { + return next_socket(domain, type, protocol); + } + + + if (iface_len > 0) { + if (iface_len == IFNAMSIZ) { + fprintf(stderr,"socket: Too long iface name\n"); + fflush(stderr); + exit(EXIT_FAILURE); + } + } + + if (has_srcip) { + int s; + void * addr_buf; + if (domain == AF_INET) { + addr_buf = &source4.sin_addr; + sockaddr_size=sizeof source4; + memset(&source4, 0, sockaddr_size); + source4.sin_family = domain; + source = (struct sockaddr*)&source4; + } +#ifdef CONFIG_IPV6 + else { + addr_buf = &source6.sin6_addr; + sockaddr_size=sizeof source6; + memset(&source6, 0, sockaddr_size); + source6.sin6_family=domain; + source = (struct sockaddr*)&source6; + } +#endif + s = inet_pton(domain, srcip_str, addr_buf); + if (s == 0) { + fprintf(stderr, "socket: ip address invalid format for family %s\n", + domain == AF_INET ? "AF_INET" : domain == AF_INET6 ? + "AF_INET6" : "unknown"); + return -1; + } + if (s < 0) { + perror("inet_pton"); + exit(EXIT_FAILURE); + } + } + + handle = next_socket(domain, type, protocol); + if (handle == -1 ) { + return handle; + } + + if (iface_len > 0) { + device=if_nametoindex(socket_str); + if (next_setsockopt(handle, SOL_SOCKET, SO_BINDTODEVICE, + socket_str, iface_len + 1)) { + perror("socket: setting interface name failed with error"); + next_close(handle); + exit(EXIT_FAILURE); + } + } + + if (fwmark > 0) { + if (next_setsockopt(handle, SOL_SOCKET, SO_MARK, + &fwmark, sizeof fwmark)) { + perror("failed setting mark for socket"); + next_close(handle); + exit(EXIT_FAILURE); + } + } + return handle; +}