FYI: Autonomously testing bonding through TAP/VDE-switches

[Kirill Smelkov]

Hello up there,

I was studying netconf2011 slides and saw there is an LNST effort to
automatically test net, bonding, vlan, etc..., so I'd like to say thanks
and also I think I have something to share on the topic:

When I was working on bonding some time ago, it turned out that with the
help of TAP devices, virtual VDE switches and route-based send-to-self,
it is possible to create virtual nets on non-virtualized host and then
see/test how e.g. bonding works over such network.

Attached is a sample script which creates two virtual ethernet segments,
and bonds them together.

Hope it will be somewhat useful idea/sample and thanks,
Kirill

---- 8< ----
#!/bin/bash -e
# Handy program to prepare environment for testing etherdup:
#
# - setup tap interfaces and switches and plug cables between them;
# - setup bonding interfaces above taps;
# - setup routing so that send-to-self works;
# - for convenience, screen sessions with switch/cables control (yes, you can
#   break and restore cables) and tcpdump's are started;
# - everything is cleaned up on exit.
#
#
#            tap0 --- switch0 --- tap1
#          / 1.10                 1.11 \
# bond0 --                               -- bond1
#  4.10    \                           /     4.11
#            tap2 --- switch1 --- tap3
#            2.10                 2.11

# die <msg>...
die() {
    echo 1>&2 $@
    exit 1
}

verbose=
# trace <msg>
trace() {
    test -z "$verbose" || echo $@
}

# lower priority of kernel local table from 0 to 500
# (so that we can install our own rules before it)
ip rule del pref   0 lookup local 2>/dev/null || :
ip rule del pref 500 lookup local 2>/dev/null || :
ip rule add pref 500 lookup local

# ifcfg_tap <tap-name>
ifcfg_tap() {
    tap=$1
    trace ifcfg_tap $tap

# reset interface
    ip link del $tap 2>/dev/null || :

# create interface
    vde_tunctl -t $tap
}

# xifup <ifname> <ip>
xifup() {
    xif=$1
    ip=$2
    trace xifup $xif $ip

# assign IP address
    ip addr add $ip/24 dev $xif

# put if up
    ip link set $xif up
}

# if_get_ipaddr <iface>
if_get_ipaddr() {
    iface=$1

ip=$(ip -4 addr show $iface | tail -1 | awk '{print $2}')
    ip=${ip%/*}     # 192.168.1.10/24   ->  192.168.1.10

echo $ip
}

# if_get_n <iface>
# tap<n>    -- n
# bond<n>   -- 10+n
if_get_n() {
    ifx=$1

case $ifx in
        tap*)
            n=${ifx#tap}
            ;;
        bond*)
            n=${ifx#bond}
            n=$((10+$n))
            ;;
        *)
            die "unsupported iface $ifx"
            ;;
    esac

echo $n
}

# rt_setup_loop <if1> <if2>
# setup routing loop between two interfaces
# http://marc.info/?l=linux-netdev&m=129500124207256&w=2
rt_setup_loop() {
    if1=$1
    if2=$2
    trace rt_setup_loop $if1 $if2

n1=`if_get_n $if1`
    n2=`if_get_n $if2`
    ip1=`if_get_ipaddr $if1`
    ip2=`if_get_ipaddr $if2`

# on rx side handle packets by local table, so we can receive them
    echo 1 >/proc/sys/net/ipv4/conf/$if1/accept_local
    echo 1 >/proc/sys/net/ipv4/conf/$if2/accept_local
    ip rule del pref $((0+$n1)) 2>/dev/null || :
    ip rule del pref $((0+$n2)) 2>/dev/null || :
    ip rule add pref $((0+$n1)) iif $if1 lookup local
    ip rule add pref $((0+$n2)) iif $if2 lookup local

# tx
    ip rule del pref $((100+$n1)) 2>/dev/null || :
    ip rule del pref $((100+$n2)) 2>/dev/null || :
    ip rule add pref $((100+$n1)) to $ip1 lookup $((100+$n1))	# if1 <- if2
    ip rule add pref $((100+$n2)) to $ip2 lookup $((100+$n2))	# if2 <- if1

ip route flush table $((100+$n1))
    ip route flush table $((100+$n2))
    ip route add default dev $if2 table $((100+$n1))
    ip route add default dev $if1 table $((100+$n2))
}

# ifcfg_bond <bond-name> <slave1> <slave2>
ifcfg_bond() {
    bond=$1
    slave1=$2
    slave2=$3
    trace ifcfg_bond $bond $slave1 $slave2

# FIXME bonding wants slave to be initially down, why?
    ip link set $slave1 down
    ip link set $slave2 down

echo -$bond >/sys/class/net/bonding_masters 2>/dev/null || :
    echo +$bond >/sys/class/net/bonding_masters

echo broadcast >/sys/class/net/$bond/bonding/mode
    echo +$slave1  >/sys/class/net/$bond/bonding/slaves
    echo +$slave2  >/sys/class/net/$bond/bonding/slaves
}

ifcfg_tap tap0 && xifup tap0 192.168.1.10
ifcfg_tap tap1 && xifup tap1 192.168.1.11
ifcfg_tap tap2 && xifup tap2 192.168.2.10
ifcfg_tap tap3 && xifup tap3 192.168.2.11

rt_setup_loop tap0 tap1
rt_setup_loop tap2 tap3

ifcfg_bond bond0 tap0 tap2  &&  xifup bond0  192.168.4.10
ifcfg_bond bond1 tap1 tap3  &&  xifup bond1  192.168.4.11

rt_setup_loop bond0 bond1

# kill unnneeded routes from 'table main' set by `ip addr add`
for ifx in tap0 tap1 tap2 tap3 bond0 bond1 ; do
    ip route del `ip route list table main | grep $ifx`
done

# ensure (visually) we've set up it ok

echo
echo " >>> rules:"
ip rule

echo
echo " >>> tap/bond routing table:"
routel | grep '<(tap|bond).>'
#ip route show table all | grep '<(tap|bond).>'

# tx path
echo
echo " >>> checking routing for tx path:"
ip route get 192.168.1.10 connected
ip route get 192.168.1.11 connected
echo
ip route get 192.168.2.10 connected
ip route get 192.168.2.11 connected
echo
ip route get 192.168.4.10 connected
ip route get 192.168.4.11 connected

# rx path
echo
echo " >>> checking routing for rx path:"
ip route get from 192.168.1.10 to 192.168.1.11 iif tap1
ip route get from 192.168.1.11 to 192.168.1.10 iif tap0
echo
ip route get from 192.168.2.10 to 192.168.2.11 iif tap3
ip route get from 192.168.2.11 to 192.168.2.10 iif tap2
echo
ip route get from 192.168.4.10 to 192.168.4.11 iif bond1
ip route get from 192.168.4.11 to 192.168.4.10 iif bond0

echo
echo " >>> ready to start switches, connect wires, etc ..."
read

S() {
    screen -S switches -X $@
}
screen -d -m -S switches -t S0 vde_switch -s switch0

S zombie qr
S verbose on
sleep 0.2
S screen -t 'S0-tap0' vde_plug2tap -s switch0 tap0
S screen -t 'S0-tap1' vde_plug2tap -s switch0 tap1

S screen -t 'S1' vde_switch -s switch1
sleep 0.2
S screen -t 'S1-tap2' vde_plug2tap -s switch1 tap2
S screen -t 'S1-tap3' vde_plug2tap -s switch1 tap3

S() {
    screen -S dumps -X $@
    sleep 0.01  # :( or else it can miss some commands
}
screen -d -m -S dumps -t tap0 tcpdump -i tap0 -n -t -e

S zombie qr
S verbose on
S screen -t tap1 tcpdump -i tap1 -n -t -e
S screen -t tap2 tcpdump -i tap2 -n -t -e
S screen -t tap3 tcpdump -i tap3 -n -t -e
S screen -t bond0 tcpdump -i bond0 -n -t -e
S screen -t bond1 tcpdump -i bond1 -n -t -e

XTERM="xterm -fn -misc-fixed-medium-r-semicondensed--13-120-75-75-c-60-iso10646-1"
$XTERM -T "tap/bond dumps" -e screen -r dumps &
$XTERM -T "tap/bond switches" -e screen -r switches &

# wait for xterm with `screen -r dumps` started
sleep 1

# layout dumps as follows
# tap0   tap1
# tap2   tap3
# bond0  bond1
S select -
S split
S focus down
S split
S focus up
S split -v
S select tap0
S focus down
S select tap1
S focus down
S split -v
S select tap2
S focus down
S select tap3
S focus down
S split -v
S select bond0
S focus down
S select bond1

wait

# cleanup
echo " >>> cleanup..."
for ifx in bond0 bond1 tap0 tap1 tap2 tap3 ; do
    # kill 'to <if-ip>' rule
    ip rule del to `if_get_ipaddr $ifx`

# kill 'iif <if>' rule
    ip rule del iif $ifx

# kills interfaces and routes
    ip link del $ifx
done

echo done