Merge pull request #45 from alauda/feat/iface

feat: support user define iface and mtu

dist/images/start-ovs.sh

@@ -49,6 +49,5 @@ fi
 # Set remote ovn-sb for ovn-controller to connect to
 ovs-vsctl set open . external-ids:ovn-remote=tcp:${OVN_SB_SERVICE_HOST}:${OVN_SB_SERVICE_PORT}
 ovs-vsctl set open . external-ids:ovn-encap-type=geneve
-ovs-vsctl set open . external-ids:ovn-encap-ip=${POD_IP}
 
 tail -f /var/log/openvswitch/ovs-vswitchd.log

docs/install.md

@@ -50,6 +50,15 @@ For high-available ovn db, see [high available](high-available.md)
     --kubeconfig: Path to kubeconfig file with authorization and master location information. If not set use the inCluster token
 ```
 
+### Daemon Configuration
+
+```bash
+    --iface: The iface used to inter-host pod communication, default: the default route iface
+    --mtu: The MTU used by pod iface, default: iface MTU - 55
+    --service-cluster-ip-range: The kubernetes service cluster ip range, default: 10.96.0.0/12
+    --kubeconfig: Path to kubeconfig file with authorization and master location information. If not set use the inCluster token
+```
+
 ## To uninstall
 
 1. Remove Kubernetes resources:

pkg/daemon/config.go

 package daemon
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
+	"os/exec"
+	"strings"
+	"syscall"
 
+	"github.com/alauda/kube-ovn/pkg/util"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
+	"github.com/vishvananda/netlink"
+
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -16,6 +24,8 @@ import (
 
 // Configuration is the daemon conf
 type Configuration struct {
+	Iface                 string
+	MTU                   int
 	BindSocket            string
 	OvsSocket             string
 	KubeConfigFile        string
@@ -29,11 +39,13 @@ type Configuration struct {
 // TODO: validate configuration
 func ParseFlags() (*Configuration, error) {
 	var (
+		argIface                 = pflag.String("iface", "", "The iface used to inter-host pod communication, default: the default route iface")
+		argMTU                   = pflag.Int("mtu", 0, "The MTU used by pod iface, default: iface MTU - 55")
 		argBindSocket            = pflag.String("bind-socket", "/var/run/cniserver.sock", "The socket daemon bind to.")
 		argOvsSocket             = pflag.String("ovs-socket", "", "The socket to local ovs-server")
 		argKubeConfigFile        = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.")
-		argServiceClusterIPRange = pflag.String("service-cluster-ip-range", "10.96.0.0/12", "The kubernetes service cluster ip range")
-		argPprofPort             = pflag.Int("pprof-port", 10665, "The port to get profiling data, default 10665")
+		argServiceClusterIPRange = pflag.String("service-cluster-ip-range", "10.96.0.0/12", "The kubernetes service cluster ip range, default: 10.96.0.0/12")
+		argPprofPort             = pflag.Int("pprof-port", 10665, "The port to get profiling data, default: 10665")
 	)
 
 	// mute log for ipset lib
@@ -64,6 +76,8 @@ func ParseFlags() (*Configuration, error) {
 	}
 
 	config := &Configuration{
+		Iface:                 *argIface,
+		MTU:                   *argMTU,
 		BindSocket:            *argBindSocket,
 		OvsSocket:             *argOvsSocket,
 		KubeConfigFile:        *argKubeConfigFile,
@@ -71,12 +85,39 @@ func ParseFlags() (*Configuration, error) {
 		NodeName:              nodeName,
 		ServiceClusterIPRange: *argServiceClusterIPRange,
 	}
-	err := config.initKubeClient()
+
+	if config.Iface == "" {
+		iface, err := getDefaultGatewayIface()
+		if err != nil {
+			return nil, err
+		} else {
+			config.Iface = iface
+		}
+	}
+	iface, err := net.InterfaceByName(config.Iface)
 	if err != nil {
 		return nil, err
 	}
-	klog.Infof("bind socket: %s", config.BindSocket)
-	klog.Infof("ovs socket at %s", config.OvsSocket)
+	if config.MTU == 0 {
+		config.MTU = iface.MTU - util.GeneveHeaderLength
+	}
+
+	addrs, err := iface.Addrs()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get iface addr. %v", err)
+	}
+	if len(addrs) == 0 {
+		return nil, fmt.Errorf("iface %s has no ip address", config.Iface)
+	}
+	if err := setEncapIP(strings.Split(addrs[0].String(), "/")[0]); err != nil {
+		return nil, err
+	}
+
+	err = config.initKubeClient()
+	if err != nil {
+		return nil, err
+	}
+	klog.Infof("daemon config: %v", config)
 	return config, nil
 }
 
@@ -106,3 +147,34 @@ func (config *Configuration) initKubeClient() error {
 	config.KubeClient = kubeClient
 	return nil
 }
+
+func getDefaultGatewayIface() (string, error) {
+	routes, err := netlink.RouteList(nil, syscall.AF_INET)
+	if err != nil {
+		return "", err
+	}
+
+	for _, route := range routes {
+		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" {
+			if route.LinkIndex <= 0 {
+				return "", errors.New("found default route but could not determine interface")
+			}
+			iface, err := net.InterfaceByIndex(route.LinkIndex)
+			if err != nil {
+				return "", fmt.Errorf("failed to get iface %v", err)
+			}
+			return iface.Name, nil
+		}
+	}
+
+	return "", errors.New("unable to find default route")
+}
+
+func setEncapIP(ip string) error {
+	raw, err := exec.Command(
+		"ovs-vsctl", "set", "open", ".", fmt.Sprintf("external-ids:ovn-encap-ip=%s", ip)).CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("failed to set ovn-encap-ip, %s", string(raw))
+	}
+	return nil
+}

pkg/daemon/init.go

@@ -33,5 +33,5 @@ func InitNodeGateway(config *Configuration) error {
 			break
 		}
 	}
-	return configureNodeNic(portName, ipAddr, macAddr, gw)
+	return configureNodeNic(portName, ipAddr, macAddr, gw, config.MTU)
 }

pkg/daemon/ovs.go

@@ -18,7 +18,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, netns, container
 	// NOTE: DO NOT use ovs internal type interface for container.
 	// Kubernetes will detect 'eth0' nic in pod, so the nic name in pod must be 'eth0'.
 	// When renaming internal interface to 'eth0', ovs will delete and recreate this interface.
-	veth := netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostNicName, MTU: 1400}, PeerName: containerNicName}
+	veth := netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostNicName, MTU: csh.Config.MTU}, PeerName: containerNicName}
 	defer func() {
 		// Remove veth link in case any error during creating pod network.
 		if err != nil {
@@ -164,7 +164,7 @@ func configureContainerNic(nicName, ipAddr, gateway string, macAddr net.Hardware
 	})
 }
 
-func configureNodeNic(portName, ip, mac, gw string) error {
+func configureNodeNic(portName, ip, mac, gw string, mtu int) error {
 	macAddr, err := net.ParseMAC(mac)
 	if err != nil {
 		return fmt.Errorf("failed to parse mac %s %v", macAddr, err)
@@ -199,7 +199,7 @@ func configureNodeNic(portName, ip, mac, gw string) error {
 		return fmt.Errorf("can not set mac address to node nic %v", err)
 	}
 
-	err = netlink.LinkSetMTU(nodeLink, 1400)
+	err = netlink.LinkSetMTU(nodeLink, mtu)
 	if err != nil {
 		return fmt.Errorf("can not set mtu %v", err)
 	}

pkg/util/const.go

@@ -41,4 +41,7 @@ const (
 	GWCentralizedMode = "centralized"
 	GWNode            = "ovn.kubernetes.io/gateway_node"
 	GWNat             = "ovn.kubernetes.io/gateway_nat"
+
+	// Refer to http://www.openvswitch.org/support/dist-docs/ovn-architecture.7.html "Tunnel Encapsulations"
+	GeneveHeaderLength = 55
 )