diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh
index a8b660cc174eabdf949fc665e9d8f78fb8323870..0ca0b51ea96aed52f18cf9fdbc4d75d9a756b925 100755
--- a/dist/images/start-ovs.sh
+++ b/dist/images/start-ovs.sh
@@ -49,6 +49,5 @@ fi
# Set remote ovn-sb for ovn-controller to connect to
ovs-vsctl set open . external-ids:ovn-remote=tcp:${OVN_SB_SERVICE_HOST}:${OVN_SB_SERVICE_PORT}
ovs-vsctl set open . external-ids:ovn-encap-type=geneve
-ovs-vsctl set open . external-ids:ovn-encap-ip=${POD_IP}
tail -f /var/log/openvswitch/ovs-vswitchd.log
diff --git a/docs/install.md b/docs/install.md
index 380c8a8c40a1840a4e0a4720f4002966cbc66c87..f1a69c8f925824eaa8dc0c6b622bc6110fc717ef 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -50,6 +50,15 @@ For high-available ovn db, see [high available](high-available.md)
--kubeconfig: Path to kubeconfig file with authorization and master location information. If not set use the inCluster token
```
+### Daemon Configuration
+
+```bash
+ --iface: The iface used to inter-host pod communication, default: the default route iface
+ --mtu: The MTU used by pod iface, default: iface MTU - 55
+ --service-cluster-ip-range: The kubernetes service cluster ip range, default: 10.96.0.0/12
+ --kubeconfig: Path to kubeconfig file with authorization and master location information. If not set use the inCluster token
+```
+
## To uninstall
1. Remove Kubernetes resources:
diff --git a/pkg/daemon/config.go b/pkg/daemon/config.go
index 5bc038567f25e2e88cb3a7fada8b32d0779ffc7b..2d5c64ee2b7303b45203abf5b3c05fc4ab7f3959 100644
--- a/pkg/daemon/config.go
+++ b/pkg/daemon/config.go
@@ -1,13 +1,21 @@
package daemon
import (
+ "errors"
"flag"
"fmt"
"io/ioutil"
+ "net"
"os"
+ "os/exec"
+ "strings"
+ "syscall"
+ "github.com/alauda/kube-ovn/pkg/util"
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"
+ "github.com/vishvananda/netlink"
+
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
@@ -16,6 +24,8 @@ import (
// Configuration is the daemon conf
type Configuration struct {
+ Iface string
+ MTU int
BindSocket string
OvsSocket string
KubeConfigFile string
@@ -29,11 +39,13 @@ type Configuration struct {
// TODO: validate configuration
func ParseFlags() (*Configuration, error) {
var (
+ argIface = pflag.String("iface", "", "The iface used to inter-host pod communication, default: the default route iface")
+ argMTU = pflag.Int("mtu", 0, "The MTU used by pod iface, default: iface MTU - 55")
argBindSocket = pflag.String("bind-socket", "/var/run/cniserver.sock", "The socket daemon bind to.")
argOvsSocket = pflag.String("ovs-socket", "", "The socket to local ovs-server")
argKubeConfigFile = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.")
- argServiceClusterIPRange = pflag.String("service-cluster-ip-range", "10.96.0.0/12", "The kubernetes service cluster ip range")
- argPprofPort = pflag.Int("pprof-port", 10665, "The port to get profiling data, default 10665")
+ argServiceClusterIPRange = pflag.String("service-cluster-ip-range", "10.96.0.0/12", "The kubernetes service cluster ip range, default: 10.96.0.0/12")
+ argPprofPort = pflag.Int("pprof-port", 10665, "The port to get profiling data, default: 10665")
)
// mute log for ipset lib
@@ -64,6 +76,8 @@ func ParseFlags() (*Configuration, error) {
}
config := &Configuration{
+ Iface: *argIface,
+ MTU: *argMTU,
BindSocket: *argBindSocket,
OvsSocket: *argOvsSocket,
KubeConfigFile: *argKubeConfigFile,
@@ -71,12 +85,39 @@ func ParseFlags() (*Configuration, error) {
NodeName: nodeName,
ServiceClusterIPRange: *argServiceClusterIPRange,
}
- err := config.initKubeClient()
+
+ if config.Iface == "" {
+ iface, err := getDefaultGatewayIface()
+ if err != nil {
+ return nil, err
+ } else {
+ config.Iface = iface
+ }
+ }
+ iface, err := net.InterfaceByName(config.Iface)
if err != nil {
return nil, err
}
- klog.Infof("bind socket: %s", config.BindSocket)
- klog.Infof("ovs socket at %s", config.OvsSocket)
+ if config.MTU == 0 {
+ config.MTU = iface.MTU - util.GeneveHeaderLength
+ }
+
+ addrs, err := iface.Addrs()
+ if err != nil {
+ return nil, fmt.Errorf("failed to get iface addr. %v", err)
+ }
+ if len(addrs) == 0 {
+ return nil, fmt.Errorf("iface %s has no ip address", config.Iface)
+ }
+ if err := setEncapIP(strings.Split(addrs[0].String(), "/")[0]); err != nil {
+ return nil, err
+ }
+
+ err = config.initKubeClient()
+ if err != nil {
+ return nil, err
+ }
+ klog.Infof("daemon config: %v", config)
return config, nil
}
@@ -106,3 +147,34 @@ func (config *Configuration) initKubeClient() error {
config.KubeClient = kubeClient
return nil
}
+
+func getDefaultGatewayIface() (string, error) {
+ routes, err := netlink.RouteList(nil, syscall.AF_INET)
+ if err != nil {
+ return "", err
+ }
+
+ for _, route := range routes {
+ if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" {
+ if route.LinkIndex <= 0 {
+ return "", errors.New("found default route but could not determine interface")
+ }
+ iface, err := net.InterfaceByIndex(route.LinkIndex)
+ if err != nil {
+ return "", fmt.Errorf("failed to get iface %v", err)
+ }
+ return iface.Name, nil
+ }
+ }
+
+ return "", errors.New("unable to find default route")
+}
+
+func setEncapIP(ip string) error {
+ raw, err := exec.Command(
+ "ovs-vsctl", "set", "open", ".", fmt.Sprintf("external-ids:ovn-encap-ip=%s", ip)).CombinedOutput()
+ if err != nil {
+ return fmt.Errorf("failed to set ovn-encap-ip, %s", string(raw))
+ }
+ return nil
+}
diff --git a/pkg/daemon/init.go b/pkg/daemon/init.go
index 1527fe321670a1eaecd7f9a8dacfde56f8ef1682..71bb27c71bcb731309a8f340c3ab2d556e414ff3 100644
--- a/pkg/daemon/init.go
+++ b/pkg/daemon/init.go
@@ -33,5 +33,5 @@ func InitNodeGateway(config *Configuration) error {
break
}
}
- return configureNodeNic(portName, ipAddr, macAddr, gw)
+ return configureNodeNic(portName, ipAddr, macAddr, gw, config.MTU)
}
diff --git a/pkg/daemon/ovs.go b/pkg/daemon/ovs.go
index 18e28f98659ab708fcddeb7b6eb5c6ad9fa085b5..a1dfadb08101b937f596a3c4d0819cf854296328 100644
--- a/pkg/daemon/ovs.go
+++ b/pkg/daemon/ovs.go
@@ -18,7 +18,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, netns, container
// NOTE: DO NOT use ovs internal type interface for container.
// Kubernetes will detect 'eth0' nic in pod, so the nic name in pod must be 'eth0'.
// When renaming internal interface to 'eth0', ovs will delete and recreate this interface.
- veth := netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostNicName, MTU: 1400}, PeerName: containerNicName}
+ veth := netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: hostNicName, MTU: csh.Config.MTU}, PeerName: containerNicName}
defer func() {
// Remove veth link in case any error during creating pod network.
if err != nil {
@@ -164,7 +164,7 @@ func configureContainerNic(nicName, ipAddr, gateway string, macAddr net.Hardware
})
}
-func configureNodeNic(portName, ip, mac, gw string) error {
+func configureNodeNic(portName, ip, mac, gw string, mtu int) error {
macAddr, err := net.ParseMAC(mac)
if err != nil {
return fmt.Errorf("failed to parse mac %s %v", macAddr, err)
@@ -199,7 +199,7 @@ func configureNodeNic(portName, ip, mac, gw string) error {
return fmt.Errorf("can not set mac address to node nic %v", err)
}
- err = netlink.LinkSetMTU(nodeLink, 1400)
+ err = netlink.LinkSetMTU(nodeLink, mtu)
if err != nil {
return fmt.Errorf("can not set mtu %v", err)
}
diff --git a/pkg/util/const.go b/pkg/util/const.go
index 87df53c11436012a2a79365f2f0d76477b4117e4..76417f56f0d99aa49ec80a7ad5db7cd705a567d1 100644
--- a/pkg/util/const.go
+++ b/pkg/util/const.go
@@ -41,4 +41,7 @@ const (
GWCentralizedMode = "centralized"
GWNode = "ovn.kubernetes.io/gateway_node"
GWNat = "ovn.kubernetes.io/gateway_nat"
+
+ // Refer to http://www.openvswitch.org/support/dist-docs/ovn-architecture.7.html "Tunnel Encapsulations"
+ GeneveHeaderLength = 55
)