support to add multiple subnets for a namespace

pkg/controller/namespace.go

@@ -71,6 +71,10 @@ func (c *Controller) enqueueUpdateNamespace(old, new interface{}) {
 		klog.Warningf("no logical switch annotation for ns %s", newNs.Name)
 		c.addNamespaceQueue.Add(newNs.Name)
 	}
+
+	if newNs.Annotations != nil && newNs.Annotations[util.LogicalSwitchAnnotation] != "" && !reflect.DeepEqual(oldNs.Annotations, newNs.Annotations) {
+		c.addNamespaceQueue.Add(newNs.Name)
+	}
 }
 
 func (c *Controller) runAddNamespaceWorker() {
@@ -119,8 +123,8 @@ func (c *Controller) handleAddNamespace(key string) error {
 		return err
 	}
 
-	var ls, cidr string
-	var excludeIps []string
+	var ls string
+	var lss []string
 	subnets, err := c.subnetsLister.List(labels.Everything())
 	if err != nil {
 		klog.Errorf("failed to list subnets %v", err)
@@ -130,18 +134,12 @@ func (c *Controller) handleAddNamespace(key string) error {
 	for _, s := range subnets {
 		for _, ns := range s.Spec.Namespaces {
 			if ns == key {
-				ls = s.Name
-				cidr = s.Spec.CIDRBlock
-				excludeIps = s.Spec.ExcludeIps
-				break
+				lss = append(lss, s.Name)
 			}
 		}
-		if ls != "" {
-			break
-		}
 	}
 
-	if ls == "" {
+	if lss == nil {
 		// If NS does not belong to any custom VPC, then this NS belongs to the default VPC
 		vpc, err := c.vpcsLister.Get(c.config.ClusterRouter)
 		if err != nil {
@@ -170,10 +168,7 @@ func (c *Controller) handleAddNamespace(key string) error {
 			klog.Errorf("failed to get default subnet %v", err)
 			return err
 		}
-		ls = subnet.Name
-		cidr = subnet.Spec.CIDRBlock
-		excludeIps = subnet.Spec.ExcludeIps
-
+		lss = append(lss, subnet.Name)
 	}
 
 	op := "replace"
@@ -181,16 +176,11 @@ func (c *Controller) handleAddNamespace(key string) error {
 		op = "add"
 		namespace.Annotations = map[string]string{}
 	} else {
-		if namespace.Annotations[util.LogicalSwitchAnnotation] == ls &&
-			namespace.Annotations[util.CidrAnnotation] == cidr &&
-			namespace.Annotations[util.ExcludeIpsAnnotation] == strings.Join(excludeIps, ",") {
+		if namespace.Annotations[util.LogicalSwitchAnnotation] == strings.Join(lss, ",") {
 			return nil
 		}
 	}
-
-	namespace.Annotations[util.LogicalSwitchAnnotation] = ls
-	namespace.Annotations[util.CidrAnnotation] = cidr
-	namespace.Annotations[util.ExcludeIpsAnnotation] = strings.Join(excludeIps, ",")
+	namespace.Annotations[util.LogicalSwitchAnnotation] = strings.Join(lss, ",")
 
 	if _, err = c.config.KubeClient.CoreV1().Namespaces().Patch(context.Background(), key, types.JSONPatchType, generatePatchPayload(namespace.Annotations, op), metav1.PatchOptions{}, ""); err != nil {
 		klog.Errorf("patch namespace %s failed %v", key, err)

pkg/controller/pod.go

@@ -863,11 +863,16 @@ func needAllocateSubnets(pod *v1.Pod, nets []*kubeovnNet) []*kubeovnNet {
 }
 
 func (c *Controller) getPodDefaultSubnet(pod *v1.Pod) (*kubeovnv1.Subnet, error) {
-	var subnetName string
+	var subnet *kubeovnv1.Subnet
+	var err error
 	// 1. check annotation subnet
 	lsName, lsExist := pod.Annotations[util.LogicalSwitchAnnotation]
 	if lsExist {
-		subnetName = lsName
+		subnet, err = c.subnetsLister.Get(lsName)
+		if err != nil {
+			klog.Errorf("failed to get subnet %v", err)
+			return nil, err
+		}
 	} else {
 		ns, err := c.namespacesLister.Get(pod.Namespace)
 		if err != nil {
@@ -880,18 +885,35 @@ func (c *Controller) getPodDefaultSubnet(pod *v1.Pod) (*kubeovnv1.Subnet, error)
 			return nil, err
 		}
 
-		subnetName = ns.Annotations[util.LogicalSwitchAnnotation]
-		if subnetName == "" {
-			err = fmt.Errorf("namespace default logical switch is not found")
-			klog.Error(err)
-			return nil, err
-		}
-	}
+		subnetNames := ns.Annotations[util.LogicalSwitchAnnotation]
+		for _, subnetName := range strings.Split(subnetNames, ",") {
+			if subnetName == "" {
+				err = fmt.Errorf("namespace %s default logical switch is not found", pod.Namespace)
+				klog.Error(err)
+				return nil, err
+			}
+			subnet, err = c.subnetsLister.Get(subnetName)
+			if err != nil {
+				klog.Errorf("failed to get subnet %v", err)
+				return nil, err
+			}
 
-	subnet, err := c.subnetsLister.Get(subnetName)
-	if err != nil {
-		klog.Errorf("failed to get subnet %v", err)
-		return nil, err
+			switch subnet.Spec.Protocol {
+			case kubeovnv1.ProtocolIPv4:
+				fallthrough
+			case kubeovnv1.ProtocolDual:
+				if subnet.Status.V4AvailableIPs == 0 {
+					klog.V(3).Infof("there's no available ips for subnet %v, try next subnet", subnet.Name)
+					continue
+				}
+			case kubeovnv1.ProtocolIPv6:
+				if subnet.Status.V6AvailableIPs == 0 {
+					klog.Infof("there's no available ips for subnet %v, try next subnet", subnet.Name)
+					continue
+				}
+			}
+			break
+		}
 	}
 	return subnet, nil
 }
@@ -1179,7 +1201,9 @@ func appendCheckStatefulSetPodToDel(c *Controller, pod *v1.Pod) (bool, error) {
 		klog.Errorf("failed to get namespace %s, %v", pod.Namespace, err)
 		return false, err
 	}
-	if podNs.Annotations[util.LogicalSwitchAnnotation] != "" && pod.Annotations[util.LogicalSwitchAnnotation] != "" && strings.TrimSpace(podNs.Annotations[util.LogicalSwitchAnnotation]) != strings.TrimSpace(pod.Annotations[util.LogicalSwitchAnnotation]) {
+
+	subnetNames := podNs.Annotations[util.LogicalSwitchAnnotation]
+	if subnetNames != "" && pod.Annotations[util.LogicalSwitchAnnotation] != "" && !util.ContainsString(strings.Split(subnetNames, ","), strings.TrimSpace(pod.Annotations[util.LogicalSwitchAnnotation])) {
 		klog.Infof("ns %s annotation subnet is %s, which is inconstant with subnet for pod %s, delete pod", pod.Namespace, podNs.Annotations[util.LogicalSwitchAnnotation], pod.Name)
 		return true, nil
 	}

pkg/controller/subnet.go

@@ -680,7 +680,8 @@ func (c *Controller) handleDeleteLogicalSwitch(key string) (err error) {
 		if annotations == nil {
 			continue
 		}
-		if annotations[util.LogicalSwitchAnnotation] == key {
+
+		if util.ContainsString(strings.Split(annotations[util.LogicalSwitchAnnotation], ","), key) {
 			c.enqueueAddNamespace(ns)
 		}
 	}
@@ -778,47 +779,13 @@ func (c *Controller) reconcileSubnet(subnet *kubeovnv1.Subnet) error {
 
 func (c *Controller) reconcileNamespaces(subnet *kubeovnv1.Subnet) error {
 	var err error
-	// 1. unbind from previous subnet
-	subnets, err := c.subnetsLister.List(labels.Everything())
-	if err != nil {
-		return err
-	}
-
-	namespaceMap := map[string]bool{}
-	for _, ns := range subnet.Spec.Namespaces {
-		namespaceMap[ns] = true
-	}
-
-	for _, sub := range subnets {
-		if sub.Name == subnet.Name || len(sub.Spec.Namespaces) == 0 {
-			continue
-		}
-
-		changed := false
-		reservedNamespaces := []string{}
-		for _, ns := range sub.Spec.Namespaces {
-			if namespaceMap[ns] {
-				changed = true
-			} else {
-				reservedNamespaces = append(reservedNamespaces, ns)
-			}
-		}
-		if changed {
-			sub.Spec.Namespaces = reservedNamespaces
-			_, err = c.config.KubeOvnClient.KubeovnV1().Subnets().Update(context.Background(), sub, metav1.UpdateOptions{})
-			if err != nil {
-				klog.Errorf("failed to unbind namespace from subnet %s, %v", sub.Name, err)
-				return err
-			}
-		}
-	}
 
-	// 2. add annotations to bind namespace
+	// 1. add annotations to bind namespace
 	for _, ns := range subnet.Spec.Namespaces {
 		c.addNamespaceQueue.Add(ns)
 	}
 
-	// 3. update unbind namespace annotation
+	// 2. update unbind namespace annotation
 	namespaces, err := c.namespacesLister.List(labels.Everything())
 	if err != nil {
 		klog.Errorf("failed to list namespaces, %v", err)
@@ -826,7 +793,7 @@ func (c *Controller) reconcileNamespaces(subnet *kubeovnv1.Subnet) error {
 	}
 
 	for _, ns := range namespaces {
-		if ns.Annotations != nil && ns.Annotations[util.LogicalSwitchAnnotation] == subnet.Name && !namespaceMap[ns.Name] {
+		if ns.Annotations != nil && util.ContainsString(strings.Split(ns.Annotations[util.LogicalSwitchAnnotation], ","), subnet.Name) {
 			c.addNamespaceQueue.Add(ns.Name)
 		}
 	}