Develop custom vpc-dns (#1662)

* 新增switch lb rule功能

* 新增自定义coredns功能

* 修复ovn switch lb更新删除功能

* 优化vpc dns

* Optimize the code

* Optimize the code

* Optimize the code

* fix crd used by CoreDNS

Optimize the code

* Optimize the code

* Optimize the code
Co-authored-by: chenjianhui <chenjianhui@yealink.com>

dist/images/cleanup.sh

@@ -29,6 +29,10 @@ for slr in $(kubectl get switch-lb-rule -o name); do
    kubectl delete --ignore-not-found $slr
 done
 
+for vd in $(kubectl  get vpc-dns -o name); do
+  kubectl delete --ignore-not-found $vd
+done
+
 for vip in $(kubectl get vip -o name); do
    kubectl delete --ignore-not-found $vip
 done
@@ -79,11 +83,17 @@ kubectl delete --ignore-not-found sa ovn -n kube-system
 kubectl delete --ignore-not-found clusterrole system:ovn
 kubectl delete --ignore-not-found clusterrolebinding ovn
 
+# delete vpc-dns content
+kubectl delete --ignore-not-found cm vpc-dns-config -n kube-system
+kubectl delete --ignore-not-found clusterrole system:vpc-dns
+kubectl delete --ignore-not-found clusterrolebinding vpc-dns
+kubectl delete --ignore-not-found sa vpc-dns -n kube-system
+
 # delete CRD
 kubectl delete --ignore-not-found crd htbqoses.kubeovn.io security-groups.kubeovn.io ips.kubeovn.io subnets.kubeovn.io \
                                       vpc-nat-gateways.kubeovn.io vpcs.kubeovn.io vlans.kubeovn.io provider-networks.kubeovn.io \
                                       iptables-dnat-rules.kubeovn.io  iptables-eips.kubeovn.io  iptables-fip-rules.kubeovn.io \
-                                      iptables-snat-rules.kubeovn.io vips.kubeovn.io switch-lb-rules.kubeovn.io
+                                      iptables-snat-rules.kubeovn.io vips.kubeovn.io switch-lb-rules.kubeovn.io vpc-dnses.kubeovn.io
 
 # Remove annotations/labels in namespaces and nodes
 kubectl annotate no --all ovn.kubernetes.io/cidr-

dist/images/install.sh

@@ -208,6 +208,144 @@ addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{p
 echo "Install OVN DB in $addresses"
 
 cat <<EOF > kube-ovn-crd.yaml
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: vpc-dnses.kubeovn.io
+spec:
+  group: kubeovn.io
+  names:
+    plural: vpc-dnses
+    singular: vpc-dns
+    shortNames:
+      - vpc-dns
+    kind: VpcDns
+    listKind: VpcDnsList
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .status.active
+          name: Active
+          type: boolean
+        - jsonPath: .spec.vpc
+          name: Vpc
+          type: string
+        - jsonPath: .spec.subnet
+          name: Subnet
+          type: string
+      name: v1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                vpc:
+                  type: string
+                subnet:
+                  type: string
+            status:
+              type: object
+              properties:
+                active:
+                  type: boolean
+                conditions:
+                  type: array
+                  items:
+                    type: object
+                    properties:
+                      type:
+                        type: string
+                      status:
+                        type: string
+                      reason:
+                        type: string
+                      message:
+                        type: string
+                      lastUpdateTime:
+                        type: string
+                      lastTransitionTime:
+                        type: string
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: switch-lb-rules.kubeovn.io
+spec:
+  group: kubeovn.io
+  names:
+    plural: switch-lb-rules
+    singular: switch-lb-rule
+    shortNames:
+      - slr
+    kind: SwitchLBRule
+    listKind: SwitchLBRuleList
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .spec.vip
+          name: vip
+          type: string
+        - jsonPath: .status.ports
+          name: port(s)
+          type: string
+        - jsonPath: .status.service
+          name: service
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: age
+          type: date
+      name: v1
+      served: true
+      storage: true
+      subresources:
+        status: {}
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                namespace:
+                  type: string
+                vip:
+                  type: string
+                sessionAffinity:
+                  type: string
+                ports:
+                  items:
+                    properties:
+                      name:
+                        type: string
+                      port:
+                        type: integer
+                        minimum: 1
+                        maximum: 65535
+                      protocol:
+                        type: string
+                      targetPort:
+                        type: integer
+                        minimum: 1
+                        maximum: 65535
+                    type: object
+                  type: array
+                selector:
+                  items:
+                    type: string
+                  type: array
+            status:
+              type: object
+              properties:
+                ports:
+                  type: string
+                service:
+                  type: string
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -1494,6 +1632,8 @@ rules:
       - iptables-snat-rules/status
       - switch-lb-rules
       - switch-lb-rules/status
+      - vpc-dnses
+      - vpc-dnses/status
     verbs:
       - "*"
   - apiGroups:
@@ -1990,6 +2130,8 @@ rules:
       - iptables-fip-rules/status
       - iptables-dnat-rules/status
       - iptables-snat-rules/status
+      - vpc-dnses
+      - vpc-dnses/status
       - switch-lb-rules
       - switch-lb-rules/status
     verbs:

pkg/apis/kubeovn/v1/register.go

@@ -59,6 +59,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&HtbQosList{},
 		&SwitchLBRule{},
 		&SwitchLBRuleList{},
+		&VpcDns{},
+		&VpcDnsList{},
 	)
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
 	return nil

pkg/apis/kubeovn/v1/types.go

@@ -860,6 +860,63 @@ type VipList struct {
 	Items []Vip `json:"items"`
 }
 
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +genclient:nonNamespaced
+// +resourceName=vpc-dnses
+
+type VpcDns struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   VpcDnsSpec   `json:"spec"`
+	Status VpcDnsStatus `json:"status,omitempty"`
+}
+
+type VpcDnsSpec struct {
+	Vpc    string `json:"vpc"`
+	Subnet string `json:"subnet"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+type VpcDnsList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []VpcDns `json:"items"`
+}
+
+type VpcDnsStatus struct {
+	// +optional
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	Conditions []VpcDnsCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
+
+	Active bool `json:"active" patchStrategy:"merge"`
+}
+
+// Condition describes the state of an object at a certain point.
+// +k8s:deepcopy-gen=true
+type VpcDnsCondition struct {
+	// Type of condition.
+	Type ConditionType `json:"type"`
+	// Status of the condition, one of True, False, Unknown.
+	Status corev1.ConditionStatus `json:"status"`
+	// The reason for the condition's last transition.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+	// A human readable message indicating details about the transition.
+	// +optional
+	Message string `json:"message,omitempty"`
+	// Last time the condition was probed
+	// +optional
+	LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
+	// Last time the condition transitioned from one status to another.
+	// +optional
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+}
+
 type SlrPort struct {
 	Name       string `json:"name"`
 	Port       int32  `json:"port"`

pkg/apis/kubeovn/v1/zz_generated.deepcopy.go

@@ -1583,6 +1583,124 @@ func (in *VpcCondition) DeepCopy() *VpcCondition {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VpcDns) DeepCopyInto(out *VpcDns) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Spec = in.Spec
+	in.Status.DeepCopyInto(&out.Status)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDns.
+func (in *VpcDns) DeepCopy() *VpcDns {
+	if in == nil {
+		return nil
+	}
+	out := new(VpcDns)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VpcDns) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VpcDnsCondition) DeepCopyInto(out *VpcDnsCondition) {
+	*out = *in
+	in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime)
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsCondition.
+func (in *VpcDnsCondition) DeepCopy() *VpcDnsCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(VpcDnsCondition)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VpcDnsList) DeepCopyInto(out *VpcDnsList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]VpcDns, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsList.
+func (in *VpcDnsList) DeepCopy() *VpcDnsList {
+	if in == nil {
+		return nil
+	}
+	out := new(VpcDnsList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *VpcDnsList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VpcDnsSpec) DeepCopyInto(out *VpcDnsSpec) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsSpec.
+func (in *VpcDnsSpec) DeepCopy() *VpcDnsSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(VpcDnsSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VpcDnsStatus) DeepCopyInto(out *VpcDnsStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]VpcDnsCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsStatus.
+func (in *VpcDnsStatus) DeepCopy() *VpcDnsStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(VpcDnsStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *VpcList) DeepCopyInto(out *VpcList) {
 	*out = *in

pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go

@@ -80,6 +80,10 @@ func (c *FakeKubeovnV1) Vpcs() v1.VpcInterface {
 	return &FakeVpcs{c}
 }
 
+func (c *FakeKubeovnV1) VpcDnses() v1.VpcDnsInterface {
+	return &FakeVpcDnses{c}
+}
+
 func (c *FakeKubeovnV1) VpcNatGateways() v1.VpcNatGatewayInterface {
 	return &FakeVpcNatGateways{c}
 }

pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcdns.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	"context"
+
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+)
+
+// FakeVpcDnses implements VpcDnsInterface
+type FakeVpcDnses struct {
+	Fake *FakeKubeovnV1
+}
+
+var vpcdnsesResource = schema.GroupVersionResource{Group: "kubeovn.io", Version: "v1", Resource: "vpc-dnses"}
+
+var vpcdnsesKind = schema.GroupVersionKind{Group: "kubeovn.io", Version: "v1", Kind: "VpcDns"}
+
+// Get takes name of the vpcDns, and returns the corresponding vpcDns object, and an error if there is any.
+func (c *FakeVpcDnses) Get(ctx context.Context, name string, options v1.GetOptions) (result *kubeovnv1.VpcDns, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootGetAction(vpcdnsesResource, name), &kubeovnv1.VpcDns{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*kubeovnv1.VpcDns), err
+}
+
+// List takes label and field selectors, and returns the list of VpcDnses that match those selectors.
+func (c *FakeVpcDnses) List(ctx context.Context, opts v1.ListOptions) (result *kubeovnv1.VpcDnsList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootListAction(vpcdnsesResource, vpcdnsesKind, opts), &kubeovnv1.VpcDnsList{})
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &kubeovnv1.VpcDnsList{ListMeta: obj.(*kubeovnv1.VpcDnsList).ListMeta}
+	for _, item := range obj.(*kubeovnv1.VpcDnsList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested vpcDnses.
+func (c *FakeVpcDnses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewRootWatchAction(vpcdnsesResource, opts))
+}
+
+// Create takes the representation of a vpcDns and creates it.  Returns the server's representation of the vpcDns, and an error, if there is any.
+func (c *FakeVpcDnses) Create(ctx context.Context, vpcDns *kubeovnv1.VpcDns, opts v1.CreateOptions) (result *kubeovnv1.VpcDns, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootCreateAction(vpcdnsesResource, vpcDns), &kubeovnv1.VpcDns{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*kubeovnv1.VpcDns), err
+}
+
+// Update takes the representation of a vpcDns and updates it. Returns the server's representation of the vpcDns, and an error, if there is any.
+func (c *FakeVpcDnses) Update(ctx context.Context, vpcDns *kubeovnv1.VpcDns, opts v1.UpdateOptions) (result *kubeovnv1.VpcDns, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootUpdateAction(vpcdnsesResource, vpcDns), &kubeovnv1.VpcDns{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*kubeovnv1.VpcDns), err
+}
+
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *FakeVpcDnses) UpdateStatus(ctx context.Context, vpcDns *kubeovnv1.VpcDns, opts v1.UpdateOptions) (*kubeovnv1.VpcDns, error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootUpdateSubresourceAction(vpcdnsesResource, "status", vpcDns), &kubeovnv1.VpcDns{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*kubeovnv1.VpcDns), err
+}
+
+// Delete takes name of the vpcDns and deletes it. Returns an error if one occurs.
+func (c *FakeVpcDnses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewRootDeleteActionWithOptions(vpcdnsesResource, name, opts), &kubeovnv1.VpcDns{})
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeVpcDnses) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	action := testing.NewRootDeleteCollectionAction(vpcdnsesResource, listOpts)
+
+	_, err := c.Fake.Invokes(action, &kubeovnv1.VpcDnsList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched vpcDns.
+func (c *FakeVpcDnses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *kubeovnv1.VpcDns, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootPatchSubresourceAction(vpcdnsesResource, name, pt, data, subresources...), &kubeovnv1.VpcDns{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*kubeovnv1.VpcDns), err
+}

pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go

@@ -44,4 +44,6 @@ type VlanExpansion interface{}
 
 type VpcExpansion interface{}
 
+type VpcDnsExpansion interface{}
+
 type VpcNatGatewayExpansion interface{}

pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go

@@ -41,6 +41,7 @@ type KubeovnV1Interface interface {
 	VipsGetter
 	VlansGetter
 	VpcsGetter
+	VpcDnsesGetter
 	VpcNatGatewaysGetter
 }
 
@@ -101,6 +102,10 @@ func (c *KubeovnV1Client) Vpcs() VpcInterface {
 	return newVpcs(c)
 }
 
+func (c *KubeovnV1Client) VpcDnses() VpcDnsInterface {
+	return newVpcDnses(c)
+}
+
 func (c *KubeovnV1Client) VpcNatGateways() VpcNatGatewayInterface {
 	return newVpcNatGateways(c)
 }

pkg/client/clientset/versioned/typed/kubeovn/v1/vpcdns.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	"context"
+	"time"
+
+	v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+)
+
+// VpcDnsesGetter has a method to return a VpcDnsInterface.
+// A group's client should implement this interface.
+type VpcDnsesGetter interface {
+	VpcDnses() VpcDnsInterface
+}
+
+// VpcDnsInterface has methods to work with VpcDns resources.
+type VpcDnsInterface interface {
+	Create(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.CreateOptions) (*v1.VpcDns, error)
+	Update(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (*v1.VpcDns, error)
+	UpdateStatus(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (*v1.VpcDns, error)
+	Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error
+	DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error
+	Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.VpcDns, error)
+	List(ctx context.Context, opts metav1.ListOptions) (*v1.VpcDnsList, error)
+	Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcDns, err error)
+	VpcDnsExpansion
+}
+
+// vpcDnses implements VpcDnsInterface
+type vpcDnses struct {
+	client rest.Interface
+}
+
+// newVpcDnses returns a VpcDnses
+func newVpcDnses(c *KubeovnV1Client) *vpcDnses {
+	return &vpcDnses{
+		client: c.RESTClient(),
+	}
+}
+
+// Get takes name of the vpcDns, and returns the corresponding vpcDns object, and an error if there is any.
+func (c *vpcDnses) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.VpcDns, err error) {
+	result = &v1.VpcDns{}
+	err = c.client.Get().
+		Resource("vpc-dnses").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of VpcDnses that match those selectors.
+func (c *vpcDnses) List(ctx context.Context, opts metav1.ListOptions) (result *v1.VpcDnsList, err error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	result = &v1.VpcDnsList{}
+	err = c.client.Get().
+		Resource("vpc-dnses").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested vpcDnses.
+func (c *vpcDnses) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	opts.Watch = true
+	return c.client.Get().
+		Resource("vpc-dnses").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Watch(ctx)
+}
+
+// Create takes the representation of a vpcDns and creates it.  Returns the server's representation of the vpcDns, and an error, if there is any.
+func (c *vpcDnses) Create(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.CreateOptions) (result *v1.VpcDns, err error) {
+	result = &v1.VpcDns{}
+	err = c.client.Post().
+		Resource("vpc-dnses").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(vpcDns).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Update takes the representation of a vpcDns and updates it. Returns the server's representation of the vpcDns, and an error, if there is any.
+func (c *vpcDnses) Update(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (result *v1.VpcDns, err error) {
+	result = &v1.VpcDns{}
+	err = c.client.Put().
+		Resource("vpc-dnses").
+		Name(vpcDns.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(vpcDns).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// UpdateStatus was generated because the type contains a Status member.
+// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
+func (c *vpcDnses) UpdateStatus(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (result *v1.VpcDns, err error) {
+	result = &v1.VpcDns{}
+	err = c.client.Put().
+		Resource("vpc-dnses").
+		Name(vpcDns.Name).
+		SubResource("status").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(vpcDns).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Delete takes name of the vpcDns and deletes it. Returns an error if one occurs.
+func (c *vpcDnses) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error {
+	return c.client.Delete().
+		Resource("vpc-dnses").
+		Name(name).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *vpcDnses) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error {
+	var timeout time.Duration
+	if listOpts.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
+	}
+	return c.client.Delete().
+		Resource("vpc-dnses").
+		VersionedParams(&listOpts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// Patch applies the patch and returns the patched vpcDns.
+func (c *vpcDnses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcDns, err error) {
+	result = &v1.VpcDns{}
+	err = c.client.Patch(pt).
+		Resource("vpc-dnses").
+		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(data).
+		Do(ctx).
+		Into(result)
+	return
+}

pkg/client/informers/externalversions/generic.go

@@ -79,6 +79,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().Vlans().Informer()}, nil
 	case v1.SchemeGroupVersion.WithResource("vpcs"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().Vpcs().Informer()}, nil
+	case v1.SchemeGroupVersion.WithResource("vpc-dnses"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcDnses().Informer()}, nil
 	case v1.SchemeGroupVersion.WithResource("vpc-nat-gateways"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcNatGateways().Informer()}, nil
 

pkg/client/informers/externalversions/kubeovn/v1/interface.go

@@ -50,6 +50,8 @@ type Interface interface {
 	Vlans() VlanInformer
 	// Vpcs returns a VpcInformer.
 	Vpcs() VpcInformer
+	// VpcDnses returns a VpcDnsInformer.
+	VpcDnses() VpcDnsInformer
 	// VpcNatGateways returns a VpcNatGatewayInformer.
 	VpcNatGateways() VpcNatGatewayInformer
 }
@@ -130,6 +132,11 @@ func (v *version) Vpcs() VpcInformer {
 	return &vpcInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
 }
 
+// VpcDnses returns a VpcDnsInformer.
+func (v *version) VpcDnses() VpcDnsInformer {
+	return &vpcDnsInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
+}
+
 // VpcNatGateways returns a VpcNatGatewayInformer.
 func (v *version) VpcNatGateways() VpcNatGatewayInformer {
 	return &vpcNatGatewayInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}

pkg/client/informers/externalversions/kubeovn/v1/vpcdns.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	"context"
+	time "time"
+
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned"
+	internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces"
+	v1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+)
+
+// VpcDnsInformer provides access to a shared informer and lister for
+// VpcDnses.
+type VpcDnsInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1.VpcDnsLister
+}
+
+type vpcDnsInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// NewVpcDnsInformer constructs a new informer for VpcDns type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewVpcDnsInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredVpcDnsInformer(client, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredVpcDnsInformer constructs a new informer for VpcDns type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredVpcDnsInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.KubeovnV1().VpcDnses().List(context.TODO(), options)
+			},
+			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.KubeovnV1().VpcDnses().Watch(context.TODO(), options)
+			},
+		},
+		&kubeovnv1.VpcDns{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *vpcDnsInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredVpcDnsInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *vpcDnsInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&kubeovnv1.VpcDns{}, f.defaultInformer)
+}
+
+func (f *vpcDnsInformer) Lister() v1.VpcDnsLister {
+	return v1.NewVpcDnsLister(f.Informer().GetIndexer())
+}

pkg/client/listers/kubeovn/v1/expansion_generated.go

@@ -70,6 +70,10 @@ type VlanListerExpansion interface{}
 // VpcLister.
 type VpcListerExpansion interface{}
 
+// VpcDnsListerExpansion allows custom methods to be added to
+// VpcDnsLister.
+type VpcDnsListerExpansion interface{}
+
 // VpcNatGatewayListerExpansion allows custom methods to be added to
 // VpcNatGatewayLister.
 type VpcNatGatewayListerExpansion interface{}

pkg/client/listers/kubeovn/v1/vpcdns.go

+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1
+
+import (
+	v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+)
+
+// VpcDnsLister helps list VpcDnses.
+// All objects returned here must be treated as read-only.
+type VpcDnsLister interface {
+	// List lists all VpcDnses in the indexer.
+	// Objects returned here must be treated as read-only.
+	List(selector labels.Selector) (ret []*v1.VpcDns, err error)
+	// Get retrieves the VpcDns from the index for a given name.
+	// Objects returned here must be treated as read-only.
+	Get(name string) (*v1.VpcDns, error)
+	VpcDnsListerExpansion
+}
+
+// vpcDnsLister implements the VpcDnsLister interface.
+type vpcDnsLister struct {
+	indexer cache.Indexer
+}
+
+// NewVpcDnsLister returns a new VpcDnsLister.
+func NewVpcDnsLister(indexer cache.Indexer) VpcDnsLister {
+	return &vpcDnsLister{indexer: indexer}
+}
+
+// List lists all VpcDnses in the indexer.
+func (s *vpcDnsLister) List(selector labels.Selector) (ret []*v1.VpcDns, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1.VpcDns))
+	})
+	return ret, err
+}
+
+// Get retrieves the VpcDns from the index for a given name.
+func (s *vpcDnsLister) Get(name string) (*v1.VpcDns, error) {
+	obj, exists, err := s.indexer.GetByKey(name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1.Resource("vpcdns"), name)
+	}
+	return obj.(*v1.VpcDns), nil
+}

pkg/controller/controller.go

@@ -76,6 +76,11 @@ type Controller struct {
 	UpdateSwitchLBRuleQueue workqueue.RateLimitingInterface
 	delSwitchLBRuleQueue    workqueue.RateLimitingInterface
 
+	vpcDnsLister           kubeovnlister.VpcDnsLister
+	vpcDnsSynced           cache.InformerSynced
+	addOrUpdateVpcDnsQueue workqueue.RateLimitingInterface
+	delVpcDnsQueue         workqueue.RateLimitingInterface
+
 	subnetsLister           kubeovnlister.SubnetLister
 	subnetSynced            cache.InformerSynced
 	addOrUpdateSubnetQueue  workqueue.RateLimitingInterface
@@ -398,6 +403,17 @@ func NewController(config *Configuration) *Controller {
 			UpdateFunc: controller.enqueueUpdateSwitchLBRule,
 			DeleteFunc: controller.enqueueDeleteSwitchLBRule,
 		})
+
+		vpcDnsInformer := kubeovnInformerFactory.Kubeovn().V1().VpcDnses()
+		controller.vpcDnsLister = vpcDnsInformer.Lister()
+		controller.vpcDnsSynced = vpcDnsInformer.Informer().HasSynced
+		controller.addOrUpdateVpcDnsQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "AddOrUpdateVpcDns")
+		controller.delVpcDnsQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "DeleteVpcDns")
+		vpcDnsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+			AddFunc:    controller.enqueueAddVpcDns,
+			UpdateFunc: controller.enqueueUpdateVpcDns,
+			DeleteFunc: controller.enqueueDeleteVpcDns,
+		})
 	}
 
 	subnetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -502,7 +518,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) {
 	}
 
 	if c.config.EnableLb {
-		cacheSyncs = append(cacheSyncs, c.switchLBRuleSynced)
+		cacheSyncs = append(cacheSyncs, c.switchLBRuleSynced, c.vpcDnsSynced)
 	}
 
 	if ok := cache.WaitForCacheSync(stopCh, cacheSyncs...); !ok {
@@ -555,6 +571,13 @@ func (c *Controller) Run(stopCh <-chan struct{}) {
 	if err := c.initSyncCrdVlans(); err != nil {
 		klog.Errorf("failed to sync crd vlans: %v", err)
 	}
+
+	if c.config.EnableLb {
+		if err := c.initVpcDnsConfig(); err != nil {
+			klog.Errorf("failed to init vpc-dns: %v", err)
+		}
+	}
+
 	// The static route for node gw can be deleted when gc static route, so add it after gc process
 	dstIp := "0.0.0.0/0,::/0"
 	if err := c.ovnLegacyClient.AddStaticRoute("", dstIp, c.config.NodeSwitchGateway, c.config.ClusterRouter, util.NormalRouteType); err != nil {
@@ -615,6 +638,9 @@ func (c *Controller) shutdown() {
 		c.addSwitchLBRuleQueue.ShutDown()
 		c.delSwitchLBRuleQueue.ShutDown()
 		c.UpdateSwitchLBRuleQueue.ShutDown()
+
+		c.addOrUpdateVpcDnsQueue.ShutDown()
+		c.delVpcDnsQueue.ShutDown()
 	}
 
 	c.addVirtualIpQueue.ShutDown()
@@ -719,6 +745,12 @@ func (c *Controller) startWorkers(stopCh <-chan struct{}) {
 		go wait.Until(c.runAddSwitchLBRuleWorker, time.Second, stopCh)
 		go wait.Until(c.runDelSwitchLBRuleWorker, time.Second, stopCh)
 		go wait.Until(c.runUpdateSwitchLBRuleWorker, time.Second, stopCh)
+
+		go wait.Until(c.runAddOrUpdateVpcDnsWorker, time.Second, stopCh)
+		go wait.Until(c.runDelVpcDnsWorker, time.Second, stopCh)
+		go wait.Until(func() {
+			c.resyncVpcDnsConfig()
+		}, 5*time.Second, stopCh)
 	}
 
 	for i := 0; i < c.config.WorkerNum; i++ {

pkg/controller/gc.go

@@ -35,6 +35,7 @@ func (c *Controller) gc() error {
 		c.gcLogicalRouterPort,
 		c.gcVip,
 		c.gcLbSvcPods,
+		c.gcVpcDns,
 	}
 	for _, gcFunc := range gcFunctions {
 		if err := gcFunc(); err != nil {
@@ -766,3 +767,73 @@ func (c *Controller) gcLbSvcPods() error {
 	}
 	return nil
 }
+
+func (c *Controller) gcVpcDns() error {
+	if !c.config.EnableLb {
+		return nil
+	}
+
+	klog.Infof("start to gc vpc dns")
+	vds, err := c.vpcDnsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to list vpc-dns, %v", err)
+		return err
+	}
+
+	sel, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.VpcDnsNameLabel: "true"}})
+
+	deps, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).List(context.Background(), metav1.ListOptions{
+		LabelSelector: sel.String(),
+	})
+	if err != nil {
+		klog.Errorf("failed to list vpc-dns deployment, %s", err)
+		return err
+	}
+
+	for _, dep := range deps.Items {
+		canFind := false
+		for _, vd := range vds {
+			name := genVpcDnsDpName(vd.Name)
+			if dep.Name == name {
+				canFind = true
+				break
+			}
+		}
+		if !canFind {
+			err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Delete(context.Background(),
+				dep.Name, metav1.DeleteOptions{})
+			if err != nil {
+				klog.Errorf("failed to delete vpc-dns deployment, %s", err)
+				return err
+			}
+		}
+	}
+
+	slrs, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().List(context.Background(), metav1.ListOptions{
+		LabelSelector: sel.String(),
+	})
+	if err != nil {
+		klog.Errorf("failed to list vpc-dns SwitchLBRules, %s", err)
+		return err
+	}
+
+	for _, slr := range slrs.Items {
+		canFind := false
+		for _, vd := range vds {
+			name := genVpcDnsDpName(vd.Name)
+			if slr.Name == name {
+				canFind = true
+				break
+			}
+		}
+		if !canFind {
+			err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Delete(context.Background(),
+				slr.Name, metav1.DeleteOptions{})
+			if err != nil {
+				klog.Errorf("failed to delete vpc-dns SwitchLBRule, %s", err)
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/controller/vpc_dns.go

+package controller
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/kubeovn/kube-ovn/versions"
+	"io"
+	"io/ioutil"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"net/http"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	"github.com/kubeovn/kube-ovn/pkg/util"
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/klog/v2"
+)
+
+var (
+	corednsYamlUrl  = ""
+	corednsImage    = ""
+	corednsVip      = ""
+	nadName         = ""
+	nadProvider     = ""
+	cmVersion       = ""
+	k8sServiceHost  = ""
+	k8sServicePort  = ""
+	enableCoredns   = false
+	hostNameservers []string
+)
+
+const (
+	CorednsContainerName = "coredns"
+	CorednsLabelKey      = "k8s-app"
+	CorednsTemplateDep   = "coredns-template.yaml"
+	InitRouteImage       = "kubeovn/vpc-nat-gateway:v1.11.0"
+)
+
+func genVpcDnsDpName(name string) string {
+	return fmt.Sprintf("vpc-dns-%s", name)
+}
+
+func hostConfigFromReader() error {
+	file, err := os.Open("/etc/resolv.conf")
+	if err != nil {
+		return err
+	}
+	defer func(file *os.File) {
+		if err := file.Close(); err != nil {
+			klog.Errorf("failed to close file, %s", err)
+		}
+	}(file)
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		if err := scanner.Err(); err != nil {
+			return err
+		}
+		line := scanner.Text()
+		f := strings.Fields(line)
+		if len(f) < 1 {
+			continue
+		}
+		if f[0] == "nameserver" && len(f) > 1 {
+			name := f[1]
+			hostNameservers = append(hostNameservers, name)
+		}
+	}
+
+	return err
+}
+
+func (c *Controller) enqueueAddVpcDns(obj interface{}) {
+	if !c.isLeader() {
+		return
+	}
+	var key string
+	var err error
+	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	klog.V(3).Infof("enqueue add vpc-dns %s", key)
+	c.addOrUpdateVpcDnsQueue.Add(key)
+}
+
+func (c *Controller) enqueueUpdateVpcDns(old, new interface{}) {
+	if !c.isLeader() {
+		return
+	}
+	var key string
+	var err error
+	if key, err = cache.MetaNamespaceKeyFunc(new); err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+
+	oldVpcDns := old.(*kubeovnv1.VpcDns)
+	newVpcDns := new.(*kubeovnv1.VpcDns)
+	if oldVpcDns.ResourceVersion != newVpcDns.ResourceVersion &&
+		!reflect.DeepEqual(oldVpcDns.Spec, newVpcDns.Spec) {
+		klog.V(3).Infof("enqueue update vpc-dns %s", key)
+		c.addOrUpdateVpcDnsQueue.Add(key)
+	}
+}
+
+func (c *Controller) enqueueDeleteVpcDns(obj interface{}) {
+	if !c.isLeader() {
+		return
+	}
+	var key string
+	var err error
+	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
+		utilruntime.HandleError(err)
+		return
+	}
+	klog.V(3).Infof("enqueue delete vpc-dns %s", key)
+	c.delVpcDnsQueue.Add(key)
+}
+
+func (c *Controller) runAddOrUpdateVpcDnsWorker() {
+	for c.processNextWorkItem("addOrUpdateVpcDns", c.addOrUpdateVpcDnsQueue, c.handleAddOrUpdateVpcDns) {
+	}
+}
+
+func (c *Controller) runDelVpcDnsWorker() {
+	for c.processNextWorkItem("delVpcDns", c.delVpcDnsQueue, c.handleDelVpcDns) {
+	}
+}
+
+func (c *Controller) handleAddOrUpdateVpcDns(key string) error {
+	klog.V(3).Infof("handleAddOrUpdateVpcDns %s", key)
+	if !enableCoredns {
+		time.Sleep(10 * time.Second)
+		if !enableCoredns {
+			return fmt.Errorf("failed to  add/update vpc-dns, enable ='%v'", enableCoredns)
+		}
+	}
+
+	vpcDns, err := c.vpcDnsLister.Get(key)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
+	defer func() {
+		newVpcDns := vpcDns.DeepCopy()
+		newVpcDns.Status.Active = true
+		if err != nil {
+			newVpcDns.Status.Active = false
+		}
+
+		_, err = c.config.KubeOvnClient.KubeovnV1().VpcDnses().UpdateStatus(context.Background(),
+			newVpcDns, metav1.UpdateOptions{})
+		if err != nil {
+			klog.Errorf("update vpc-dns status failed, %v", err)
+		}
+	}()
+
+	if len(corednsImage) == 0 {
+		err := fmt.Errorf("failed to get the vpc-dns coredns image parameter")
+		klog.Errorf("failed to get corednsImage, err: %s", err)
+		return err
+	}
+
+	if len(corednsVip) == 0 {
+		err := fmt.Errorf("the configuration parameter corednsVip is empty")
+		klog.Errorf("failed to get corednsVip, err: %s", err)
+		return err
+	}
+
+	if _, err := c.vpcsLister.Get(vpcDns.Spec.Vpc); err != nil {
+		klog.Errorf("failed to get vpc '%s', err: %v", vpcDns.Spec.Vpc, err)
+		return err
+	}
+
+	if _, err := c.subnetsLister.Get(vpcDns.Spec.Subnet); err != nil {
+		klog.Errorf("failed to get subnet '%s', err: %v", vpcDns.Spec.Subnet, err)
+		return err
+	}
+
+	if err := c.checkOvnNad(); err != nil {
+		klog.Errorf("failed to check nad, %v", err)
+		return err
+	}
+
+	if err := c.checkOvnProvided(); err != nil {
+		klog.Errorf("failed to check %s provided, %v", util.DefaultSubnet, err)
+		return err
+	}
+
+	if err := c.checkVpcDnsDuplicated(vpcDns); err != nil {
+		klog.Errorf("failed to deploy %s, %v", vpcDns.Name, err)
+		return err
+	}
+
+	if err := c.createOrUpdateVpcDnsDep(vpcDns); err != nil {
+		return err
+	}
+
+	if err := c.createOrUpdateVpcDnsSlr(vpcDns); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Controller) handleDelVpcDns(key string) error {
+	klog.V(3).Infof("handleDelVpcDns,%s", key)
+	name := genVpcDnsDpName(key)
+	err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Delete(context.Background(), name, metav1.DeleteOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		klog.Errorf("failed to delete Deployments: %v", err)
+		return err
+	}
+
+	err = c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Delete(context.Background(), name, metav1.DeleteOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		klog.Errorf("failed to delete SwitchLBRule: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (c *Controller) checkVpcDnsDuplicated(vpcDns *kubeovnv1.VpcDns) error {
+	vpcDnsList, err := c.vpcDnsLister.List(labels.Everything())
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+
+	for _, item := range vpcDnsList {
+		if item.Status.Active &&
+			item.Name != vpcDns.Name &&
+			item.Spec.Vpc == vpcDns.Spec.Vpc {
+			err = fmt.Errorf("only one vpc-dns can be deployed in a vpc")
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Controller) createOrUpdateVpcDnsDep(vpcDns *kubeovnv1.VpcDns) error {
+	needToCreateDp := false
+	oldDp, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).
+		Get(context.Background(), genVpcDnsDpName(vpcDns.Name), metav1.GetOptions{})
+
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			needToCreateDp = true
+		} else {
+			return err
+		}
+	}
+
+	newDp, err := c.genVpcDnsDeployment(vpcDns, oldDp)
+	if err != nil {
+		klog.Errorf("failed to generate vpc-dns deployment, %v", err)
+		return err
+	}
+
+	if needToCreateDp {
+		_, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).
+			Create(context.Background(), newDp, metav1.CreateOptions{})
+
+		if err != nil {
+			klog.Errorf("failed to create deployment '%s', err: %s", newDp.Name, err)
+			return err
+		}
+	} else {
+		_, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).
+			Update(context.Background(), newDp, metav1.UpdateOptions{})
+
+		if err != nil {
+			klog.Errorf("failed to update deployment '%s', err: %v", newDp.Name, err)
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Controller) createOrUpdateVpcDnsSlr(vpcDns *kubeovnv1.VpcDns) error {
+	needToCreateSlr := false
+	oldSlr, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Get(context.Background(),
+		genVpcDnsDpName(vpcDns.Name), metav1.GetOptions{})
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			needToCreateSlr = true
+		} else {
+			return err
+		}
+	}
+
+	newSlr, err := c.genVpcDnsSlr(vpcDns.Name, c.config.PodNamespace)
+	if err != nil {
+		klog.Errorf("failed to generate vpc-dns switchLBRule, %v", err)
+		return err
+	}
+
+	if needToCreateSlr {
+		_, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Create(context.Background(), newSlr, metav1.CreateOptions{})
+		if err != nil {
+			klog.Errorf("failed to create switchLBRules '%s', err: %v", newSlr.Name, err)
+			return err
+		}
+	} else {
+		if reflect.DeepEqual(oldSlr.Spec, newSlr.Spec) {
+			return nil
+		}
+
+		newSlr.ResourceVersion = oldSlr.ResourceVersion
+		_, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Update(context.Background(), newSlr, metav1.UpdateOptions{})
+		if err != nil {
+			klog.Errorf("failed to update switchLBRules '%s', err: %v", newSlr.Name, err)
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Controller) genVpcDnsDeployment(vpcDns *kubeovnv1.VpcDns, oldDeploy *v1.Deployment) (*v1.Deployment, error) {
+	if _, err := os.Stat(CorednsTemplateDep); errors.Is(err, os.ErrNotExist) {
+		if err := getCoreDnsTemplateFile(corednsYamlUrl); err != nil {
+			klog.Errorf("failed to get coredns template file, %v", err)
+			return nil, err
+		}
+	}
+
+	tmp, err := template.ParseFiles(CorednsTemplateDep)
+	if err != nil {
+		return nil, err
+	}
+
+	buffer := new(bytes.Buffer)
+	name := genVpcDnsDpName(vpcDns.Name)
+	if err := tmp.Execute(buffer, map[string]interface{}{
+		"DeployName":   name,
+		"CorednsImage": corednsImage,
+	}); err != nil {
+		return nil, err
+	}
+
+	dep := &v1.Deployment{}
+	retJson, err := yaml.ToJSON(buffer.Bytes())
+	if err != nil {
+		klog.Errorf("failed to switch yaml, %v", err)
+		return nil, err
+	}
+
+	if err := json.Unmarshal(retJson, dep); err != nil {
+		klog.Errorf("failed to switch json, %v", err)
+		return nil, err
+	}
+
+	dep.Spec.Template.Annotations = make(map[string]string)
+
+	if oldDeploy != nil && len(oldDeploy.Annotations) != 0 {
+		dep.Spec.Template.Annotations = oldDeploy.Annotations
+	}
+
+	dep.ObjectMeta.Labels = map[string]string{
+		util.VpcDnsNameLabel: "true",
+	}
+
+	setCoreDnsEnv(dep)
+	setVpcDnsInterface(dep, vpcDns.Spec.Subnet)
+
+	defaultSubnet, err := c.subnetsLister.Get(util.DefaultSubnet)
+	if err != nil {
+		klog.Errorf("failed to get default subnet %v", err)
+		return nil, err
+	}
+
+	setVpcDnsRoute(dep, defaultSubnet.Spec.Gateway)
+	return dep, nil
+}
+
+func (c *Controller) genVpcDnsSlr(vpcName, namespace string) (*kubeovnv1.SwitchLBRule, error) {
+	name := genVpcDnsDpName(vpcName)
+	label := fmt.Sprintf("%s:%s", CorednsLabelKey, name)
+
+	ports := []kubeovnv1.SlrPort{
+		{Name: "dns", Port: 53, Protocol: "UDP"},
+		{Name: "dns-tcp", Port: 53, Protocol: "TCP"},
+		{Name: "metrics", Port: 9153, Protocol: "TCP"},
+	}
+
+	slr := &kubeovnv1.SwitchLBRule{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+			Labels: map[string]string{
+				util.VpcDnsNameLabel: "true",
+			},
+		},
+		Spec: kubeovnv1.SwitchLBRuleSpec{
+			Vip:             corednsVip,
+			Namespace:       namespace,
+			Selector:        []string{label},
+			SessionAffinity: "",
+			Ports:           ports,
+		},
+	}
+
+	return slr, nil
+}
+
+func setVpcDnsInterface(dp *v1.Deployment, subnetName string) {
+	annotations := dp.Spec.Template.Annotations
+	annotations[util.LogicalSwitchAnnotation] = subnetName
+	annotations[util.AttachmentNetworkAnnotation] = fmt.Sprintf("%s/%s", corev1.NamespaceDefault, nadName)
+	annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, nadProvider)] = util.DefaultSubnet
+}
+
+func setCoreDnsEnv(dp *v1.Deployment) {
+	var env []corev1.EnvVar
+
+	if len(k8sServiceHost) != 0 {
+		env = append(env, corev1.EnvVar{Name: "KUBERNETES_SERVICE_HOST", Value: k8sServiceHost})
+	}
+
+	if len(k8sServicePort) != 0 {
+		env = append(env, corev1.EnvVar{Name: "KUBERNETES_SERVICE_PORT", Value: k8sServicePort})
+	}
+
+	for i, container := range dp.Spec.Template.Spec.Containers {
+		if container.Name == CorednsContainerName {
+			dp.Spec.Template.Spec.Containers[i].Env = env
+			break
+		}
+	}
+}
+
+func setVpcDnsRoute(dp *v1.Deployment, subnetGw string) {
+	var serviceHost string
+	if len(k8sServiceHost) == 0 {
+		serviceHost = "${KUBERNETES_SERVICE_HOST}"
+	} else {
+		serviceHost = k8sServiceHost
+	}
+
+	var routeCmd string
+	routeCmd = fmt.Sprintf("ip route add %s via %s dev net1;", serviceHost, subnetGw)
+	for _, nameserver := range hostNameservers {
+		routeCmd += fmt.Sprintf("ip route add %s via %s dev net1;", nameserver, subnetGw)
+	}
+
+	privileged := true
+	allowPrivilegeEscalation := true
+	dp.Spec.Template.Spec.InitContainers = append(dp.Spec.Template.Spec.InitContainers, corev1.Container{
+		Name:            "init-route",
+		Image:           InitRouteImage,
+		Command:         []string{"sh", "-c", routeCmd},
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		SecurityContext: &corev1.SecurityContext{
+			Privileged:               &privileged,
+			AllowPrivilegeEscalation: &allowPrivilegeEscalation,
+		},
+	})
+}
+
+func (c *Controller) checkOvnNad() error {
+	_, err := c.config.AttachNetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(corev1.NamespaceDefault).
+		Get(context.Background(), nadName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Controller) checkOvnProvided() error {
+	cachedSubnet, err := c.subnetsLister.Get(util.DefaultSubnet)
+	if err != nil {
+		return fmt.Errorf("failed to get default subnet %v", err)
+	}
+
+	if cachedSubnet.Spec.Provider != nadProvider {
+		return fmt.Errorf("the %s provider does not exist", nadProvider)
+	}
+
+	return nil
+}
+
+func (c *Controller) resyncVpcDnsConfig() {
+	cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.VpcDnsConfig)
+	if err != nil && !k8serrors.IsNotFound(err) {
+		klog.Errorf("failed to get %s, %v", util.VpcDnsConfig, err)
+		return
+	}
+
+	if k8serrors.IsNotFound(err) {
+		klog.V(3).Infof("the vpc-dns configuration is not set ")
+		if len(cmVersion) != 0 {
+			if err := c.cleanVpcDns(); err != nil {
+				klog.Errorf("failed to clear all vpc-dns, %v", err)
+				return
+			}
+			cmVersion = ""
+		}
+		return
+	}
+
+	if cmVersion == cm.ResourceVersion {
+		return
+	} else {
+		cmVersion = cm.ResourceVersion
+		klog.V(3).Infof("the vpc-dns ConfigMap update")
+	}
+
+	getValue := func(key string) string {
+		if v, ok := cm.Data[key]; ok {
+			return v
+		}
+		return ""
+	}
+
+	corednsImage = getValue("coredns-image")
+	if len(corednsImage) == 0 {
+		defaultImage, err := c.getDefaultCoreDnsImage()
+		if err != nil {
+			klog.Errorf("failed to get kube-system/coredns image, %s", err)
+			return
+		}
+		corednsImage = defaultImage
+		klog.V(3).Infof("use the cluster default coredns image version, %s", corednsImage)
+	}
+
+	newTemplateUrl := getValue("coredns-template")
+	if len(newTemplateUrl) != 0 && newTemplateUrl != corednsYamlUrl {
+		if err := getCoreDnsTemplateFile(newTemplateUrl); err != nil {
+			klog.Errorf("failed to get coredns template file, %v", err)
+		}
+		corednsYamlUrl = newTemplateUrl
+	}
+
+	nadName = getValue("nad-name")
+	nadProvider = getValue("nad-provider")
+	corednsVip = getValue("coredns-vip")
+	k8sServiceHost = getValue("k8s-service-host")
+	k8sServicePort = getValue("k8s-service-port")
+
+	newEnableCoredns := true
+	if v, ok := cm.Data["enable-vpc-dns"]; ok {
+		raw, err := strconv.ParseBool(v)
+		if err != nil {
+			klog.Errorf("failed to parse cm enable, %v", err)
+			return
+		}
+		newEnableCoredns = raw
+	}
+
+	if enableCoredns && !newEnableCoredns {
+		if err := c.cleanVpcDns(); err != nil {
+			klog.Errorf("failed to clear all vpc-dns, %v", err)
+			return
+		}
+	} else {
+		if err := c.updateVpcDns(); err != nil {
+			klog.Errorf("failed to update vpc-dns deployment")
+			return
+		}
+	}
+
+	enableCoredns = newEnableCoredns
+}
+
+func (c *Controller) getDefaultCoreDnsImage() (string, error) {
+	dp, err := c.config.KubeClient.AppsV1().Deployments("kube-system").
+		Get(context.Background(), "coredns", metav1.GetOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	for _, container := range dp.Spec.Template.Spec.Containers {
+		if container.Name == CorednsContainerName {
+			return container.Image, nil
+		}
+	}
+
+	return "", fmt.Errorf("coredns container no fonud")
+}
+
+func (c *Controller) initVpcDnsConfig() error {
+	url := "https://raw.githubusercontent.com/kubeovn/kube-ovn/%s/yamls/coredns-template.yaml"
+	corednsYamlUrl = fmt.Sprintf(url, versions.VERSION)
+
+	if err := hostConfigFromReader(); err != nil {
+		klog.Errorf("failed to get get host nameserver, %v", err)
+		return err
+	}
+
+	c.resyncVpcDnsConfig()
+	return nil
+}
+
+func (c *Controller) cleanVpcDns() error {
+	klog.Infof("clear all vpc-dns")
+	err := c.config.KubeOvnClient.KubeovnV1().VpcDnses().DeleteCollection(context.Background(), metav1.DeleteOptions{},
+		metav1.ListOptions{})
+	if err != nil {
+		klog.Errorf("Failed to clear all vpc-dns %s", err)
+		return err
+	}
+
+	return nil
+}
+
+func (c *Controller) updateVpcDns() error {
+	list, err := c.vpcDnsLister.List(labels.Everything())
+	if err != nil {
+		klog.Errorf("failed to get vpc-dns list, %s", err)
+		return err
+	}
+
+	for _, vd := range list {
+		c.addOrUpdateVpcDnsQueue.Add(vd.Name)
+	}
+	return nil
+}
+
+func getCoreDnsTemplateFile(url string) error {
+	client := http.Client{
+		Timeout: 5 * time.Second,
+	}
+
+	resp, err := client.Get(url)
+	if err != nil {
+		return err
+	}
+	defer func(Body io.ReadCloser) {
+		err := Body.Close()
+		if err != nil {
+			klog.Errorf("failed to close http, %s", err)
+		}
+	}(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("access errors, return code:%d", resp.StatusCode)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(CorednsTemplateDep, data, 0644)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/util/const.go

@@ -80,14 +80,14 @@ const (
 
 	OvsDpTypeLabel = "ovn.kubernetes.io/ovs_dp_type"
 
-	SubnetNameLabel        = "ovn.kubernetes.io/subnet"
-	ICGatewayLabel         = "ovn.kubernetes.io/ic-gw"
-	ExGatewayLabel         = "ovn.kubernetes.io/external-gw"
-	VpcNatGatewayLabel     = "ovn.kubernetes.io/vpc-nat-gw"
-	IpReservedLabel        = "ovn.kubernetes.io/ip_reserved"
-	VpcNatGatewayNameLabel = "ovn.kubernetes.io/vpc-nat-gw-name"
-	VpcLbLabel             = "ovn.kubernetes.io/vpc_lb"
-
+	SubnetNameLabel            = "ovn.kubernetes.io/subnet"
+	ICGatewayLabel             = "ovn.kubernetes.io/ic-gw"
+	ExGatewayLabel             = "ovn.kubernetes.io/external-gw"
+	VpcNatGatewayLabel         = "ovn.kubernetes.io/vpc-nat-gw"
+	IpReservedLabel            = "ovn.kubernetes.io/ip_reserved"
+	VpcNatGatewayNameLabel     = "ovn.kubernetes.io/vpc-nat-gw-name"
+	VpcLbLabel                 = "ovn.kubernetes.io/vpc_lb"
+	VpcDnsNameLabel            = "ovn.kubernetes.io/vpc-dns"
 	NetworkPolicyLogAnnotation = "ovn.kubernetes.io/enable_log"
 
 	ProtocolTCP = "tcp"
@@ -132,6 +132,8 @@ const (
 	VpcNatGatewayConfig    = "ovn-vpc-nat-gw-config"
 	VpcExternalNet         = "ovn-vpc-external-network"
 	VpcLbNetworkAttachment = "ovn-vpc-lb"
+	VpcDnsConfig           = "vpc-dns-config"
+	VpcDnsDepTemplate      = "vpc-dns-dep"
 
 	DefaultVpc    = "ovn-cluster"
 	DefaultSubnet = "ovn-default"

yamls/coredns-template.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .DeployName }}
+  labels:
+    k8s-app: {{ .DeployName }}
+spec:
+  replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      k8s-app: {{ .DeployName }}
+  template:
+    metadata:
+      labels:
+        k8s-app: {{ .DeployName }}
+    spec:
+      priorityClassName: system-cluster-critical
+      serviceAccountName: vpc-dns
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      nodeSelector:
+        kubernetes.io/os: linux
+      affinity:
+         podAntiAffinity:
+           requiredDuringSchedulingIgnoredDuringExecution:
+           - labelSelector:
+               matchExpressions:
+               - key: k8s-app
+                 operator: In
+                 values: ["{{ .DeployName }}"]
+             topologyKey: kubernetes.io/hostname
+      containers:
+      - name: coredns
+        image: {{ .CorednsImage }}
+        imagePullPolicy: IfNotPresent
+        resources:
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        args: [ "-conf", "/etc/coredns/Corefile" ]
+        volumeMounts:
+        - name: config-volume
+          mountPath: /etc/coredns
+          readOnly: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9153
+          name: metrics
+          protocol: TCP
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            add:
+            - NET_BIND_SERVICE
+            drop:
+            - all
+          readOnlyRootFilesystem: true
+      dnsPolicy: Default
+      volumes:
+        - name: config-volume
+          configMap:
+            name: vpc-dns-corefile
+            items:
+            - key: Corefile
+              path: Corefile
\ No newline at end of file