diff --git a/dist/images/cleanup.sh b/dist/images/cleanup.sh index a9eab7ab058c517e10d92f834f8efe2f7dc6b7c4..8895c35f5f14e22c5f55fb85466977d4635914b8 100644 --- a/dist/images/cleanup.sh +++ b/dist/images/cleanup.sh @@ -29,6 +29,10 @@ for slr in $(kubectl get switch-lb-rule -o name); do kubectl delete --ignore-not-found $slr done +for vd in $(kubectl get vpc-dns -o name); do + kubectl delete --ignore-not-found $vd +done + for vip in $(kubectl get vip -o name); do kubectl delete --ignore-not-found $vip done @@ -79,11 +83,17 @@ kubectl delete --ignore-not-found sa ovn -n kube-system kubectl delete --ignore-not-found clusterrole system:ovn kubectl delete --ignore-not-found clusterrolebinding ovn +# delete vpc-dns content +kubectl delete --ignore-not-found cm vpc-dns-config -n kube-system +kubectl delete --ignore-not-found clusterrole system:vpc-dns +kubectl delete --ignore-not-found clusterrolebinding vpc-dns +kubectl delete --ignore-not-found sa vpc-dns -n kube-system + # delete CRD kubectl delete --ignore-not-found crd htbqoses.kubeovn.io security-groups.kubeovn.io ips.kubeovn.io subnets.kubeovn.io \ vpc-nat-gateways.kubeovn.io vpcs.kubeovn.io vlans.kubeovn.io provider-networks.kubeovn.io \ iptables-dnat-rules.kubeovn.io iptables-eips.kubeovn.io iptables-fip-rules.kubeovn.io \ - iptables-snat-rules.kubeovn.io vips.kubeovn.io switch-lb-rules.kubeovn.io + iptables-snat-rules.kubeovn.io vips.kubeovn.io switch-lb-rules.kubeovn.io vpc-dnses.kubeovn.io # Remove annotations/labels in namespaces and nodes kubectl annotate no --all ovn.kubernetes.io/cidr- diff --git a/dist/images/install.sh b/dist/images/install.sh index 372befa90ba59d4003876f722829edb3dadc4a42..dd275af7e7c09b446bede4f27ce0fb652f5b42af 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -208,6 +208,144 @@ addresses=$(kubectl get no -lkube-ovn/role=master --no-headers -o wide | awk '{p echo "Install OVN DB in $addresses" cat <<EOF > kube-ovn-crd.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: vpc-dnses.kubeovn.io +spec: + group: kubeovn.io + names: + plural: vpc-dnses + singular: vpc-dns + shortNames: + - vpc-dns + kind: VpcDns + listKind: VpcDnsList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.active + name: Active + type: boolean + - jsonPath: .spec.vpc + name: Vpc + type: string + - jsonPath: .spec.subnet + name: Subnet + type: string + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + vpc: + type: string + subnet: + type: string + status: + type: object + properties: + active: + type: boolean + conditions: + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + reason: + type: string + message: + type: string + lastUpdateTime: + type: string + lastTransitionTime: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: switch-lb-rules.kubeovn.io +spec: + group: kubeovn.io + names: + plural: switch-lb-rules + singular: switch-lb-rule + shortNames: + - slr + kind: SwitchLBRule + listKind: SwitchLBRuleList + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.vip + name: vip + type: string + - jsonPath: .status.ports + name: port(s) + type: string + - jsonPath: .status.service + name: service + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + namespace: + type: string + vip: + type: string + sessionAffinity: + type: string + ports: + items: + properties: + name: + type: string + port: + type: integer + minimum: 1 + maximum: 65535 + protocol: + type: string + targetPort: + type: integer + minimum: 1 + maximum: 65535 + type: object + type: array + selector: + items: + type: string + type: array + status: + type: object + properties: + ports: + type: string + service: + type: string --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -1494,6 +1632,8 @@ rules: - iptables-snat-rules/status - switch-lb-rules - switch-lb-rules/status + - vpc-dnses + - vpc-dnses/status verbs: - "*" - apiGroups: @@ -1990,6 +2130,8 @@ rules: - iptables-fip-rules/status - iptables-dnat-rules/status - iptables-snat-rules/status + - vpc-dnses + - vpc-dnses/status - switch-lb-rules - switch-lb-rules/status verbs: diff --git a/pkg/apis/kubeovn/v1/register.go b/pkg/apis/kubeovn/v1/register.go index b370fba5a8c4c2aeb58eb92e3d87dd7b3079e3fe..8543cf433ea29d803bfa761e39c73f1137c4fbc5 100644 --- a/pkg/apis/kubeovn/v1/register.go +++ b/pkg/apis/kubeovn/v1/register.go @@ -59,6 +59,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &HtbQosList{}, &SwitchLBRule{}, &SwitchLBRuleList{}, + &VpcDns{}, + &VpcDnsList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) return nil diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index d14bfa643a231d938f8d869e487b4a9d72b0c748..010e87c683b88a4cc5c0621f8ff30a56d6af6331 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -860,6 +860,63 @@ type VipList struct { Items []Vip `json:"items"` } +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +genclient:nonNamespaced +// +resourceName=vpc-dnses + +type VpcDns struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec VpcDnsSpec `json:"spec"` + Status VpcDnsStatus `json:"status,omitempty"` +} + +type VpcDnsSpec struct { + Vpc string `json:"vpc"` + Subnet string `json:"subnet"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type VpcDnsList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []VpcDns `json:"items"` +} + +type VpcDnsStatus struct { + // +optional + // +patchMergeKey=type + // +patchStrategy=merge + Conditions []VpcDnsCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` + + Active bool `json:"active" patchStrategy:"merge"` +} + +// Condition describes the state of an object at a certain point. +// +k8s:deepcopy-gen=true +type VpcDnsCondition struct { + // Type of condition. + Type ConditionType `json:"type"` + // Status of the condition, one of True, False, Unknown. + Status corev1.ConditionStatus `json:"status"` + // The reason for the condition's last transition. + // +optional + Reason string `json:"reason,omitempty"` + // A human readable message indicating details about the transition. + // +optional + Message string `json:"message,omitempty"` + // Last time the condition was probed + // +optional + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + // Last time the condition transitioned from one status to another. + // +optional + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` +} + type SlrPort struct { Name string `json:"name"` Port int32 `json:"port"` diff --git a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go index f4600d7f8be7d55aefd391b19c97ba5c821d9891..e466fe6fb8868b22497d37f2820ff73524818fe3 100644 --- a/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go +++ b/pkg/apis/kubeovn/v1/zz_generated.deepcopy.go @@ -1583,6 +1583,124 @@ func (in *VpcCondition) DeepCopy() *VpcCondition { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcDns) DeepCopyInto(out *VpcDns) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDns. +func (in *VpcDns) DeepCopy() *VpcDns { + if in == nil { + return nil + } + out := new(VpcDns) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VpcDns) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcDnsCondition) DeepCopyInto(out *VpcDnsCondition) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsCondition. +func (in *VpcDnsCondition) DeepCopy() *VpcDnsCondition { + if in == nil { + return nil + } + out := new(VpcDnsCondition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcDnsList) DeepCopyInto(out *VpcDnsList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]VpcDns, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsList. +func (in *VpcDnsList) DeepCopy() *VpcDnsList { + if in == nil { + return nil + } + out := new(VpcDnsList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *VpcDnsList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcDnsSpec) DeepCopyInto(out *VpcDnsSpec) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsSpec. +func (in *VpcDnsSpec) DeepCopy() *VpcDnsSpec { + if in == nil { + return nil + } + out := new(VpcDnsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *VpcDnsStatus) DeepCopyInto(out *VpcDnsStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]VpcDnsCondition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VpcDnsStatus. +func (in *VpcDnsStatus) DeepCopy() *VpcDnsStatus { + if in == nil { + return nil + } + out := new(VpcDnsStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VpcList) DeepCopyInto(out *VpcList) { *out = *in diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go index eb5d7695303442b78fadd11eab7383535fefb86d..906bc6172c76a4d367011154faf512ca3e9bcac4 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_kubeovn_client.go @@ -80,6 +80,10 @@ func (c *FakeKubeovnV1) Vpcs() v1.VpcInterface { return &FakeVpcs{c} } +func (c *FakeKubeovnV1) VpcDnses() v1.VpcDnsInterface { + return &FakeVpcDnses{c} +} + func (c *FakeKubeovnV1) VpcNatGateways() v1.VpcNatGatewayInterface { return &FakeVpcNatGateways{c} } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcdns.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcdns.go new file mode 100644 index 0000000000000000000000000000000000000000..69127f7e2c495b2d8bd2b1132e583edda96fb530 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/fake/fake_vpcdns.go @@ -0,0 +1,133 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + "context" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeVpcDnses implements VpcDnsInterface +type FakeVpcDnses struct { + Fake *FakeKubeovnV1 +} + +var vpcdnsesResource = schema.GroupVersionResource{Group: "kubeovn.io", Version: "v1", Resource: "vpc-dnses"} + +var vpcdnsesKind = schema.GroupVersionKind{Group: "kubeovn.io", Version: "v1", Kind: "VpcDns"} + +// Get takes name of the vpcDns, and returns the corresponding vpcDns object, and an error if there is any. +func (c *FakeVpcDnses) Get(ctx context.Context, name string, options v1.GetOptions) (result *kubeovnv1.VpcDns, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootGetAction(vpcdnsesResource, name), &kubeovnv1.VpcDns{}) + if obj == nil { + return nil, err + } + return obj.(*kubeovnv1.VpcDns), err +} + +// List takes label and field selectors, and returns the list of VpcDnses that match those selectors. +func (c *FakeVpcDnses) List(ctx context.Context, opts v1.ListOptions) (result *kubeovnv1.VpcDnsList, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootListAction(vpcdnsesResource, vpcdnsesKind, opts), &kubeovnv1.VpcDnsList{}) + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &kubeovnv1.VpcDnsList{ListMeta: obj.(*kubeovnv1.VpcDnsList).ListMeta} + for _, item := range obj.(*kubeovnv1.VpcDnsList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested vpcDnses. +func (c *FakeVpcDnses) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewRootWatchAction(vpcdnsesResource, opts)) +} + +// Create takes the representation of a vpcDns and creates it. Returns the server's representation of the vpcDns, and an error, if there is any. +func (c *FakeVpcDnses) Create(ctx context.Context, vpcDns *kubeovnv1.VpcDns, opts v1.CreateOptions) (result *kubeovnv1.VpcDns, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootCreateAction(vpcdnsesResource, vpcDns), &kubeovnv1.VpcDns{}) + if obj == nil { + return nil, err + } + return obj.(*kubeovnv1.VpcDns), err +} + +// Update takes the representation of a vpcDns and updates it. Returns the server's representation of the vpcDns, and an error, if there is any. +func (c *FakeVpcDnses) Update(ctx context.Context, vpcDns *kubeovnv1.VpcDns, opts v1.UpdateOptions) (result *kubeovnv1.VpcDns, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateAction(vpcdnsesResource, vpcDns), &kubeovnv1.VpcDns{}) + if obj == nil { + return nil, err + } + return obj.(*kubeovnv1.VpcDns), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeVpcDnses) UpdateStatus(ctx context.Context, vpcDns *kubeovnv1.VpcDns, opts v1.UpdateOptions) (*kubeovnv1.VpcDns, error) { + obj, err := c.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(vpcdnsesResource, "status", vpcDns), &kubeovnv1.VpcDns{}) + if obj == nil { + return nil, err + } + return obj.(*kubeovnv1.VpcDns), err +} + +// Delete takes name of the vpcDns and deletes it. Returns an error if one occurs. +func (c *FakeVpcDnses) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewRootDeleteActionWithOptions(vpcdnsesResource, name, opts), &kubeovnv1.VpcDns{}) + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeVpcDnses) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { + action := testing.NewRootDeleteCollectionAction(vpcdnsesResource, listOpts) + + _, err := c.Fake.Invokes(action, &kubeovnv1.VpcDnsList{}) + return err +} + +// Patch applies the patch and returns the patched vpcDns. +func (c *FakeVpcDnses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *kubeovnv1.VpcDns, err error) { + obj, err := c.Fake. + Invokes(testing.NewRootPatchSubresourceAction(vpcdnsesResource, name, pt, data, subresources...), &kubeovnv1.VpcDns{}) + if obj == nil { + return nil, err + } + return obj.(*kubeovnv1.VpcDns), err +} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go index a5341c219ae9abc8d297c72f535bcaa4fd6699f0..311c7c987676f9f00a9b8de035c64eea7dc01d79 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/generated_expansion.go @@ -44,4 +44,6 @@ type VlanExpansion interface{} type VpcExpansion interface{} +type VpcDnsExpansion interface{} + type VpcNatGatewayExpansion interface{} diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go index 36c234f660bc245b7f25d18558e6798b75fcf7dd..a44bb99580530d305b2acec5028049609f0e95e2 100644 --- a/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/kubeovn_client.go @@ -41,6 +41,7 @@ type KubeovnV1Interface interface { VipsGetter VlansGetter VpcsGetter + VpcDnsesGetter VpcNatGatewaysGetter } @@ -101,6 +102,10 @@ func (c *KubeovnV1Client) Vpcs() VpcInterface { return newVpcs(c) } +func (c *KubeovnV1Client) VpcDnses() VpcDnsInterface { + return newVpcDnses(c) +} + func (c *KubeovnV1Client) VpcNatGateways() VpcNatGatewayInterface { return newVpcNatGateways(c) } diff --git a/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcdns.go b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcdns.go new file mode 100644 index 0000000000000000000000000000000000000000..a5190ce56af0baa8efdf2accd81a6b217290a28f --- /dev/null +++ b/pkg/client/clientset/versioned/typed/kubeovn/v1/vpcdns.go @@ -0,0 +1,184 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + "time" + + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + scheme "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/scheme" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// VpcDnsesGetter has a method to return a VpcDnsInterface. +// A group's client should implement this interface. +type VpcDnsesGetter interface { + VpcDnses() VpcDnsInterface +} + +// VpcDnsInterface has methods to work with VpcDns resources. +type VpcDnsInterface interface { + Create(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.CreateOptions) (*v1.VpcDns, error) + Update(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (*v1.VpcDns, error) + UpdateStatus(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (*v1.VpcDns, error) + Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error + Get(ctx context.Context, name string, opts metav1.GetOptions) (*v1.VpcDns, error) + List(ctx context.Context, opts metav1.ListOptions) (*v1.VpcDnsList, error) + Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcDns, err error) + VpcDnsExpansion +} + +// vpcDnses implements VpcDnsInterface +type vpcDnses struct { + client rest.Interface +} + +// newVpcDnses returns a VpcDnses +func newVpcDnses(c *KubeovnV1Client) *vpcDnses { + return &vpcDnses{ + client: c.RESTClient(), + } +} + +// Get takes name of the vpcDns, and returns the corresponding vpcDns object, and an error if there is any. +func (c *vpcDnses) Get(ctx context.Context, name string, options metav1.GetOptions) (result *v1.VpcDns, err error) { + result = &v1.VpcDns{} + err = c.client.Get(). + Resource("vpc-dnses"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(ctx). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of VpcDnses that match those selectors. +func (c *vpcDnses) List(ctx context.Context, opts metav1.ListOptions) (result *v1.VpcDnsList, err error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + result = &v1.VpcDnsList{} + err = c.client.Get(). + Resource("vpc-dnses"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Do(ctx). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested vpcDnses. +func (c *vpcDnses) Watch(ctx context.Context, opts metav1.ListOptions) (watch.Interface, error) { + var timeout time.Duration + if opts.TimeoutSeconds != nil { + timeout = time.Duration(*opts.TimeoutSeconds) * time.Second + } + opts.Watch = true + return c.client.Get(). + Resource("vpc-dnses"). + VersionedParams(&opts, scheme.ParameterCodec). + Timeout(timeout). + Watch(ctx) +} + +// Create takes the representation of a vpcDns and creates it. Returns the server's representation of the vpcDns, and an error, if there is any. +func (c *vpcDnses) Create(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.CreateOptions) (result *v1.VpcDns, err error) { + result = &v1.VpcDns{} + err = c.client.Post(). + Resource("vpc-dnses"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(vpcDns). + Do(ctx). + Into(result) + return +} + +// Update takes the representation of a vpcDns and updates it. Returns the server's representation of the vpcDns, and an error, if there is any. +func (c *vpcDnses) Update(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (result *v1.VpcDns, err error) { + result = &v1.VpcDns{} + err = c.client.Put(). + Resource("vpc-dnses"). + Name(vpcDns.Name). + VersionedParams(&opts, scheme.ParameterCodec). + Body(vpcDns). + Do(ctx). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *vpcDnses) UpdateStatus(ctx context.Context, vpcDns *v1.VpcDns, opts metav1.UpdateOptions) (result *v1.VpcDns, err error) { + result = &v1.VpcDns{} + err = c.client.Put(). + Resource("vpc-dnses"). + Name(vpcDns.Name). + SubResource("status"). + VersionedParams(&opts, scheme.ParameterCodec). + Body(vpcDns). + Do(ctx). + Into(result) + return +} + +// Delete takes name of the vpcDns and deletes it. Returns an error if one occurs. +func (c *vpcDnses) Delete(ctx context.Context, name string, opts metav1.DeleteOptions) error { + return c.client.Delete(). + Resource("vpc-dnses"). + Name(name). + Body(&opts). + Do(ctx). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *vpcDnses) DeleteCollection(ctx context.Context, opts metav1.DeleteOptions, listOpts metav1.ListOptions) error { + var timeout time.Duration + if listOpts.TimeoutSeconds != nil { + timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second + } + return c.client.Delete(). + Resource("vpc-dnses"). + VersionedParams(&listOpts, scheme.ParameterCodec). + Timeout(timeout). + Body(&opts). + Do(ctx). + Error() +} + +// Patch applies the patch and returns the patched vpcDns. +func (c *vpcDnses) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.VpcDns, err error) { + result = &v1.VpcDns{} + err = c.client.Patch(pt). + Resource("vpc-dnses"). + Name(name). + SubResource(subresources...). + VersionedParams(&opts, scheme.ParameterCodec). + Body(data). + Do(ctx). + Into(result) + return +} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index c63695b04ee062960091cd54bf6b2987b9459b8d..7bb62042533ab3f08c3aec23e80af533200465cb 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -79,6 +79,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().Vlans().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpcs"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().Vpcs().Informer()}, nil + case v1.SchemeGroupVersion.WithResource("vpc-dnses"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcDnses().Informer()}, nil case v1.SchemeGroupVersion.WithResource("vpc-nat-gateways"): return &genericInformer{resource: resource.GroupResource(), informer: f.Kubeovn().V1().VpcNatGateways().Informer()}, nil diff --git a/pkg/client/informers/externalversions/kubeovn/v1/interface.go b/pkg/client/informers/externalversions/kubeovn/v1/interface.go index b83808ab2ed3d3975f10079c3c0bce50e39a62c6..fd9ca5c7c4c3cb516025a41f640b601bae70ddd8 100644 --- a/pkg/client/informers/externalversions/kubeovn/v1/interface.go +++ b/pkg/client/informers/externalversions/kubeovn/v1/interface.go @@ -50,6 +50,8 @@ type Interface interface { Vlans() VlanInformer // Vpcs returns a VpcInformer. Vpcs() VpcInformer + // VpcDnses returns a VpcDnsInformer. + VpcDnses() VpcDnsInformer // VpcNatGateways returns a VpcNatGatewayInformer. VpcNatGateways() VpcNatGatewayInformer } @@ -130,6 +132,11 @@ func (v *version) Vpcs() VpcInformer { return &vpcInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} } +// VpcDnses returns a VpcDnsInformer. +func (v *version) VpcDnses() VpcDnsInformer { + return &vpcDnsInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} +} + // VpcNatGateways returns a VpcNatGatewayInformer. func (v *version) VpcNatGateways() VpcNatGatewayInformer { return &vpcNatGatewayInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/kubeovn/v1/vpcdns.go b/pkg/client/informers/externalversions/kubeovn/v1/vpcdns.go new file mode 100644 index 0000000000000000000000000000000000000000..57ec5af21e5dae97f3676a12538859fe9d4cf5c9 --- /dev/null +++ b/pkg/client/informers/externalversions/kubeovn/v1/vpcdns.go @@ -0,0 +1,89 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v1 + +import ( + "context" + time "time" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + versioned "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned" + internalinterfaces "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/kubeovn/kube-ovn/pkg/client/listers/kubeovn/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// VpcDnsInformer provides access to a shared informer and lister for +// VpcDnses. +type VpcDnsInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1.VpcDnsLister +} + +type vpcDnsInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// NewVpcDnsInformer constructs a new informer for VpcDns type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewVpcDnsInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredVpcDnsInformer(client, resyncPeriod, indexers, nil) +} + +// NewFilteredVpcDnsInformer constructs a new informer for VpcDns type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredVpcDnsInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().VpcDnses().List(context.TODO(), options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.KubeovnV1().VpcDnses().Watch(context.TODO(), options) + }, + }, + &kubeovnv1.VpcDns{}, + resyncPeriod, + indexers, + ) +} + +func (f *vpcDnsInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredVpcDnsInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *vpcDnsInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&kubeovnv1.VpcDns{}, f.defaultInformer) +} + +func (f *vpcDnsInformer) Lister() v1.VpcDnsLister { + return v1.NewVpcDnsLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/listers/kubeovn/v1/expansion_generated.go b/pkg/client/listers/kubeovn/v1/expansion_generated.go index 0dca45065dd9f97eb99153f18bb145442c9456a2..f5f029aa8ca5d48b9a041abf524de8710fa812d9 100644 --- a/pkg/client/listers/kubeovn/v1/expansion_generated.go +++ b/pkg/client/listers/kubeovn/v1/expansion_generated.go @@ -70,6 +70,10 @@ type VlanListerExpansion interface{} // VpcLister. type VpcListerExpansion interface{} +// VpcDnsListerExpansion allows custom methods to be added to +// VpcDnsLister. +type VpcDnsListerExpansion interface{} + // VpcNatGatewayListerExpansion allows custom methods to be added to // VpcNatGatewayLister. type VpcNatGatewayListerExpansion interface{} diff --git a/pkg/client/listers/kubeovn/v1/vpcdns.go b/pkg/client/listers/kubeovn/v1/vpcdns.go new file mode 100644 index 0000000000000000000000000000000000000000..5eaf1929c5f61b24d3f497d635661ef18aa4b805 --- /dev/null +++ b/pkg/client/listers/kubeovn/v1/vpcdns.go @@ -0,0 +1,68 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v1 + +import ( + v1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// VpcDnsLister helps list VpcDnses. +// All objects returned here must be treated as read-only. +type VpcDnsLister interface { + // List lists all VpcDnses in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*v1.VpcDns, err error) + // Get retrieves the VpcDns from the index for a given name. + // Objects returned here must be treated as read-only. + Get(name string) (*v1.VpcDns, error) + VpcDnsListerExpansion +} + +// vpcDnsLister implements the VpcDnsLister interface. +type vpcDnsLister struct { + indexer cache.Indexer +} + +// NewVpcDnsLister returns a new VpcDnsLister. +func NewVpcDnsLister(indexer cache.Indexer) VpcDnsLister { + return &vpcDnsLister{indexer: indexer} +} + +// List lists all VpcDnses in the indexer. +func (s *vpcDnsLister) List(selector labels.Selector) (ret []*v1.VpcDns, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1.VpcDns)) + }) + return ret, err +} + +// Get retrieves the VpcDns from the index for a given name. +func (s *vpcDnsLister) Get(name string) (*v1.VpcDns, error) { + obj, exists, err := s.indexer.GetByKey(name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1.Resource("vpcdns"), name) + } + return obj.(*v1.VpcDns), nil +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c3ed2e475ea334480c62e341229fb3d6f0fde70e..0c51ed9ec90f8bfb24f426ce53104f512eb00d2f 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -76,6 +76,11 @@ type Controller struct { UpdateSwitchLBRuleQueue workqueue.RateLimitingInterface delSwitchLBRuleQueue workqueue.RateLimitingInterface + vpcDnsLister kubeovnlister.VpcDnsLister + vpcDnsSynced cache.InformerSynced + addOrUpdateVpcDnsQueue workqueue.RateLimitingInterface + delVpcDnsQueue workqueue.RateLimitingInterface + subnetsLister kubeovnlister.SubnetLister subnetSynced cache.InformerSynced addOrUpdateSubnetQueue workqueue.RateLimitingInterface @@ -398,6 +403,17 @@ func NewController(config *Configuration) *Controller { UpdateFunc: controller.enqueueUpdateSwitchLBRule, DeleteFunc: controller.enqueueDeleteSwitchLBRule, }) + + vpcDnsInformer := kubeovnInformerFactory.Kubeovn().V1().VpcDnses() + controller.vpcDnsLister = vpcDnsInformer.Lister() + controller.vpcDnsSynced = vpcDnsInformer.Informer().HasSynced + controller.addOrUpdateVpcDnsQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "AddOrUpdateVpcDns") + controller.delVpcDnsQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "DeleteVpcDns") + vpcDnsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueAddVpcDns, + UpdateFunc: controller.enqueueUpdateVpcDns, + DeleteFunc: controller.enqueueDeleteVpcDns, + }) } subnetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -502,7 +518,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) { } if c.config.EnableLb { - cacheSyncs = append(cacheSyncs, c.switchLBRuleSynced) + cacheSyncs = append(cacheSyncs, c.switchLBRuleSynced, c.vpcDnsSynced) } if ok := cache.WaitForCacheSync(stopCh, cacheSyncs...); !ok { @@ -555,6 +571,13 @@ func (c *Controller) Run(stopCh <-chan struct{}) { if err := c.initSyncCrdVlans(); err != nil { klog.Errorf("failed to sync crd vlans: %v", err) } + + if c.config.EnableLb { + if err := c.initVpcDnsConfig(); err != nil { + klog.Errorf("failed to init vpc-dns: %v", err) + } + } + // The static route for node gw can be deleted when gc static route, so add it after gc process dstIp := "0.0.0.0/0,::/0" if err := c.ovnLegacyClient.AddStaticRoute("", dstIp, c.config.NodeSwitchGateway, c.config.ClusterRouter, util.NormalRouteType); err != nil { @@ -615,6 +638,9 @@ func (c *Controller) shutdown() { c.addSwitchLBRuleQueue.ShutDown() c.delSwitchLBRuleQueue.ShutDown() c.UpdateSwitchLBRuleQueue.ShutDown() + + c.addOrUpdateVpcDnsQueue.ShutDown() + c.delVpcDnsQueue.ShutDown() } c.addVirtualIpQueue.ShutDown() @@ -719,6 +745,12 @@ func (c *Controller) startWorkers(stopCh <-chan struct{}) { go wait.Until(c.runAddSwitchLBRuleWorker, time.Second, stopCh) go wait.Until(c.runDelSwitchLBRuleWorker, time.Second, stopCh) go wait.Until(c.runUpdateSwitchLBRuleWorker, time.Second, stopCh) + + go wait.Until(c.runAddOrUpdateVpcDnsWorker, time.Second, stopCh) + go wait.Until(c.runDelVpcDnsWorker, time.Second, stopCh) + go wait.Until(func() { + c.resyncVpcDnsConfig() + }, 5*time.Second, stopCh) } for i := 0; i < c.config.WorkerNum; i++ { diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index 7145e08efa1f63daf8547cbe6d545502bbd123d6..36469467a8fdd9879e7ee9d933f156d6a9d5da9a 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -35,6 +35,7 @@ func (c *Controller) gc() error { c.gcLogicalRouterPort, c.gcVip, c.gcLbSvcPods, + c.gcVpcDns, } for _, gcFunc := range gcFunctions { if err := gcFunc(); err != nil { @@ -766,3 +767,73 @@ func (c *Controller) gcLbSvcPods() error { } return nil } + +func (c *Controller) gcVpcDns() error { + if !c.config.EnableLb { + return nil + } + + klog.Infof("start to gc vpc dns") + vds, err := c.vpcDnsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to list vpc-dns, %v", err) + return err + } + + sel, _ := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{MatchLabels: map[string]string{util.VpcDnsNameLabel: "true"}}) + + deps, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).List(context.Background(), metav1.ListOptions{ + LabelSelector: sel.String(), + }) + if err != nil { + klog.Errorf("failed to list vpc-dns deployment, %s", err) + return err + } + + for _, dep := range deps.Items { + canFind := false + for _, vd := range vds { + name := genVpcDnsDpName(vd.Name) + if dep.Name == name { + canFind = true + break + } + } + if !canFind { + err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Delete(context.Background(), + dep.Name, metav1.DeleteOptions{}) + if err != nil { + klog.Errorf("failed to delete vpc-dns deployment, %s", err) + return err + } + } + } + + slrs, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().List(context.Background(), metav1.ListOptions{ + LabelSelector: sel.String(), + }) + if err != nil { + klog.Errorf("failed to list vpc-dns SwitchLBRules, %s", err) + return err + } + + for _, slr := range slrs.Items { + canFind := false + for _, vd := range vds { + name := genVpcDnsDpName(vd.Name) + if slr.Name == name { + canFind = true + break + } + } + if !canFind { + err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Delete(context.Background(), + slr.Name, metav1.DeleteOptions{}) + if err != nil { + klog.Errorf("failed to delete vpc-dns SwitchLBRule, %s", err) + return err + } + } + } + return nil +} diff --git a/pkg/controller/vpc_dns.go b/pkg/controller/vpc_dns.go new file mode 100644 index 0000000000000000000000000000000000000000..0efab253fd2c381072b0c635ce88afbe3fba2ef6 --- /dev/null +++ b/pkg/controller/vpc_dns.go @@ -0,0 +1,671 @@ +package controller + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "github.com/kubeovn/kube-ovn/versions" + "io" + "io/ioutil" + "k8s.io/apimachinery/pkg/util/yaml" + "net/http" + "os" + "reflect" + "strconv" + "strings" + "text/template" + "time" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/util" + v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/tools/cache" + "k8s.io/klog/v2" +) + +var ( + corednsYamlUrl = "" + corednsImage = "" + corednsVip = "" + nadName = "" + nadProvider = "" + cmVersion = "" + k8sServiceHost = "" + k8sServicePort = "" + enableCoredns = false + hostNameservers []string +) + +const ( + CorednsContainerName = "coredns" + CorednsLabelKey = "k8s-app" + CorednsTemplateDep = "coredns-template.yaml" + InitRouteImage = "kubeovn/vpc-nat-gateway:v1.11.0" +) + +func genVpcDnsDpName(name string) string { + return fmt.Sprintf("vpc-dns-%s", name) +} + +func hostConfigFromReader() error { + file, err := os.Open("/etc/resolv.conf") + if err != nil { + return err + } + defer func(file *os.File) { + if err := file.Close(); err != nil { + klog.Errorf("failed to close file, %s", err) + } + }(file) + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if err := scanner.Err(); err != nil { + return err + } + line := scanner.Text() + f := strings.Fields(line) + if len(f) < 1 { + continue + } + if f[0] == "nameserver" && len(f) > 1 { + name := f[1] + hostNameservers = append(hostNameservers, name) + } + } + + return err +} + +func (c *Controller) enqueueAddVpcDns(obj interface{}) { + if !c.isLeader() { + return + } + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue add vpc-dns %s", key) + c.addOrUpdateVpcDnsQueue.Add(key) +} + +func (c *Controller) enqueueUpdateVpcDns(old, new interface{}) { + if !c.isLeader() { + return + } + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(new); err != nil { + utilruntime.HandleError(err) + return + } + + oldVpcDns := old.(*kubeovnv1.VpcDns) + newVpcDns := new.(*kubeovnv1.VpcDns) + if oldVpcDns.ResourceVersion != newVpcDns.ResourceVersion && + !reflect.DeepEqual(oldVpcDns.Spec, newVpcDns.Spec) { + klog.V(3).Infof("enqueue update vpc-dns %s", key) + c.addOrUpdateVpcDnsQueue.Add(key) + } +} + +func (c *Controller) enqueueDeleteVpcDns(obj interface{}) { + if !c.isLeader() { + return + } + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + klog.V(3).Infof("enqueue delete vpc-dns %s", key) + c.delVpcDnsQueue.Add(key) +} + +func (c *Controller) runAddOrUpdateVpcDnsWorker() { + for c.processNextWorkItem("addOrUpdateVpcDns", c.addOrUpdateVpcDnsQueue, c.handleAddOrUpdateVpcDns) { + } +} + +func (c *Controller) runDelVpcDnsWorker() { + for c.processNextWorkItem("delVpcDns", c.delVpcDnsQueue, c.handleDelVpcDns) { + } +} + +func (c *Controller) handleAddOrUpdateVpcDns(key string) error { + klog.V(3).Infof("handleAddOrUpdateVpcDns %s", key) + if !enableCoredns { + time.Sleep(10 * time.Second) + if !enableCoredns { + return fmt.Errorf("failed to add/update vpc-dns, enable ='%v'", enableCoredns) + } + } + + vpcDns, err := c.vpcDnsLister.Get(key) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + + defer func() { + newVpcDns := vpcDns.DeepCopy() + newVpcDns.Status.Active = true + if err != nil { + newVpcDns.Status.Active = false + } + + _, err = c.config.KubeOvnClient.KubeovnV1().VpcDnses().UpdateStatus(context.Background(), + newVpcDns, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("update vpc-dns status failed, %v", err) + } + }() + + if len(corednsImage) == 0 { + err := fmt.Errorf("failed to get the vpc-dns coredns image parameter") + klog.Errorf("failed to get corednsImage, err: %s", err) + return err + } + + if len(corednsVip) == 0 { + err := fmt.Errorf("the configuration parameter corednsVip is empty") + klog.Errorf("failed to get corednsVip, err: %s", err) + return err + } + + if _, err := c.vpcsLister.Get(vpcDns.Spec.Vpc); err != nil { + klog.Errorf("failed to get vpc '%s', err: %v", vpcDns.Spec.Vpc, err) + return err + } + + if _, err := c.subnetsLister.Get(vpcDns.Spec.Subnet); err != nil { + klog.Errorf("failed to get subnet '%s', err: %v", vpcDns.Spec.Subnet, err) + return err + } + + if err := c.checkOvnNad(); err != nil { + klog.Errorf("failed to check nad, %v", err) + return err + } + + if err := c.checkOvnProvided(); err != nil { + klog.Errorf("failed to check %s provided, %v", util.DefaultSubnet, err) + return err + } + + if err := c.checkVpcDnsDuplicated(vpcDns); err != nil { + klog.Errorf("failed to deploy %s, %v", vpcDns.Name, err) + return err + } + + if err := c.createOrUpdateVpcDnsDep(vpcDns); err != nil { + return err + } + + if err := c.createOrUpdateVpcDnsSlr(vpcDns); err != nil { + return err + } + + return nil +} + +func (c *Controller) handleDelVpcDns(key string) error { + klog.V(3).Infof("handleDelVpcDns,%s", key) + name := genVpcDnsDpName(key) + err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace).Delete(context.Background(), name, metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete Deployments: %v", err) + return err + } + + err = c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Delete(context.Background(), name, metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete SwitchLBRule: %v", err) + return err + } + return nil +} + +func (c *Controller) checkVpcDnsDuplicated(vpcDns *kubeovnv1.VpcDns) error { + vpcDnsList, err := c.vpcDnsLister.List(labels.Everything()) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + + for _, item := range vpcDnsList { + if item.Status.Active && + item.Name != vpcDns.Name && + item.Spec.Vpc == vpcDns.Spec.Vpc { + err = fmt.Errorf("only one vpc-dns can be deployed in a vpc") + return err + } + } + return nil +} + +func (c *Controller) createOrUpdateVpcDnsDep(vpcDns *kubeovnv1.VpcDns) error { + needToCreateDp := false + oldDp, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace). + Get(context.Background(), genVpcDnsDpName(vpcDns.Name), metav1.GetOptions{}) + + if err != nil { + if k8serrors.IsNotFound(err) { + needToCreateDp = true + } else { + return err + } + } + + newDp, err := c.genVpcDnsDeployment(vpcDns, oldDp) + if err != nil { + klog.Errorf("failed to generate vpc-dns deployment, %v", err) + return err + } + + if needToCreateDp { + _, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace). + Create(context.Background(), newDp, metav1.CreateOptions{}) + + if err != nil { + klog.Errorf("failed to create deployment '%s', err: %s", newDp.Name, err) + return err + } + } else { + _, err := c.config.KubeClient.AppsV1().Deployments(c.config.PodNamespace). + Update(context.Background(), newDp, metav1.UpdateOptions{}) + + if err != nil { + klog.Errorf("failed to update deployment '%s', err: %v", newDp.Name, err) + return err + } + } + return nil +} + +func (c *Controller) createOrUpdateVpcDnsSlr(vpcDns *kubeovnv1.VpcDns) error { + needToCreateSlr := false + oldSlr, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Get(context.Background(), + genVpcDnsDpName(vpcDns.Name), metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + needToCreateSlr = true + } else { + return err + } + } + + newSlr, err := c.genVpcDnsSlr(vpcDns.Name, c.config.PodNamespace) + if err != nil { + klog.Errorf("failed to generate vpc-dns switchLBRule, %v", err) + return err + } + + if needToCreateSlr { + _, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Create(context.Background(), newSlr, metav1.CreateOptions{}) + if err != nil { + klog.Errorf("failed to create switchLBRules '%s', err: %v", newSlr.Name, err) + return err + } + } else { + if reflect.DeepEqual(oldSlr.Spec, newSlr.Spec) { + return nil + } + + newSlr.ResourceVersion = oldSlr.ResourceVersion + _, err := c.config.KubeOvnClient.KubeovnV1().SwitchLBRules().Update(context.Background(), newSlr, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("failed to update switchLBRules '%s', err: %v", newSlr.Name, err) + return err + } + } + return nil +} + +func (c *Controller) genVpcDnsDeployment(vpcDns *kubeovnv1.VpcDns, oldDeploy *v1.Deployment) (*v1.Deployment, error) { + if _, err := os.Stat(CorednsTemplateDep); errors.Is(err, os.ErrNotExist) { + if err := getCoreDnsTemplateFile(corednsYamlUrl); err != nil { + klog.Errorf("failed to get coredns template file, %v", err) + return nil, err + } + } + + tmp, err := template.ParseFiles(CorednsTemplateDep) + if err != nil { + return nil, err + } + + buffer := new(bytes.Buffer) + name := genVpcDnsDpName(vpcDns.Name) + if err := tmp.Execute(buffer, map[string]interface{}{ + "DeployName": name, + "CorednsImage": corednsImage, + }); err != nil { + return nil, err + } + + dep := &v1.Deployment{} + retJson, err := yaml.ToJSON(buffer.Bytes()) + if err != nil { + klog.Errorf("failed to switch yaml, %v", err) + return nil, err + } + + if err := json.Unmarshal(retJson, dep); err != nil { + klog.Errorf("failed to switch json, %v", err) + return nil, err + } + + dep.Spec.Template.Annotations = make(map[string]string) + + if oldDeploy != nil && len(oldDeploy.Annotations) != 0 { + dep.Spec.Template.Annotations = oldDeploy.Annotations + } + + dep.ObjectMeta.Labels = map[string]string{ + util.VpcDnsNameLabel: "true", + } + + setCoreDnsEnv(dep) + setVpcDnsInterface(dep, vpcDns.Spec.Subnet) + + defaultSubnet, err := c.subnetsLister.Get(util.DefaultSubnet) + if err != nil { + klog.Errorf("failed to get default subnet %v", err) + return nil, err + } + + setVpcDnsRoute(dep, defaultSubnet.Spec.Gateway) + return dep, nil +} + +func (c *Controller) genVpcDnsSlr(vpcName, namespace string) (*kubeovnv1.SwitchLBRule, error) { + name := genVpcDnsDpName(vpcName) + label := fmt.Sprintf("%s:%s", CorednsLabelKey, name) + + ports := []kubeovnv1.SlrPort{ + {Name: "dns", Port: 53, Protocol: "UDP"}, + {Name: "dns-tcp", Port: 53, Protocol: "TCP"}, + {Name: "metrics", Port: 9153, Protocol: "TCP"}, + } + + slr := &kubeovnv1.SwitchLBRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + util.VpcDnsNameLabel: "true", + }, + }, + Spec: kubeovnv1.SwitchLBRuleSpec{ + Vip: corednsVip, + Namespace: namespace, + Selector: []string{label}, + SessionAffinity: "", + Ports: ports, + }, + } + + return slr, nil +} + +func setVpcDnsInterface(dp *v1.Deployment, subnetName string) { + annotations := dp.Spec.Template.Annotations + annotations[util.LogicalSwitchAnnotation] = subnetName + annotations[util.AttachmentNetworkAnnotation] = fmt.Sprintf("%s/%s", corev1.NamespaceDefault, nadName) + annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, nadProvider)] = util.DefaultSubnet +} + +func setCoreDnsEnv(dp *v1.Deployment) { + var env []corev1.EnvVar + + if len(k8sServiceHost) != 0 { + env = append(env, corev1.EnvVar{Name: "KUBERNETES_SERVICE_HOST", Value: k8sServiceHost}) + } + + if len(k8sServicePort) != 0 { + env = append(env, corev1.EnvVar{Name: "KUBERNETES_SERVICE_PORT", Value: k8sServicePort}) + } + + for i, container := range dp.Spec.Template.Spec.Containers { + if container.Name == CorednsContainerName { + dp.Spec.Template.Spec.Containers[i].Env = env + break + } + } +} + +func setVpcDnsRoute(dp *v1.Deployment, subnetGw string) { + var serviceHost string + if len(k8sServiceHost) == 0 { + serviceHost = "${KUBERNETES_SERVICE_HOST}" + } else { + serviceHost = k8sServiceHost + } + + var routeCmd string + routeCmd = fmt.Sprintf("ip route add %s via %s dev net1;", serviceHost, subnetGw) + for _, nameserver := range hostNameservers { + routeCmd += fmt.Sprintf("ip route add %s via %s dev net1;", nameserver, subnetGw) + } + + privileged := true + allowPrivilegeEscalation := true + dp.Spec.Template.Spec.InitContainers = append(dp.Spec.Template.Spec.InitContainers, corev1.Container{ + Name: "init-route", + Image: InitRouteImage, + Command: []string{"sh", "-c", routeCmd}, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + AllowPrivilegeEscalation: &allowPrivilegeEscalation, + }, + }) +} + +func (c *Controller) checkOvnNad() error { + _, err := c.config.AttachNetClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(corev1.NamespaceDefault). + Get(context.Background(), nadName, metav1.GetOptions{}) + if err != nil { + return err + } + + return nil +} + +func (c *Controller) checkOvnProvided() error { + cachedSubnet, err := c.subnetsLister.Get(util.DefaultSubnet) + if err != nil { + return fmt.Errorf("failed to get default subnet %v", err) + } + + if cachedSubnet.Spec.Provider != nadProvider { + return fmt.Errorf("the %s provider does not exist", nadProvider) + } + + return nil +} + +func (c *Controller) resyncVpcDnsConfig() { + cm, err := c.configMapsLister.ConfigMaps(c.config.PodNamespace).Get(util.VpcDnsConfig) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to get %s, %v", util.VpcDnsConfig, err) + return + } + + if k8serrors.IsNotFound(err) { + klog.V(3).Infof("the vpc-dns configuration is not set ") + if len(cmVersion) != 0 { + if err := c.cleanVpcDns(); err != nil { + klog.Errorf("failed to clear all vpc-dns, %v", err) + return + } + cmVersion = "" + } + return + } + + if cmVersion == cm.ResourceVersion { + return + } else { + cmVersion = cm.ResourceVersion + klog.V(3).Infof("the vpc-dns ConfigMap update") + } + + getValue := func(key string) string { + if v, ok := cm.Data[key]; ok { + return v + } + return "" + } + + corednsImage = getValue("coredns-image") + if len(corednsImage) == 0 { + defaultImage, err := c.getDefaultCoreDnsImage() + if err != nil { + klog.Errorf("failed to get kube-system/coredns image, %s", err) + return + } + corednsImage = defaultImage + klog.V(3).Infof("use the cluster default coredns image version, %s", corednsImage) + } + + newTemplateUrl := getValue("coredns-template") + if len(newTemplateUrl) != 0 && newTemplateUrl != corednsYamlUrl { + if err := getCoreDnsTemplateFile(newTemplateUrl); err != nil { + klog.Errorf("failed to get coredns template file, %v", err) + } + corednsYamlUrl = newTemplateUrl + } + + nadName = getValue("nad-name") + nadProvider = getValue("nad-provider") + corednsVip = getValue("coredns-vip") + k8sServiceHost = getValue("k8s-service-host") + k8sServicePort = getValue("k8s-service-port") + + newEnableCoredns := true + if v, ok := cm.Data["enable-vpc-dns"]; ok { + raw, err := strconv.ParseBool(v) + if err != nil { + klog.Errorf("failed to parse cm enable, %v", err) + return + } + newEnableCoredns = raw + } + + if enableCoredns && !newEnableCoredns { + if err := c.cleanVpcDns(); err != nil { + klog.Errorf("failed to clear all vpc-dns, %v", err) + return + } + } else { + if err := c.updateVpcDns(); err != nil { + klog.Errorf("failed to update vpc-dns deployment") + return + } + } + + enableCoredns = newEnableCoredns +} + +func (c *Controller) getDefaultCoreDnsImage() (string, error) { + dp, err := c.config.KubeClient.AppsV1().Deployments("kube-system"). + Get(context.Background(), "coredns", metav1.GetOptions{}) + if err != nil { + return "", err + } + + for _, container := range dp.Spec.Template.Spec.Containers { + if container.Name == CorednsContainerName { + return container.Image, nil + } + } + + return "", fmt.Errorf("coredns container no fonud") +} + +func (c *Controller) initVpcDnsConfig() error { + url := "https://raw.githubusercontent.com/kubeovn/kube-ovn/%s/yamls/coredns-template.yaml" + corednsYamlUrl = fmt.Sprintf(url, versions.VERSION) + + if err := hostConfigFromReader(); err != nil { + klog.Errorf("failed to get get host nameserver, %v", err) + return err + } + + c.resyncVpcDnsConfig() + return nil +} + +func (c *Controller) cleanVpcDns() error { + klog.Infof("clear all vpc-dns") + err := c.config.KubeOvnClient.KubeovnV1().VpcDnses().DeleteCollection(context.Background(), metav1.DeleteOptions{}, + metav1.ListOptions{}) + if err != nil { + klog.Errorf("Failed to clear all vpc-dns %s", err) + return err + } + + return nil +} + +func (c *Controller) updateVpcDns() error { + list, err := c.vpcDnsLister.List(labels.Everything()) + if err != nil { + klog.Errorf("failed to get vpc-dns list, %s", err) + return err + } + + for _, vd := range list { + c.addOrUpdateVpcDnsQueue.Add(vd.Name) + } + return nil +} + +func getCoreDnsTemplateFile(url string) error { + client := http.Client{ + Timeout: 5 * time.Second, + } + + resp, err := client.Get(url) + if err != nil { + return err + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + klog.Errorf("failed to close http, %s", err) + } + }(resp.Body) + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("access errors, return code:%d", resp.StatusCode) + } + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + err = ioutil.WriteFile(CorednsTemplateDep, data, 0644) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/util/const.go b/pkg/util/const.go index 5110140bdaea7269627753a7021c97042c3ee0e7..e24b3e4afae3bacfafbf4c3ee5020f89b61915b1 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -80,14 +80,14 @@ const ( OvsDpTypeLabel = "ovn.kubernetes.io/ovs_dp_type" - SubnetNameLabel = "ovn.kubernetes.io/subnet" - ICGatewayLabel = "ovn.kubernetes.io/ic-gw" - ExGatewayLabel = "ovn.kubernetes.io/external-gw" - VpcNatGatewayLabel = "ovn.kubernetes.io/vpc-nat-gw" - IpReservedLabel = "ovn.kubernetes.io/ip_reserved" - VpcNatGatewayNameLabel = "ovn.kubernetes.io/vpc-nat-gw-name" - VpcLbLabel = "ovn.kubernetes.io/vpc_lb" - + SubnetNameLabel = "ovn.kubernetes.io/subnet" + ICGatewayLabel = "ovn.kubernetes.io/ic-gw" + ExGatewayLabel = "ovn.kubernetes.io/external-gw" + VpcNatGatewayLabel = "ovn.kubernetes.io/vpc-nat-gw" + IpReservedLabel = "ovn.kubernetes.io/ip_reserved" + VpcNatGatewayNameLabel = "ovn.kubernetes.io/vpc-nat-gw-name" + VpcLbLabel = "ovn.kubernetes.io/vpc_lb" + VpcDnsNameLabel = "ovn.kubernetes.io/vpc-dns" NetworkPolicyLogAnnotation = "ovn.kubernetes.io/enable_log" ProtocolTCP = "tcp" @@ -132,6 +132,8 @@ const ( VpcNatGatewayConfig = "ovn-vpc-nat-gw-config" VpcExternalNet = "ovn-vpc-external-network" VpcLbNetworkAttachment = "ovn-vpc-lb" + VpcDnsConfig = "vpc-dns-config" + VpcDnsDepTemplate = "vpc-dns-dep" DefaultVpc = "ovn-cluster" DefaultSubnet = "ovn-default" diff --git a/yamls/coredns-template.yaml b/yamls/coredns-template.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d1f8bc6285c3705e66482a261e9700945cf29de --- /dev/null +++ b/yamls/coredns-template.yaml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .DeployName }} + labels: + k8s-app: {{ .DeployName }} +spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: {{ .DeployName }} + template: + metadata: + labels: + k8s-app: {{ .DeployName }} + spec: + priorityClassName: system-cluster-critical + serviceAccountName: vpc-dns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + nodeSelector: + kubernetes.io/os: linux + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: k8s-app + operator: In + values: ["{{ .DeployName }}"] + topologyKey: kubernetes.io/hostname + containers: + - name: coredns + image: {{ .CorednsImage }} + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: vpc-dns-corefile + items: + - key: Corefile + path: Corefile \ No newline at end of file