k8s 1.18 import (#2651)

* k8s 1.18 import wip

backup, cmd, controller, generated, restic, restore, serverstatusrequest, test and util
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* go mod tidy
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* add changelog file
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* go fmt
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* update code-generator and controller-gen in CI
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* checkout proper code-generator version, regen
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* fix remaining calls
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* regenerate CRDs with ./hack/update-generated-crd-code.sh
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* use existing context in restic and server
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* fix test cases by resetting resource version

also use main library go context, not golang.org/x/net/context, in pkg/restore/restore.go
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* clarify changelog message
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* use github.com/kubernetes-csi/external-snapshotter/v2@v2.2.0-rc1
Signed-off-by: Andrew Lavery <laverya@umich.edu>

* run 'go mod tidy' to remove old external-snapshotter version
Signed-off-by: Andrew Lavery <laverya@umich.edu>

changelogs/unreleased/2651-laverya

+bump Kubernetes module dependencies to v0.18.4 to fix https://github.com/vmware-tanzu/velero/issues/2540 by adding code compatibility with kubernetes v1.18

config/crd/bases/velero.io_backups.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: backups.velero.io
 spec:

config/crd/bases/velero.io_backupstoragelocations.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: backupstoragelocations.velero.io
 spec:

config/crd/bases/velero.io_deletebackuprequests.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: deletebackuprequests.velero.io
 spec:

config/crd/bases/velero.io_downloadrequests.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: downloadrequests.velero.io
 spec:

config/crd/bases/velero.io_podvolumebackups.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: podvolumebackups.velero.io
 spec:

config/crd/bases/velero.io_podvolumerestores.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: podvolumerestores.velero.io
 spec:

config/crd/bases/velero.io_resticrepositories.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: resticrepositories.velero.io
 spec:

config/crd/bases/velero.io_restores.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: restores.velero.io
 spec:

config/crd/bases/velero.io_schedules.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: schedules.velero.io
 spec:

config/crd/bases/velero.io_serverstatusrequests.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: serverstatusrequests.velero.io
 spec:

config/crd/bases/velero.io_volumesnapshotlocations.yaml

@@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.2.4
+    controller-gen.kubebuilder.io/version: v0.3.0
   creationTimestamp: null
   name: volumesnapshotlocations.velero.io
 spec:

config/rbac/role.yaml

+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: manager-role
+rules:
+- apiGroups:
+  - velero.io
+  resources:
+  - backupstoragelocations
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - velero.io
+  resources:
+  - backupstoragelocations/status
+  verbs:
+  - get
+  - patch
+  - update

go.mod

@@ -9,40 +9,75 @@ require (
 	github.com/Azure/go-autorest/autorest/azure/auth v0.4.2
 	github.com/Azure/go-autorest/autorest/to v0.3.0
 	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
-	github.com/aws/aws-sdk-go v1.13.12
+	github.com/aws/aws-sdk-go v1.28.2
 	github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 // indirect
 	github.com/evanphx/json-patch v4.5.0+incompatible
-	github.com/go-ini/ini v1.28.2 // indirect
 	github.com/gobwas/glob v0.2.3
 	github.com/gofrs/uuid v3.2.0+incompatible
 	github.com/golang/protobuf v1.4.2
 	github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd
 	github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26
-	github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 // indirect
 	github.com/joho/godotenv v1.3.0
-	github.com/kubernetes-csi/external-snapshotter/v2 v2.1.0
+	github.com/kubernetes-csi/external-snapshotter/v2 v2.2.0-rc1
 	github.com/onsi/ginkgo v1.13.0
 	github.com/onsi/gomega v1.10.1
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v1.0.0
-	github.com/robfig/cron v0.0.0-20170309132418-df38d32658d8
+	github.com/robfig/cron v1.1.0
 	github.com/sirupsen/logrus v1.4.2
-	github.com/smartystreets/goconvey v1.6.4 // indirect
 	github.com/spf13/afero v1.2.2
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	golang.org/x/net v0.0.0-20200625001655-4c5254603344
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
-	golang.org/x/text v0.3.3 // indirect
-	google.golang.org/grpc v1.27.0
-	google.golang.org/protobuf v1.25.0 // indirect
-	k8s.io/api v0.17.4
-	k8s.io/apiextensions-apiserver v0.17.4
-	k8s.io/apimachinery v0.17.4
-	k8s.io/cli-runtime v0.17.4
-	k8s.io/client-go v0.17.4
+	golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7
+	google.golang.org/grpc v1.28.0
+	k8s.io/api v0.18.4
+	k8s.io/apiextensions-apiserver v0.18.4
+	k8s.io/apimachinery v0.18.4
+	k8s.io/cli-runtime v0.18.4
+	k8s.io/client-go v0.18.4
 	k8s.io/klog v1.0.0
-	k8s.io/utils v0.0.0-20191218082557-f07c713de883 // indirect
-	sigs.k8s.io/controller-runtime v0.5.2
+	sigs.k8s.io/controller-runtime v0.6.1
 )
+
+replace k8s.io/api => k8s.io/api v0.18.4
+
+replace k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.18.4
+
+replace k8s.io/apimachinery => k8s.io/apimachinery v0.18.4
+
+replace k8s.io/apiserver => k8s.io/apiserver v0.18.4
+
+replace k8s.io/cli-runtime => k8s.io/cli-runtime v0.18.4
+
+replace k8s.io/client-go => k8s.io/client-go v0.18.4
+
+replace k8s.io/cloud-provider => k8s.io/cloud-provider v0.18.4
+
+replace k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.18.4
+
+replace k8s.io/code-generator => k8s.io/code-generator v0.18.4
+
+replace k8s.io/component-base => k8s.io/component-base v0.18.4
+
+replace k8s.io/cri-api => k8s.io/cri-api v0.18.4
+
+replace k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.18.4
+
+replace k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.18.4
+
+replace k8s.io/kube-proxy => k8s.io/kube-proxy v0.18.4
+
+replace k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.18.4
+
+replace k8s.io/kubectl => k8s.io/kubectl v0.18.4
+
+replace k8s.io/kubelet => k8s.io/kubelet v0.18.4
+
+replace k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.18.4
+
+replace k8s.io/metrics => k8s.io/metrics v0.18.4
+
+replace k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.18.4
+
+replace k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.18.4

hack/build-image/Dockerfile

@@ -22,7 +22,7 @@ ENV GOPROXY=https://proxy.golang.org
 RUN mkdir -p /go/src/k8s.io
 WORKDIR /go/src/k8s.io
 RUN git config --global advice.detachedHead false
-RUN git clone -b kubernetes-1.17.0 https://github.com/kubernetes/code-generator
+RUN git clone -b v0.18.4 https://github.com/kubernetes/code-generator
 
 RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_linux_amd64.tar.gz && \
     tar -zxvf kubebuilder_2.3.1_linux_amd64.tar.gz && \
@@ -32,7 +32,7 @@ RUN wget --quiet https://github.com/kubernetes-sigs/kubebuilder/releases/downloa
     rm kubebuilder_2.3.1_linux_amd64.tar.gz
 
 # get controller-tools
-RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4
+RUN go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0
 
 # get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit
 # is not important)

pkg/backup/backup.go

@@ -300,7 +300,7 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 
 	backupRequest.Status.Progress = &velerov1api.BackupProgress{TotalItems: len(items)}
 	patch := fmt.Sprintf(`{"status":{"progress":{"totalItems":%d}}}`, len(items))
-	if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(backupRequest.Name, types.MergePatchType, []byte(patch)); err != nil {
+	if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(context.TODO(), backupRequest.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
 		log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress.totalItems")
 	}
 
@@ -354,7 +354,7 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 					backupRequest.Status.Progress.ItemsBackedUp = lastUpdate.itemsBackedUp
 
 					patch := fmt.Sprintf(`{"status":{"progress":{"totalItems":%d,"itemsBackedUp":%d}}}`, lastUpdate.totalItems, lastUpdate.itemsBackedUp)
-					if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(backupRequest.Name, types.MergePatchType, []byte(patch)); err != nil {
+					if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(context.TODO(), backupRequest.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
 						log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress")
 					}
 					lastUpdate = nil
@@ -433,7 +433,7 @@ func (kb *kubernetesBackupper) Backup(log logrus.FieldLogger, backupRequest *Req
 	backupRequest.Status.Progress.ItemsBackedUp = len(backupRequest.BackedUpItems)
 
 	patch = fmt.Sprintf(`{"status":{"progress":{"totalItems":%d,"itemsBackedUp":%d}}}`, len(backupRequest.BackedUpItems), len(backupRequest.BackedUpItems))
-	if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(backupRequest.Name, types.MergePatchType, []byte(patch)); err != nil {
+	if _, err := kb.backupClient.Backups(backupRequest.Namespace).Patch(context.TODO(), backupRequest.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}); err != nil {
 		log.WithError(errors.WithStack((err))).Warn("Got error trying to update backup's status.progress")
 	}
 

pkg/backup/backup_test.go

@@ -2706,9 +2706,9 @@ func (h *harness) addItems(t *testing.T, resource *test.APIResource) {
 		unstructuredObj := &unstructured.Unstructured{Object: obj}
 
 		if resource.Namespaced {
-			_, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(unstructuredObj, metav1.CreateOptions{})
+			_, err = h.DynamicClient.Resource(resource.GVR()).Namespace(item.GetNamespace()).Create(context.TODO(), unstructuredObj, metav1.CreateOptions{})
 		} else {
-			_, err = h.DynamicClient.Resource(resource.GVR()).Create(unstructuredObj, metav1.CreateOptions{})
+			_, err = h.DynamicClient.Resource(resource.GVR()).Create(context.TODO(), unstructuredObj, metav1.CreateOptions{})
 		}
 		require.NoError(t, err)
 	}

pkg/backup/rbac.go

@@ -17,6 +17,8 @@ limitations under the License.
 package backup
 
 import (
+	"context"
+
 	"github.com/pkg/errors"
 	rbac "k8s.io/api/rbac/v1"
 	rbacbeta "k8s.io/api/rbac/v1beta1"
@@ -45,7 +47,7 @@ type v1ClusterRoleBindingLister struct {
 }
 
 func (v1 v1ClusterRoleBindingLister) List() ([]ClusterRoleBinding, error) {
-	crbList, err := v1.client.List(metav1.ListOptions{})
+	crbList, err := v1.client.List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -62,7 +64,7 @@ type v1beta1ClusterRoleBindingLister struct {
 }
 
 func (v1beta1 v1beta1ClusterRoleBindingLister) List() ([]ClusterRoleBinding, error) {
-	crbList, err := v1beta1.client.List(metav1.ListOptions{})
+	crbList, err := v1beta1.client.List(context.TODO(), metav1.ListOptions{})
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}