v1.4.1 cherrypicks (#2704)

* Adjust restic timeout and pod values up (#2696)

* Adjust restic timeout and pod values up
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

*  Use CRD version prior to remap_crd_version backup item action (#2683)

* 

 preserve crd version before remapping
Signed-off-by: Ashish Amarnath <ashisham@vmware.com>

* 

‍♂️ pass git state to build from makefile
Signed-off-by: Ashish Amarnath <ashisham@vmware.com>

* Add scripts for tagging Velero releases (#2592)

* Add release tools
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Document the tag-release release tool
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Make sure the upstream used is correct
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Add copyright statement
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Address review feedback

* Pause to allow for cherry-picking on the release branch before pushing
  it
* Move master branch logic into an else statement
* Correct typo
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>

* Uncomment check for dirty git working tree
Signed-off-by: Nolan Brubaker <brubakern@vmware.com>
Co-authored-by: Ashish Amarnath <ashisham@vmware.com>

Makefile

@@ -40,6 +40,10 @@ CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 windows-amd64 li
 CONTAINER_PLATFORMS ?= linux-amd64 linux-ppc64le linux-arm linux-arm64
 MANIFEST_PLATFORMS ?= amd64 ppc64le arm arm64
 
+# set git sha and tree state
+GIT_SHA = $(shell git rev-parse HEAD)
+GIT_DIRTY = $(shell git status --porcelain 2> /dev/null)
+
 ###
 ### These variables should not need tweaking.
 ###
@@ -113,6 +117,8 @@ local: build-dirs
 	VERSION=$(VERSION) \
 	PKG=$(PKG) \
 	BIN=$(BIN) \
+	GIT_SHA=$(GIT_SHA) \
+	GIT_DIRTY="$(GIT_DIRTY)" \
 	OUTPUT_DIR=$$(pwd)/_output/bin/$(GOOS)/$(GOARCH) \
 	./hack/build.sh
 
@@ -126,6 +132,8 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs
 		VERSION=$(VERSION) \
 		PKG=$(PKG) \
 		BIN=$(BIN) \
+		GIT_SHA=$(GIT_SHA) \
+		GIT_DIRTY=\"$(GIT_DIRTY)\" \
 		OUTPUT_DIR=/output/$(GOOS)/$(GOARCH) \
 		./hack/build.sh'"
 

changelogs/unreleased/2683-ashish-amarnath

+capture version of the CRD prior before invoking the remap_crd_version backup item action

changelogs/unreleased/2696-nrb

+Adjust restic default time out to 4 hours and base pod resource requests to 500m CPU/512Mi memory.

hack/build.sh

@@ -39,10 +39,13 @@ if [ -z "${VERSION}" ]; then
     exit 1
 fi
 
+if [ -z "${GIT_SHA}" ]; then
+    echo "GIT_SHA must be set"
+    exit 1
+fi
+
 export CGO_ENABLED=0
 
-GIT_SHA=$(git rev-parse HEAD)
-GIT_DIRTY=$(git status --porcelain 2> /dev/null)
 if [[ -z "${GIT_DIRTY}" ]]; then
   GIT_TREE_STATE=clean
 else

hack/release-tools/chk_version.go

+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+)
+
+// This regex should match both our GA format (example: v1.4.3) and pre-release format (v1.2.4-beta.2)
+// The following sub-capture groups are defined:
+//	major
+//	minor
+//	patch
+//	prerelease (this will be alpha/beta followed by a ".", followed by 1 or more digits (alpha.5)
+var release_regex *regexp.Regexp = regexp.MustCompile("^v(?P<major>[[:digit:]]+)\\.(?P<minor>[[:digit:]]+)\\.(?P<patch>[[:digit:]]+)(-{1}(?P<prerelease>(alpha|beta)\\.[[:digit:]]+))*")
+
+// This small program exists because checking the VELERO_VERSION rules in bash is difficult, and difficult to test for correctness.
+// Calling it with --verify will verify whether or not the VELERO_VERSION environment variable is a valid version string, without parsing for its components.
+// Calling it without --verify will try to parse the version into its component pieces.
+func main() {
+
+	velero_version := os.Getenv("VELERO_VERSION")
+
+	submatches := reSubMatchMap(release_regex, velero_version)
+
+	// Didn't match the regex, exit.
+	if len(submatches) == 0 {
+		fmt.Printf("VELERO_VERSION of %s was not valid. Please correct the value and retry.", velero_version)
+		os.Exit(1)
+	}
+
+	if len(os.Args) > 1 && os.Args[1] == "--verify" {
+		os.Exit(0)
+	}
+
+	// Send these in a bash variable format to stdout, so that they can be consumed by bash scripts that call the go program.
+	fmt.Printf("VELERO_MAJOR=%s\n", submatches["major"])
+	fmt.Printf("VELERO_MINOR=%s\n", submatches["minor"])
+	fmt.Printf("VELERO_PATCH=%s\n", submatches["patch"])
+	fmt.Printf("VELERO_PRERELEASE=%s\n", submatches["prerelease"])
+}
+
+// reSubMatchMap returns a map with the named submatches within a regular expression populated as keys, and their matched values within a given string as values.
+// If no matches are found, a nil map is returned
+func reSubMatchMap(r *regexp.Regexp, s string) map[string]string {
+	match := r.FindStringSubmatch(s)
+	submatches := make(map[string]string)
+	if len(match) == 0 {
+		return submatches
+	}
+	for i, name := range r.SubexpNames() {
+		// 0 will always be empty from the return values of SubexpNames's documentation, so skip it.
+		if i != 0 {
+			submatches[name] = match[i]
+		}
+	}
+
+	return submatches
+}

hack/release-tools/chk_version_test.go

+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestRegexMatching(t *testing.T) {
+	tests := []struct {
+		version     string
+		expectMatch bool
+	}{
+		{
+			version:     "v1.4.0",
+			expectMatch: true,
+		},
+		{
+			version:     "v2.0.0",
+			expectMatch: true,
+		},
+		{
+			version:     "v1.5.0-alpha.1",
+			expectMatch: true,
+		},
+		{
+			version:     "v1.16.1320-beta.14",
+			expectMatch: true,
+		},
+		{
+			version:     "1.0.0",
+			expectMatch: false,
+		},
+		{
+			// this is true because while the "--" is invalid, v1.0.0 is a valid part of the regex
+			version:     "v1.0.0--beta.1",
+			expectMatch: true,
+		},
+	}
+
+	for _, test := range tests {
+		name := fmt.Sprintf("Testing version string %s", test.version)
+		t.Run(name, func(t *testing.T) {
+			results := reSubMatchMap(release_regex, test.version)
+
+			if len(results) == 0 && test.expectMatch {
+				t.Fail()
+			}
+
+			if len(results) > 0 && !test.expectMatch {
+				fmt.Printf("%v", results)
+				t.Fail()
+			}
+		})
+	}
+}

hack/release-tools/tag-release.sh

+#!/usr/bin/env bash
+
+# Copyright 2020 the Velero contributors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This script will do the necessary checks and actions to create a release of Velero.
+# It will first validate that all prerequisites are met, then verify the version string is what the user expects.
+# A git tag will be created and pushed to GitHub, and GoReleaser will be invoked.
+
+# This script is meant to be a combination of documentation and executable.
+# If you have questions at any point, please stop and ask!
+
+# Directory in which the script itself resides, so we can use it for calling programs that are in the same directory.
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+function tag_and_push() {
+    echo "Tagging and pushing $VELERO_VERSION"
+    git tag $VELERO_VERSION
+    git push $VELERO_VERSION
+}
+
+# For now, have the person doing the release pass in the VELERO_VERSION variable as an environment variable.
+# In the future, we might be able to inspect git via `git describe --abbrev=0` to get a hint for it.
+if [[ -z "$VELERO_VERSION" ]]; then
+    printf "The \$VELERO_VERSION environment variable is not set. Please set it with\n\texport VELERO_VERSION=v<version.to.release>\nthen try again."
+    exit 1
+fi
+
+# Make sure the user's provided their github token, so we can give it to goreleaser.
+if [[ -z "$GITHUB_TOKEN" ]]; then
+    printf "The GITHUB_TOKEN environment variable is not set. Please set it with\n\t export GITHUB_TOKEN=<your github token>\n then try again."
+    exit 1
+fi
+
+ Ensure that we have a clean working tree before we let any changes happen, especially important for cutting release branches.
+if [[ -n $(git status --short) ]]; then 
+    echo "Your git working directory is dirty! Please clean up untracked files and stash any changes before proceeding."
+    exit 3
+fi
+
+# Make sure that there's no issue with the environment variable's format before trying to eval the parsed version.
+if ! go run $DIR/chk_version.go --verify;  then
+    exit 2
+fi
+# Since we're past the validation of the VELERO_VERSION, parse the version's individual components.
+eval $(go run $DIR/chk_version.go)
+
+
+printf "To clarify, you've provided a version string of $VELERO_VERSION.\n"
+printf "Based on this, the following assumptions have been made: \n"
+
+[[ "$VELERO_PATCH" != 0 ]] && printf "*\t This is a patch release.\n"
+
+# -n is "string is non-empty"
+[[ -n $VELERO_PRERELEASE ]] && printf "*\t This is a pre-release.\n"
+
+# -z is "string is empty"
+[[ -z $VELERO_PRERELEASE ]] && printf "*\t This is a GA release.\n"
+
+echo "If this is all correct, press enter/return to proceed to TAG THE RELEASE and UPLOAD THE TAG TO GITHUB."
+echo "Otherwise, press ctrl-c to CANCEL the process without making any changes."
+
+read -p "Ready to continue? "
+
+echo "Alright, let's go."
+
+echo "Pulling down all git tags and branches before doing any work."
+git fetch upstream --all --tags
+
+# If we've got a patch release, we'll need to create a release branch for it.
+if [[ "$VELERO_PATCH" > 0 ]]; then
+    release_branch_name=release-$VELERO_MAJOR.$VELERO_MINOR
+  
+    # Check if the branch exists, creating it if not.
+    # The fetch command above should have gotten all the upstream branches, so we can safely assume this check is local & upstream branches.
+    if [[ -z $(git branch | grep $release_branch_name) ]]; then
+        git checkout -b $release_branch_name
+        echo "Release branch made."
+    else
+        echo "Release branch $release_branch_name exists already."
+        git checkout $release_branch_name
+    fi
+
+    echo "Now you'll need to cherry-pick any relevant git commits into this release branch."
+    echo "Either pause this script with ctrl-z, or open a new terminal window and do the cherry-picking."
+    read -p "Press enter when you're done cherry-picking. THIS WILL MAKE A TAG PUSH THE BRANCH TO UPSTREAM"
+
+    # TODO can/should we add a way to review the cherry-picked commits before the push?
+
+    echo "Pushing $release_branch_name to upstream remote"
+    git push --set-upstream upstream/$release_branch_name $release_branch_name
+    
+    tag_and_push
+else
+    echo "Checking out upstream/master."
+    git checkout upstream/master
+
+    tag_and_push
+fi
+
+
+
+echo "Invoking Goreleaser to create the GitHub release."
+RELEASE_NOTES_FILE=changelogs/CHANGELOG-$VELERO_MAJOR.$VELERO_MINOR.md \
+    PUBLISH=TRUE \
+    make release

pkg/backup/item_backupper.go

@@ -159,6 +159,12 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr
 		}
 	}
 
+	// capture the version of the object before invoking plugin actions as the plugin may update
+	// the group version of the object.
+	// group version of this object
+	// Used on filepath to backup up all groups and versions
+	version := resourceVersion(obj)
+
 	updatedObj, err := ib.executeActions(log, obj, groupResource, name, namespace, metadata)
 	if err != nil {
 		backupErrs = append(backupErrs, err)
@@ -203,10 +209,6 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr
 		return false, kubeerrs.NewAggregate(backupErrs)
 	}
 
-	// group version of this object
-	// Used on filepath to backup up all groups and versions
-	version := resourceVersion(obj)
-
 	// Getting the preferred group version of this resource
 	preferredVersion := preferredGVR.Version
 
@@ -249,6 +251,7 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr
 
 	// backing up the preferred version backup without API Group version on path -  this is for backward compability
 
+	log.Debugf("Resource %s/%s, version= %s, preferredVersion=%s", groupResource.String(), name, version, preferredVersion)
 	if version == preferredVersion {
 		if namespace != "" {
 			filePath = filepath.Join(velerov1api.ResourcesDir, groupResource.String(), velerov1api.NamespaceScopedDir, namespace, name+".json")
@@ -271,7 +274,6 @@ func (ib *itemBackupper) backupItem(logger logrus.FieldLogger, obj runtime.Unstr
 		if _, err := ib.tarWriter.Write(itemBytes); err != nil {
 			return false, errors.WithStack(err)
 		}
-
 	}
 
 	return true, nil

pkg/backup/remap_crd_version_action_test.go

@@ -170,6 +170,7 @@ func TestRemapCRDVersionActionData(t *testing.T) {
 			require.NoError(t, err)
 
 			assert.Equal(t, "apiextensions.k8s.io/v1beta1", item.UnstructuredContent()["apiVersion"])
+			assert.Equal(t, crd.Kind, item.GetObjectKind().GroupVersionKind().GroupKind().Kind)
 			name, _, err := unstructured.NestedString(item.UnstructuredContent(), "metadata", "name")
 			require.NoError(t, err)
 			assert.Equal(t, crd.Name, name)

pkg/cmd/server/server.go

@@ -75,7 +75,7 @@ const (
 	defaultMetricsAddress = ":8085"
 
 	defaultBackupSyncPeriod           = time.Minute
-	defaultPodVolumeOperationTimeout  = 60 * time.Minute
+	defaultPodVolumeOperationTimeout  = 240 * time.Minute
 	defaultResourceTerminatingTimeout = 10 * time.Minute
 
 	// server's client default qps and burst

pkg/install/resources.go

@@ -46,10 +46,10 @@ var (
 	DefaultVeleroPodMemRequest = "128Mi"
 	DefaultVeleroPodCPULimit   = "1000m"
 	DefaultVeleroPodMemLimit   = "256Mi"
-	DefaultResticPodCPURequest = "0"
-	DefaultResticPodMemRequest = "0"
-	DefaultResticPodCPULimit   = "0"
-	DefaultResticPodMemLimit   = "0"
+	DefaultResticPodCPURequest = "500m"
+	DefaultResticPodMemRequest = "512Mi"
+	DefaultResticPodCPULimit   = "1000m"
+	DefaultResticPodMemLimit   = "1Gi"
 )
 
 func labels() map[string]string {

pkg/kuberesource/kuberesource.go

@@ -23,7 +23,7 @@ import (
 var (
 	ClusterRoleBindings       = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterrolebindings"}
 	ClusterRoles              = schema.GroupResource{Group: "rbac.authorization.k8s.io", Resource: "clusterroles"}
-	CustomResourceDefinitions = schema.GroupResource{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}
+	CustomResourceDefinitions = schema.GroupResource{Group: "apiextensions.k8s.io", Resource: "CustomResourceDefinition"}
 	Jobs                      = schema.GroupResource{Group: "batch", Resource: "jobs"}
 	Namespaces                = schema.GroupResource{Group: "", Resource: "namespaces"}
 	PersistentVolumeClaims    = schema.GroupResource{Group: "", Resource: "persistentvolumeclaims"}

site/docs/master/release-instructions.md

-# Release Instructions
-
-## Ahead of Time
-
-### (GA Only) Release Blog Post PR
-
-Prepare a PR containing the release blog post. It's usually easiest to make a copy of the most recent existing post, then replace the content as appropriate.
-
-You also need to update `site/index.html` to have "Latest Release Information" contain a link to the new post.
-
-### (Pre-Release and GA) Changelog and Docs PR
-
-1.  In a branch, create the file `changelogs/CHANGELOG-<major>.<minor>.md` (if it doesn't already exist) by copying the most recent one.
-1.  Run `make changelog` to generate a list of all unreleased changes. Copy/paste the output into `CHANGELOG-<major>.<minor>.md`, under the "All Changes" section for the release.
-    - You *may* choose to tweak formatting on the list of changes by adding code blocks, etc.
-1.  (GA Only) Remove all changelog files from `changelogs/unreleased`.
-1.  Update the main `CHANGELOG.md` file to properly reference the release-specific changelog file:
-    - (Pre-Release) List the release under "Development release"
-    - (GA) List the release  under "Current release", remove any pre-releases from "Development release", and move the previous release into "Older releases".
-1.  If there is an existing set of pre-release versioned docs for the version you are releasing (i.e. `site/docs/v1.4-pre` exists, and you're releasing `v1.4.0-beta.2` or `v1.4.0`):
-    - Remove the directory containing the pre-release docs, i.e. `site/docs/<pre-release-version>`.
-    - Delete the pre-release docs table of contents file, i.e. `site/_data/<pre-release-version>-toc.yml`.
-    - Remove the pre-release docs table of contents mapping entry from `site/_data/toc-mapping.yml`.
-    - Remove all references to the pre-release docs from `site/_config.yml`.
-1.  Run `NEW_DOCS_VERSION=v<major.minor> VELERO_VERSION=v<full-version> make gen-docs` (e.g. `NEW_DOCS_VERSION=v1.2 VELERO_VERSION=v1.2.0 make gen-docs` or `NEW_DOCS_VERSION=v1.2-pre VELERO_VERSION=v1.2.0-beta.1 make gen-docs`).
-    - Note that:
-        - **NEW_DOCS_VERSION** defines the version that the docs will be tagged with (i.e. what's in the URL, what shows up in the version dropdown on the site). This should be formatted as either `v1.4` (for a GA release), or `v1.4-pre` (for an alpha/beta/RC).
-        - **VELERO_VERSION** defines the tag of Velero that any `https://github.com/vmware-tanzu/velero/...` links in the docs should redirect to.
-1.  Follow the additional instructions at `site/README-JEKYLL.md` to complete the docs generation process.
-1.  Do a review of the diffs, and/or run `make serve-docs` and review the site.
-1.  Submit a PR containing the changelog and the version-tagged docs.
-
-### (Pre-Release and GA) GitHub Token
-
-To run the `goreleaser` process to generate a GitHub release, you'll need to have a GitHub token. See https://goreleaser.com/environment/ for more details. 
-
-You may regenerate the token for every release if you prefer.
-
-#### If you don't already have a token
-1.  Go to https://github.com/settings/tokens/new.
-1.  Choose a name for your token.
-1.  Check the "repo" scope.
-1.  Click "Generate token".
-1.  Save the token value somewhere - you'll need it during the release, in the `GITHUB_TOKEN` environment variable.
-
-#### If you do already have a token, but need to regenerate it
-1.  Go to https://github.com/settings/tokens.
-1.  Click on the name of the relevant token.
-1.  Click "Regenerate token".
-1.  Save the token value somewhere - you'll need it during the release, in the `GITHUB_TOKEN` environment variable.
-
-## During Release
-
-This process is the same for both pre-release and GA, except for the fact that there will not be a blog post PR to merge for pre-release versions.
-
-1.  Merge the changelog + docs PR, so that it's included in the release tag.
-1.  Make sure your working directory is clean: `git status` should show `nothing to commit, working tree clean`. 
-1.  Run `git fetch upstream master && git checkout upstream/master`.
-1.  Run `git tag <VERSION>` (e.g. `git tag v1.2.0` or `git tag v1.2.0-beta.1`).
-1.  Run `git push upstream <VERSION>` (e.g. `git push upstream v1.2.0` or `git push upstream v1.2.0-beta.1`). This will trigger the Travis CI job that builds/publishes the Docker images.
-1.  Generate the GitHub release (it will be created in "Draft" status, which means it's not visible to the outside world until you click "Publish"):
-
-    ```bash
-    GITHUB_TOKEN=your-github-token \
-    RELEASE_NOTES_FILE=changelogs/CHANGELOG-<major>.<minor>.md \
-    PUBLISH=true \
-    make release
-    ```
-
-1.  Navigate to the draft GitHub release, at https://github.com/vmware-tanzu/velero/releases.
-1.  If this is a patch release (e.g. `v1.2.1`), note that the full `CHANGELOG-1.2.md` contents will be included in the body of the GitHub release. You need to delete the previous releases' content (e.g. `v1.2.0`'s changelog) so that only the latest patch release's changelog shows.
-1.  Do a quick review for formatting. **Note:** the `goreleaser` process should detect if it's a pre-release version, and check that box in the GitHub release appropriately, but it's always worth double-checking.
-1.  Publish the release.
-1.  By now, the Docker images should have been published. Perform a smoke-test - for example:
-    - Download the CLI from the GitHub release
-    - Use it to install Velero into a cluster (or manually update an existing deployment to use the new images)
-    - Verify that `velero version` shows the expected output
-    - Run a backup/restore and ensure it works
-1.  (GA Only) Merge the blog post PR.
-1.  Announce the release:
-    - Twitter (mention a few highlights, link to the blog post)
-    - Slack channel
-    - Google group (this doesn't get a lot of traffic, and recent releases may not have been posted here)