Merge pull request #10495 from hakman/automated-cherry-pick-of-#10473-upstream-release-1.19

Automated cherry pick of #10473: Add config options for container runtime package URL and Hash

cmd/kops/integration_test.go

@@ -334,9 +334,19 @@ func TestMixedInstancesSpotASG(t *testing.T) {
 	newIntegrationTest("mixedinstances.example.com", "mixed_instances_spot").withZones(3).runTestCloudformation(t)
 }
 
-// TestContainerdCloudformation runs the test on a containerd configuration
-func TestContainerdCloudformation(t *testing.T) {
-	newIntegrationTest("containerd.example.com", "containerd-cloudformation").runTestCloudformation(t)
+// TestContainerd runs the test on a containerd configuration
+func TestContainerd(t *testing.T) {
+	newIntegrationTest("containerd.example.com", "containerd").runTestCloudformation(t)
+}
+
+// TestContainerdCustom runs the test on a custom containerd URL configuration
+func TestContainerdCustom(t *testing.T) {
+	newIntegrationTest("containerd.example.com", "containerd-custom").runTestCloudformation(t)
+}
+
+// TestDockerCustom runs the test on a custom Docker URL configuration
+func TestDockerCustom(t *testing.T) {
+	newIntegrationTest("docker.example.com", "docker-custom").runTestCloudformation(t)
 }
 
 // TestLaunchConfigurationASG tests ASGs using launch configurations instead of launch templates

docs/cluster_spec.md

@@ -930,21 +930,49 @@ spec:
 
 ## containerd
 
+### Configuration
+
 It is possible to override the [containerd](https://github.com/containerd/containerd/blob/master/README.md) daemon options for all the nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#ContainerdConfig) for the full list of options.
 
 ```yaml
 spec:
   containerd:
-    version: 1.3.3
+    version: 1.4.3
     logLevel: info
     configOverride: ""
 ```
 
-## docker
+### Custom Packages
+
+kOps uses the `.tar.gz` packages for installing containerd on any supported OS. This makes it easy to use a custom build or pre-release packages, by specifying its URL and sha256:
+
+```yaml
+spec:
+  containerd:
+    packages:
+      urlAmd64: https://github.com/containerd/containerd/releases/download/v1.4.3/cri-containerd-cni-1.4.3-linux-amd64.tar.gz
+      hashAmd64: 2697a342e3477c211ab48313e259fd7e32ad1f5ded19320e6a559f50a82bff3d
+```
+
+The format of the custom package must be identical to the official packages:
+
+```bash
+tar tf cri-containerd-cni-1.4.3-linux-amd64.tar.gz
+    usr/local/bin/containerd
+    usr/local/bin/containerd-shim
+    usr/local/bin/containerd-shim-runc-v1
+    usr/local/bin/containerd-shim-runc-v2
+    usr/local/bin/crictl
+    usr/local/bin/critest
+    usr/local/bin/ctr
+    usr/local/sbin/runc
+```
+
+## Docker
 
 It is possible to override Docker daemon options for all masters and nodes in the cluster. See the [API docs](https://pkg.go.dev/k8s.io/kops/pkg/apis/kops#DockerConfig) for the full list of options.
 
-### registryMirrors
+### Registry Mirrors
 
 If you have a bunch of Docker instances (physical or vm) running, each time one of them pulls an image that is not present on the host, it will fetch it from the internet (DockerHub). By caching these images, you can keep the traffic within your local network and avoid egress bandwidth usage.
 This setting benefits not only cluster provisioning but also image pulling.
@@ -971,7 +999,7 @@ spec:
 
 **NOTE:** When this field is set to `true`, it is entirely up to the user to install and configure Docker.
 
-### storage
+### Storage
 
 The Docker [Storage Driver](https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-storage-driver) can be specified in order to override the default. Be sure the driver you choose is supported by your operating system and docker version.
 
@@ -984,7 +1012,7 @@ docker:
     - "dm.use_deferred_removal=true"
 ```
 
-### networking
+### Networking
 
 In order for containers started with `docker run` instead of Kubernetes to have network and internet access you need to enable the necessary [iptables](https://docs.docker.com/network/iptables/) rules:
 
@@ -994,6 +1022,33 @@ docker:
   ipTables: true
 ```
 
+### Custom Packages
+
+kOps uses the `.tgz` (static) packages for installing Docker on any supported OS. This makes it easy to use a custom build or pre-release packages, by specifying its URL and sha256:
+
+```yaml
+spec:
+  containerd:
+    packages:
+      urlAmd64: https://download.docker.com/linux/static/stable/x86_64/docker-20.10.1.tgz
+      hashAmd64: 8790f3b94ee07ca69a9fdbd1310cbffc729af0a07e5bf9f34a79df1e13d2e50e
+```
+
+The format of the custom package must be identical to the official packages:
+
+```bash
+tar tf docker-20.10.1.tgz
+    docker/containerd
+    docker/containerd-shim
+    docker/containerd-shim-runc-v2
+    docker/ctr
+    docker/docker
+    docker/docker-init
+    docker/docker-proxy
+    docker/dockerd
+    docker/runc
+```
+
 ## sshKeyName
 
 In some cases, it may be desirable to use an existing AWS SSH key instead of allowing kops to create a new one.

k8s/crds/kops.k8s.io_clusters.yaml

@@ -385,25 +385,41 @@ spec:
                 description: Component configurations
                 properties:
                   address:
-                    description: Address of containerd's GRPC server (default "/run/containerd/containerd.sock")
+                    description: Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
                     type: string
                   configOverride:
-                    description: Complete containerd config file provided by the user
+                    description: ConfigOverride is the complete containerd config file provided by the user.
                     type: string
                   logLevel:
-                    description: Logging level [trace, debug, info, warn, error, fatal, panic] (default "info")
+                    description: LogLevel controls the logging details [trace, debug, info, warn, error, fatal, panic] (default "info").
                     type: string
+                  packages:
+                    description: Packages overrides the URL and hash for the packages.
+                    properties:
+                      hashAmd64:
+                        description: HashAmd64 overrides the hash for the AMD64 package.
+                        type: string
+                      hashArm64:
+                        description: HashArm64 overrides the hash for the ARM64 package.
+                        type: string
+                      urlAmd64:
+                        description: UrlAmd64 overrides the URL for the AMD64 package.
+                        type: string
+                      urlArm64:
+                        description: UrlArm64 overrides the URL for the ARM64 package.
+                        type: string
+                    type: object
                   root:
-                    description: Directory for persistent data (default "/var/lib/containerd")
+                    description: Root directory for persistent data (default "/var/lib/containerd").
                     type: string
                   skipInstall:
-                    description: Prevents kops from installing and modifying containerd in any way (default "false")
+                    description: SkipInstall prevents kOps from installing and modifying containerd in any way (default "false").
                     type: boolean
                   state:
-                    description: Directory for execution state files (default "/run/containerd")
+                    description: State directory for execution state files (default "/run/containerd").
                     type: string
                   version:
-                    description: Consumed by nodeup and used to pick the containerd version
+                    description: Version used to pick the containerd package.
                     type: string
                 type: object
               dnsControllerGossipConfig:
@@ -501,6 +517,22 @@ spec:
                     description: MTU is the containers network MTU
                     format: int32
                     type: integer
+                  packages:
+                    description: Packages overrides the URL and hash for the packages.
+                    properties:
+                      hashAmd64:
+                        description: HashAmd64 overrides the hash for the AMD64 package.
+                        type: string
+                      hashArm64:
+                        description: HashArm64 overrides the hash for the ARM64 package.
+                        type: string
+                      urlAmd64:
+                        description: UrlAmd64 overrides the URL for the AMD64 package.
+                        type: string
+                      urlArm64:
+                        description: UrlArm64 overrides the URL for the ARM64 package.
+                        type: string
+                    type: object
                   registryMirrors:
                     description: RegistryMirrors is a referred list of docker registry mirror
                     items:

pkg/apis/kops/cluster.go

@@ -772,3 +772,14 @@ type RollingUpdate struct {
 	// +optional
 	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
 }
+
+type PackagesConfig struct {
+	// HashAmd64 overrides the hash for the AMD64 package.
+	HashAmd64 *string `json:"hashAmd64,omitempty"`
+	// HashArm64 overrides the hash for the ARM64 package.
+	HashArm64 *string `json:"hashArm64,omitempty"`
+	// UrlAmd64 overrides the URL for the AMD64 package.
+	UrlAmd64 *string `json:"urlAmd64,omitempty"`
+	// UrlArm64 overrides the URL for the ARM64 package.
+	UrlArm64 *string `json:"urlArm64,omitempty"`
+}

pkg/apis/kops/containerdconfig.go

@@ -18,18 +18,20 @@ package kops
 
 // ContainerdConfig is the configuration for containerd
 type ContainerdConfig struct {
-	// Address of containerd's GRPC server (default "/run/containerd/containerd.sock")
+	// Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
 	Address *string `json:"address,omitempty" flag:"address"`
-	// Complete containerd config file provided by the user
+	// ConfigOverride is the complete containerd config file provided by the user.
 	ConfigOverride *string `json:"configOverride,omitempty"`
-	// Logging level [trace, debug, info, warn, error, fatal, panic] (default "info")
+	// LogLevel controls the logging details [trace, debug, info, warn, error, fatal, panic] (default "info").
 	LogLevel *string `json:"logLevel,omitempty" flag:"log-level"`
-	// Directory for persistent data (default "/var/lib/containerd")
+	// Packages overrides the URL and hash for the packages.
+	Packages *PackagesConfig `json:"packages,omitempty"`
+	// Root directory for persistent data (default "/var/lib/containerd").
 	Root *string `json:"root,omitempty" flag:"root"`
-	// Prevents kops from installing and modifying containerd in any way (default "false")
+	// SkipInstall prevents kOps from installing and modifying containerd in any way (default "false").
 	SkipInstall bool `json:"skipInstall,omitempty"`
-	// Directory for execution state files (default "/run/containerd")
+	// State directory for execution state files (default "/run/containerd").
 	State *string `json:"state,omitempty" flag:"state"`
-	// Consumed by nodeup and used to pick the containerd version
+	// Version used to pick the containerd package.
 	Version *string `json:"version,omitempty"`
 }

pkg/apis/kops/dockerconfig.go

@@ -60,6 +60,8 @@ type DockerConfig struct {
 	MetricsAddress *string `json:"metricsAddress,omitempty" flag:"metrics-addr"`
 	// MTU is the containers network MTU
 	MTU *int32 `json:"mtu,omitempty" flag:"mtu"`
+	// Packages overrides the URL and hash for the packages.
+	Packages *PackagesConfig `json:"packages,omitempty"`
 	// RegistryMirrors is a referred list of docker registry mirror
 	RegistryMirrors []string `json:"registryMirrors,omitempty" flag:"registry-mirror,repeat"`
 	// Runtimes registers an additional OCI compatible runtime (default [])

pkg/apis/kops/v1alpha2/cluster.go

@@ -657,3 +657,14 @@ type RollingUpdate struct {
 	// +optional
 	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
 }
+
+type PackagesConfig struct {
+	// HashAmd64 overrides the hash for the AMD64 package.
+	HashAmd64 *string `json:"hashAmd64,omitempty"`
+	// HashArm64 overrides the hash for the ARM64 package.
+	HashArm64 *string `json:"hashArm64,omitempty"`
+	// UrlAmd64 overrides the URL for the AMD64 package.
+	UrlAmd64 *string `json:"urlAmd64,omitempty"`
+	// UrlArm64 overrides the URL for the ARM64 package.
+	UrlArm64 *string `json:"urlArm64,omitempty"`
+}

pkg/apis/kops/v1alpha2/containerdconfig.go

@@ -18,18 +18,20 @@ package v1alpha2
 
 // ContainerdConfig is the configuration for containerd
 type ContainerdConfig struct {
-	// Address of containerd's GRPC server (default "/run/containerd/containerd.sock")
+	// Address of containerd's GRPC server (default "/run/containerd/containerd.sock").
 	Address *string `json:"address,omitempty" flag:"address"`
-	// Complete containerd config file provided by the user
+	// ConfigOverride is the complete containerd config file provided by the user.
 	ConfigOverride *string `json:"configOverride,omitempty"`
-	// Logging level [trace, debug, info, warn, error, fatal, panic] (default "info")
+	// LogLevel controls the logging details [trace, debug, info, warn, error, fatal, panic] (default "info").
 	LogLevel *string `json:"logLevel,omitempty" flag:"log-level"`
-	// Directory for persistent data (default "/var/lib/containerd")
+	// Packages overrides the URL and hash for the packages.
+	Packages *PackagesConfig `json:"packages,omitempty"`
+	// Root directory for persistent data (default "/var/lib/containerd").
 	Root *string `json:"root,omitempty" flag:"root"`
-	// Prevents kops from installing and modifying containerd in any way (default "false")
+	// SkipInstall prevents kOps from installing and modifying containerd in any way (default "false").
 	SkipInstall bool `json:"skipInstall,omitempty"`
-	// Directory for execution state files (default "/run/containerd")
+	// State directory for execution state files (default "/run/containerd").
 	State *string `json:"state,omitempty" flag:"state"`
-	// Consumed by nodeup and used to pick the containerd version
+	// Version used to pick the containerd package.
 	Version *string `json:"version,omitempty"`
 }

pkg/apis/kops/v1alpha2/dockerconfig.go

@@ -60,6 +60,8 @@ type DockerConfig struct {
 	MetricsAddress *string `json:"metricsAddress,omitempty" flag:"metrics-addr"`
 	// MTU is the containers network MTU
 	MTU *int32 `json:"mtu,omitempty" flag:"mtu"`
+	// Packages overrides the URL and hash for the packages.
+	Packages *PackagesConfig `json:"packages,omitempty"`
 	// RegistryMirrors is a referred list of docker registry mirror
 	RegistryMirrors []string `json:"registryMirrors,omitempty" flag:"registry-mirror,repeat"`
 	// Runtimes registers an additional OCI compatible runtime (default [])

pkg/apis/kops/v1alpha2/zz_generated.conversion.go

@@ -823,6 +823,16 @@ func RegisterConversions(s *runtime.Scheme) error {
 	}); err != nil {
 		return err
 	}
+	if err := s.AddGeneratedConversionFunc((*PackagesConfig)(nil), (*kops.PackagesConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(a.(*PackagesConfig), b.(*kops.PackagesConfig), scope)
+	}); err != nil {
+		return err
+	}
+	if err := s.AddGeneratedConversionFunc((*kops.PackagesConfig)(nil), (*PackagesConfig)(nil), func(a, b interface{}, scope conversion.Scope) error {
+		return Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(a.(*kops.PackagesConfig), b.(*PackagesConfig), scope)
+	}); err != nil {
+		return err
+	}
 	if err := s.AddGeneratedConversionFunc((*RBACAuthorizationSpec)(nil), (*kops.RBACAuthorizationSpec)(nil), func(a, b interface{}, scope conversion.Scope) error {
 		return Convert_v1alpha2_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(a.(*RBACAuthorizationSpec), b.(*kops.RBACAuthorizationSpec), scope)
 	}); err != nil {
@@ -2614,6 +2624,15 @@ func autoConvert_v1alpha2_ContainerdConfig_To_kops_ContainerdConfig(in *Containe
 	out.Address = in.Address
 	out.ConfigOverride = in.ConfigOverride
 	out.LogLevel = in.LogLevel
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(kops.PackagesConfig)
+		if err := Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Packages = nil
+	}
 	out.Root = in.Root
 	out.SkipInstall = in.SkipInstall
 	out.State = in.State
@@ -2630,6 +2649,15 @@ func autoConvert_kops_ContainerdConfig_To_v1alpha2_ContainerdConfig(in *kops.Con
 	out.Address = in.Address
 	out.ConfigOverride = in.ConfigOverride
 	out.LogLevel = in.LogLevel
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(PackagesConfig)
+		if err := Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Packages = nil
+	}
 	out.Root = in.Root
 	out.SkipInstall = in.SkipInstall
 	out.State = in.State
@@ -2746,6 +2774,15 @@ func autoConvert_v1alpha2_DockerConfig_To_kops_DockerConfig(in *DockerConfig, ou
 	out.LogOpt = in.LogOpt
 	out.MetricsAddress = in.MetricsAddress
 	out.MTU = in.MTU
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(kops.PackagesConfig)
+		if err := Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Packages = nil
+	}
 	out.RegistryMirrors = in.RegistryMirrors
 	out.Runtimes = in.Runtimes
 	out.SelinuxEnabled = in.SelinuxEnabled
@@ -2784,6 +2821,15 @@ func autoConvert_kops_DockerConfig_To_v1alpha2_DockerConfig(in *kops.DockerConfi
 	out.LogOpt = in.LogOpt
 	out.MetricsAddress = in.MetricsAddress
 	out.MTU = in.MTU
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(PackagesConfig)
+		if err := Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(*in, *out, s); err != nil {
+			return err
+		}
+	} else {
+		out.Packages = nil
+	}
 	out.RegistryMirrors = in.RegistryMirrors
 	out.Runtimes = in.Runtimes
 	out.SelinuxEnabled = in.SelinuxEnabled
@@ -5478,6 +5524,32 @@ func Convert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in *kops.Openstack
 	return autoConvert_kops_OpenstackRouter_To_v1alpha2_OpenstackRouter(in, out, s)
 }
 
+func autoConvert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(in *PackagesConfig, out *kops.PackagesConfig, s conversion.Scope) error {
+	out.HashAmd64 = in.HashAmd64
+	out.HashArm64 = in.HashArm64
+	out.UrlAmd64 = in.UrlAmd64
+	out.UrlArm64 = in.UrlArm64
+	return nil
+}
+
+// Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig is an autogenerated conversion function.
+func Convert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(in *PackagesConfig, out *kops.PackagesConfig, s conversion.Scope) error {
+	return autoConvert_v1alpha2_PackagesConfig_To_kops_PackagesConfig(in, out, s)
+}
+
+func autoConvert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(in *kops.PackagesConfig, out *PackagesConfig, s conversion.Scope) error {
+	out.HashAmd64 = in.HashAmd64
+	out.HashArm64 = in.HashArm64
+	out.UrlAmd64 = in.UrlAmd64
+	out.UrlArm64 = in.UrlArm64
+	return nil
+}
+
+// Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig is an autogenerated conversion function.
+func Convert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(in *kops.PackagesConfig, out *PackagesConfig, s conversion.Scope) error {
+	return autoConvert_kops_PackagesConfig_To_v1alpha2_PackagesConfig(in, out, s)
+}
+
 func autoConvert_v1alpha2_RBACAuthorizationSpec_To_kops_RBACAuthorizationSpec(in *RBACAuthorizationSpec, out *kops.RBACAuthorizationSpec, s conversion.Scope) error {
 	return nil
 }

pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go

@@ -981,6 +981,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(PackagesConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.Root != nil {
 		in, out := &in.Root, &out.Root
 		*out = new(string)
@@ -1185,6 +1190,11 @@ func (in *DockerConfig) DeepCopyInto(out *DockerConfig) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(PackagesConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.RegistryMirrors != nil {
 		in, out := &in.RegistryMirrors, &out.RegistryMirrors
 		*out = make([]string, len(*in))
@@ -3757,6 +3767,42 @@ func (in *OpenstackRouter) DeepCopy() *OpenstackRouter {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PackagesConfig) DeepCopyInto(out *PackagesConfig) {
+	*out = *in
+	if in.HashAmd64 != nil {
+		in, out := &in.HashAmd64, &out.HashAmd64
+		*out = new(string)
+		**out = **in
+	}
+	if in.HashArm64 != nil {
+		in, out := &in.HashArm64, &out.HashArm64
+		*out = new(string)
+		**out = **in
+	}
+	if in.UrlAmd64 != nil {
+		in, out := &in.UrlAmd64, &out.UrlAmd64
+		*out = new(string)
+		**out = **in
+	}
+	if in.UrlArm64 != nil {
+		in, out := &in.UrlArm64, &out.UrlArm64
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackagesConfig.
+func (in *PackagesConfig) DeepCopy() *PackagesConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(PackagesConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) {
 	*out = *in

pkg/apis/kops/validation/validation.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/url"
 	"regexp"
 	"strings"
 
@@ -1148,7 +1149,50 @@ func validateContainerdConfig(config *kops.ContainerdConfig, fldPath *field.Path
 				fmt.Sprintf("unable to parse version string: %s", err.Error())))
 		}
 		if sv.LT(semver.MustParse("1.3.4")) {
-			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, "unsupported legacy version"))
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version,
+				"unsupported legacy version"))
+		}
+	}
+
+	if config.Packages != nil {
+		if config.Packages.UrlAmd64 != nil && config.Packages.HashAmd64 != nil {
+			u := fi.StringValue(config.Packages.UrlAmd64)
+			_, err := url.Parse(u)
+			if err != nil {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.UrlAmd64,
+					fmt.Sprintf("cannot parse package URL: %v", err)))
+			}
+			h := fi.StringValue(config.Packages.HashAmd64)
+			if len(h) > 64 {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64,
+					"Package hash must be 64 characters long"))
+			}
+		} else if config.Packages.UrlAmd64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.HashAmd64,
+				"Package hash must also be set"))
+		} else if config.Packages.HashAmd64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64,
+				"Package URL must also be set"))
+		}
+
+		if config.Packages.UrlArm64 != nil && config.Packages.HashArm64 != nil {
+			u := fi.StringValue(config.Packages.UrlArm64)
+			_, err := url.Parse(u)
+			if err != nil {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.UrlArm64,
+					fmt.Sprintf("cannot parse package URL: %v", err)))
+			}
+			h := fi.StringValue(config.Packages.HashArm64)
+			if len(h) > 64 {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64,
+					"Package hash must be 64 characters long"))
+			}
+		} else if config.Packages.UrlArm64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.HashArm64,
+				"Package hash must also be set"))
+		} else if config.Packages.HashArm64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64,
+				"Package URL must also be set"))
 		}
 	}
 
@@ -1168,7 +1212,50 @@ func validateDockerConfig(config *kops.DockerConfig, fldPath *field.Path) field.
 			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version,
 				"version is no longer available: https://www.docker.com/blog/changes-dockerproject-org-apt-yum-repositories"))
 		} else if sv.LT(semver.MustParse("17.3.0")) {
-			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version, "unsupported legacy version"))
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("version"), config.Version,
+				"unsupported legacy version"))
+		}
+	}
+
+	if config.Packages != nil {
+		if config.Packages.UrlAmd64 != nil && config.Packages.HashAmd64 != nil {
+			u := fi.StringValue(config.Packages.UrlAmd64)
+			_, err := url.Parse(u)
+			if err != nil {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.UrlAmd64,
+					fmt.Sprintf("unable parse package URL string: %v", err)))
+			}
+			h := fi.StringValue(config.Packages.HashAmd64)
+			if len(h) > 64 {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64,
+					"Package hash must be 64 characters long"))
+			}
+		} else if config.Packages.UrlAmd64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrl"), config.Packages.HashAmd64,
+				"Package hash must also be set"))
+		} else if config.Packages.HashAmd64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHash"), config.Packages.HashAmd64,
+				"Package URL must also be set"))
+		}
+
+		if config.Packages.UrlArm64 != nil && config.Packages.HashArm64 != nil {
+			u := fi.StringValue(config.Packages.UrlArm64)
+			_, err := url.Parse(u)
+			if err != nil {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.UrlArm64,
+					fmt.Sprintf("unable parse package URL string: %v", err)))
+			}
+			h := fi.StringValue(config.Packages.HashArm64)
+			if len(h) > 64 {
+				allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64,
+					"Package hash must be 64 characters long"))
+			}
+		} else if config.Packages.UrlArm64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageUrlArm64"), config.Packages.HashArm64,
+				"Package hash must also be set"))
+		} else if config.Packages.HashArm64 != nil {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("packageHashArm64"), config.Packages.HashArm64,
+				"Package URL must also be set"))
 		}
 	}
 

pkg/apis/kops/zz_generated.deepcopy.go

@@ -1104,6 +1104,11 @@ func (in *ContainerdConfig) DeepCopyInto(out *ContainerdConfig) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(PackagesConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.Root != nil {
 		in, out := &in.Root, &out.Root
 		*out = new(string)
@@ -1308,6 +1313,11 @@ func (in *DockerConfig) DeepCopyInto(out *DockerConfig) {
 		*out = new(int32)
 		**out = **in
 	}
+	if in.Packages != nil {
+		in, out := &in.Packages, &out.Packages
+		*out = new(PackagesConfig)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.RegistryMirrors != nil {
 		in, out := &in.RegistryMirrors, &out.RegistryMirrors
 		*out = make([]string, len(*in))
@@ -3971,6 +3981,42 @@ func (in *OpenstackRouter) DeepCopy() *OpenstackRouter {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PackagesConfig) DeepCopyInto(out *PackagesConfig) {
+	*out = *in
+	if in.HashAmd64 != nil {
+		in, out := &in.HashAmd64, &out.HashAmd64
+		*out = new(string)
+		**out = **in
+	}
+	if in.HashArm64 != nil {
+		in, out := &in.HashArm64, &out.HashArm64
+		*out = new(string)
+		**out = **in
+	}
+	if in.UrlAmd64 != nil {
+		in, out := &in.UrlAmd64, &out.UrlAmd64
+		*out = new(string)
+		**out = **in
+	}
+	if in.UrlArm64 != nil {
+		in, out := &in.UrlArm64, &out.UrlArm64
+		*out = new(string)
+		**out = **in
+	}
+	return
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PackagesConfig.
+func (in *PackagesConfig) DeepCopy() *PackagesConfig {
+	if in == nil {
+		return nil
+	}
+	out := new(PackagesConfig)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *RBACAuthorizationSpec) DeepCopyInto(out *RBACAuthorizationSpec) {
 	*out = *in

tests/integration/update_cluster/containerd-custom/cloudformation.json.extracted.yaml

+Resources.AWSEC2LaunchTemplatemasterustest1amasterscontainerdexamplecom.Properties.LaunchTemplateData.UserData: |
+  #!/bin/bash
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup
+  NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d
+  NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup
+  NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87
+
+  export AWS_REGION=us-test-1
+
+
+
+
+  function ensure-install-dir() {
+    INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
+    mkdir -p ${INSTALL_DIR}/bin
+    mkdir -p ${INSTALL_DIR}/conf
+    cd ${INSTALL_DIR}
+  }
+
+  # Retry a download until we get it. args: name, sha, url1, url2...
+  download-or-bust() {
+    local -r file="$1"
+    local -r hash="$2"
+    shift 2
+
+    urls=( $* )
+    while true; do
+      for url in "${urls[@]}"; do
+        commands=(
+          "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+          "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        )
+        for cmd in "${commands[@]}"; do
+          echo "Attempting download with: ${cmd} {url}"
+          if ! (${cmd} "${url}"); then
+            echo "== Download failed with ${cmd} =="
+            continue
+          fi
+          if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+            echo "== Hash validation of ${url} failed. Retrying. =="
+            rm -f "${file}"
+          else
+            if [[ -n "${hash}" ]]; then
+              echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+            else
+              echo "== Downloaded ${url} =="
+            fi
+            return
+          fi
+        done
+      done
+
+      echo "All downloads failed; sleeping before retrying"
+      sleep 60
+    done
+  }
+
+  validate-hash() {
+    local -r file="$1"
+    local -r expected="$2"
+    local actual
+
+    actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+    if [[ "${actual}" != "${expected}" ]]; then
+      echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+      return 1
+    fi
+  }
+
+  function split-commas() {
+    echo $1 | tr "," "\n"
+  }
+
+  function try-download-release() {
+    local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+    if [[ -n "${NODEUP_HASH:-}" ]]; then
+      local -r nodeup_hash="${NODEUP_HASH}"
+    else
+    # TODO: Remove?
+      echo "Downloading sha256 (not found in env)"
+      download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+      local -r nodeup_hash=$(cat nodeup.sha256)
+    fi
+
+    echo "Downloading nodeup (${nodeup_urls[@]})"
+    download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+    chmod +x nodeup
+  }
+
+  function download-release() {
+    case "$(uname -m)" in
+    x86_64*|i?86_64*|amd64*)
+      NODEUP_URL="${NODEUP_URL_AMD64}"
+      NODEUP_HASH="${NODEUP_HASH_AMD64}"
+      ;;
+    aarch64*|arm64*)
+      NODEUP_URL="${NODEUP_URL_ARM64}"
+      NODEUP_HASH="${NODEUP_HASH_ARM64}"
+      ;;
+    *)
+      echo "Unsupported host arch: $(uname -m)" >&2
+      exit 1
+      ;;
+    esac
+
+    # In case of failure checking integrity of release, retry.
+    cd ${INSTALL_DIR}/bin
+    until try-download-release; do
+      sleep 15
+      echo "Couldn't download release. Retrying..."
+    done
+
+    echo "Running nodeup"
+    # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+    ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+  }
+
+  ####################################################################################
+
+  /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+  echo "== nodeup node config starting =="
+  ensure-install-dir
+
+  cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+  cloudConfig: null
+  containerRuntime: containerd
+  containerd:
+    configOverride: |-
+      version = 2
+      [plugins]
+        [plugins."io.containerd.grpc.v1.cri"]
+          [plugins."io.containerd.grpc.v1.cri".cni]
+            conf_template = "/etc/containerd/cni-config.template"
+    logLevel: info
+    packages:
+      hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000"
+      urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+    version: 1.4.3
+  docker:
+    skipInstall: true
+  encryptionConfig: null
+  etcdClusters:
+    events:
+      version: 3.4.13
+    main:
+      version: 3.4.13
+  kubeAPIServer:
+    allowPrivileged: true
+    anonymousAuth: false
+    apiServerCount: 1
+    authorizationMode: AlwaysAllow
+    bindAddress: 0.0.0.0
+    cloudProvider: aws
+    enableAdmissionPlugins:
+    - NamespaceLifecycle
+    - LimitRanger
+    - ServiceAccount
+    - PersistentVolumeLabel
+    - DefaultStorageClass
+    - DefaultTolerationSeconds
+    - MutatingAdmissionWebhook
+    - ValidatingAdmissionWebhook
+    - NodeRestriction
+    - ResourceQuota
+    etcdServers:
+    - http://127.0.0.1:4001
+    etcdServersOverrides:
+    - /events#http://127.0.0.1:4002
+    image: k8s.gcr.io/kube-apiserver:v1.19.0
+    kubeletPreferredAddressTypes:
+    - InternalIP
+    - Hostname
+    - ExternalIP
+    logLevel: 2
+    requestheaderAllowedNames:
+    - aggregator
+    requestheaderExtraHeaderPrefixes:
+    - X-Remote-Extra-
+    requestheaderGroupHeaders:
+    - X-Remote-Group
+    requestheaderUsernameHeaders:
+    - X-Remote-User
+    securePort: 443
+    serviceClusterIPRange: 100.64.0.0/13
+    storageBackend: etcd3
+  kubeControllerManager:
+    allocateNodeCIDRs: true
+    attachDetachReconcileSyncPeriod: 1m0s
+    cloudProvider: aws
+    clusterCIDR: 100.96.0.0/11
+    clusterName: containerd.example.com
+    configureCloudRoutes: true
+    image: k8s.gcr.io/kube-controller-manager:v1.19.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+    useServiceAccountCredentials: true
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    hostnameOverride: '@aws'
+    image: k8s.gcr.io/kube-proxy:v1.19.0
+    logLevel: 2
+  kubeScheduler:
+    image: k8s.gcr.io/kube-scheduler:v1.19.0
+    leaderElection:
+      leaderElect: true
+    logLevel: 2
+  kubelet:
+    anonymousAuth: false
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+  masterKubelet:
+    anonymousAuth: false
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+    registerSchedulable: false
+
+  __EOF_CLUSTER_SPEC
+
+  cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+  {}
+
+  __EOF_IG_SPEC
+
+  cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+  Assets:
+    amd64:
+    - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet
+    - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+    - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+    arm64:
+    - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet
+    - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl
+    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+    - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
+  ClusterName: containerd.example.com
+  ConfigBase: memfs://clusters.example.com/containerd.example.com
+  InstanceGroupName: master-us-test-1a
+  InstanceGroupRole: Master
+  KubeletConfig:
+    anonymousAuth: false
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    nodeLabels:
+      kubernetes.io/role: master
+      node-role.kubernetes.io/master: ""
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+    registerSchedulable: false
+  channels:
+  - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml
+  etcdManifests:
+  - memfs://clusters.example.com/containerd.example.com/manifests/etcd/main.yaml
+  - memfs://clusters.example.com/containerd.example.com/manifests/etcd/events.yaml
+  protokubeImage:
+    amd64:
+      hash: 7b3c7f6adbda11b1ec740bd6b969c84f249b7eee818af95f2d321963088245a8
+      name: protokube:1.19.0-alpha.3
+      sources:
+      - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz
+      - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-amd64.tar.gz
+      - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-amd64.tar.gz
+    arm64:
+      hash: 69270ca9c1c950be65af40337adfccec0a728930fa3224bb0d2e88f181f39ead
+      name: protokube:1.19.0-alpha.3
+      sources:
+      - https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz
+      - https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/images-protokube-arm64.tar.gz
+      - https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/images/protokube-arm64.tar.gz
+  staticManifests:
+  - key: kube-apiserver-healthcheck
+    path: manifests/static/kube-apiserver-healthcheck.yaml
+
+  __EOF_KUBE_ENV
+
+  download-release
+  echo "== nodeup node config done =="
+Resources.AWSEC2LaunchTemplatenodescontainerdexamplecom.Properties.LaunchTemplateData.UserData: |
+  #!/bin/bash
+  set -o errexit
+  set -o nounset
+  set -o pipefail
+
+  NODEUP_URL_AMD64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/amd64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-amd64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/amd64/nodeup
+  NODEUP_HASH_AMD64=6980fda4fa37bbdc043738cf4ddac6388eb57f561895c69299c1b0ee263d465d
+  NODEUP_URL_ARM64=https://artifacts.k8s.io/binaries/kops/1.19.0-alpha.3/linux/arm64/nodeup,https://github.com/kubernetes/kops/releases/download/v1.19.0-alpha.3/nodeup-linux-arm64,https://kubeupv2.s3.amazonaws.com/kops/1.19.0-alpha.3/linux/arm64/nodeup
+  NODEUP_HASH_ARM64=dcc7f9f3c180ee76a511627e46da0ac69cdcb518cdf3be348e5ed046d491eb87
+
+  export AWS_REGION=us-test-1
+
+
+
+
+  function ensure-install-dir() {
+    INSTALL_DIR="/opt/kops"
+    # On ContainerOS, we install under /var/lib/toolbox; /opt is ro and noexec
+    if [[ -d /var/lib/toolbox ]]; then
+      INSTALL_DIR="/var/lib/toolbox/kops"
+    fi
+    mkdir -p ${INSTALL_DIR}/bin
+    mkdir -p ${INSTALL_DIR}/conf
+    cd ${INSTALL_DIR}
+  }
+
+  # Retry a download until we get it. args: name, sha, url1, url2...
+  download-or-bust() {
+    local -r file="$1"
+    local -r hash="$2"
+    shift 2
+
+    urls=( $* )
+    while true; do
+      for url in "${urls[@]}"; do
+        commands=(
+          "curl -f --ipv4 --compressed -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only --compression=auto -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+          "curl -f --ipv4 -Lo "${file}" --connect-timeout 20 --retry 6 --retry-delay 10"
+          "wget --inet4-only -O "${file}" --connect-timeout=20 --tries=6 --wait=10"
+        )
+        for cmd in "${commands[@]}"; do
+          echo "Attempting download with: ${cmd} {url}"
+          if ! (${cmd} "${url}"); then
+            echo "== Download failed with ${cmd} =="
+            continue
+          fi
+          if [[ -n "${hash}" ]] && ! validate-hash "${file}" "${hash}"; then
+            echo "== Hash validation of ${url} failed. Retrying. =="
+            rm -f "${file}"
+          else
+            if [[ -n "${hash}" ]]; then
+              echo "== Downloaded ${url} (SHA1 = ${hash}) =="
+            else
+              echo "== Downloaded ${url} =="
+            fi
+            return
+          fi
+        done
+      done
+
+      echo "All downloads failed; sleeping before retrying"
+      sleep 60
+    done
+  }
+
+  validate-hash() {
+    local -r file="$1"
+    local -r expected="$2"
+    local actual
+
+    actual=$(sha256sum ${file} | awk '{ print $1 }') || true
+    if [[ "${actual}" != "${expected}" ]]; then
+      echo "== ${file} corrupted, hash ${actual} doesn't match expected ${expected} =="
+      return 1
+    fi
+  }
+
+  function split-commas() {
+    echo $1 | tr "," "\n"
+  }
+
+  function try-download-release() {
+    local -r nodeup_urls=( $(split-commas "${NODEUP_URL}") )
+    if [[ -n "${NODEUP_HASH:-}" ]]; then
+      local -r nodeup_hash="${NODEUP_HASH}"
+    else
+    # TODO: Remove?
+      echo "Downloading sha256 (not found in env)"
+      download-or-bust nodeup.sha256 "" "${nodeup_urls[@]/%/.sha256}"
+      local -r nodeup_hash=$(cat nodeup.sha256)
+    fi
+
+    echo "Downloading nodeup (${nodeup_urls[@]})"
+    download-or-bust nodeup "${nodeup_hash}" "${nodeup_urls[@]}"
+
+    chmod +x nodeup
+  }
+
+  function download-release() {
+    case "$(uname -m)" in
+    x86_64*|i?86_64*|amd64*)
+      NODEUP_URL="${NODEUP_URL_AMD64}"
+      NODEUP_HASH="${NODEUP_HASH_AMD64}"
+      ;;
+    aarch64*|arm64*)
+      NODEUP_URL="${NODEUP_URL_ARM64}"
+      NODEUP_HASH="${NODEUP_HASH_ARM64}"
+      ;;
+    *)
+      echo "Unsupported host arch: $(uname -m)" >&2
+      exit 1
+      ;;
+    esac
+
+    # In case of failure checking integrity of release, retry.
+    cd ${INSTALL_DIR}/bin
+    until try-download-release; do
+      sleep 15
+      echo "Couldn't download release. Retrying..."
+    done
+
+    echo "Running nodeup"
+    # We can't run in the foreground because of https://github.com/docker/docker/issues/23793
+    ( cd ${INSTALL_DIR}/bin; ./nodeup --install-systemd-unit --conf=${INSTALL_DIR}/conf/kube_env.yaml --v=8  )
+  }
+
+  ####################################################################################
+
+  /bin/systemd-machine-id-setup || echo "failed to set up ensure machine-id configured"
+
+  echo "== nodeup node config starting =="
+  ensure-install-dir
+
+  cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
+  cloudConfig: null
+  containerRuntime: containerd
+  containerd:
+    configOverride: |-
+      version = 2
+      [plugins]
+        [plugins."io.containerd.grpc.v1.cri"]
+          [plugins."io.containerd.grpc.v1.cri".cni]
+            conf_template = "/etc/containerd/cni-config.template"
+    logLevel: info
+    packages:
+      hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000"
+      urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+    version: 1.4.3
+  docker:
+    skipInstall: true
+  kubeProxy:
+    clusterCIDR: 100.96.0.0/11
+    cpuRequest: 100m
+    hostnameOverride: '@aws'
+    image: k8s.gcr.io/kube-proxy:v1.19.0
+    logLevel: 2
+  kubelet:
+    anonymousAuth: false
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+
+  __EOF_CLUSTER_SPEC
+
+  cat > conf/ig_spec.yaml << '__EOF_IG_SPEC'
+  {}
+
+  __EOF_IG_SPEC
+
+  cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
+  Assets:
+    amd64:
+    - 3f03e5c160a8b658d30b34824a1c00abadbac96e62c4d01bf5c9271a2debc3ab@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubelet
+    - 79bb0d2f05487ff533999a639c075043c70a0a1ba25c1629eb1eef6ebe3ba70f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/amd64/kubectl
+    - 977824932d5667c7a37aa6a3cbba40100a6873e7bd97e83e8be837e3e7afd0a8@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz
+    - 0000000000000000000000000000000000000000000000000000000000000000@https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+    arm64:
+    - d8fa5a9739ecc387dfcc55afa91ac6f4b0ccd01f1423c423dbd312d787bbb6bf@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubelet
+    - d4adf1b6b97252025cb2f7febf55daa3f42dc305822e3da133f77fd33071ec2f@https://storage.googleapis.com/kubernetes-release/release/v1.19.0/bin/linux/arm64/kubectl
+    - ae13d7b5c05bd180ea9b5b68f44bdaa7bfb41034a2ef1d68fd8e1259797d642f@https://storage.googleapis.com/k8s-artifacts-cni/release/v0.8.7/cni-plugins-linux-arm64-v0.8.7.tgz
+    - 6e3f80e8451ecbe7b3559247721c3e226be6b228acaadee7e13683f80c20e81c@https://download.docker.com/linux/static/stable/aarch64/docker-20.10.0.tgz
+  ClusterName: containerd.example.com
+  ConfigBase: memfs://clusters.example.com/containerd.example.com
+  InstanceGroupName: nodes
+  InstanceGroupRole: Node
+  KubeletConfig:
+    anonymousAuth: false
+    cgroupRoot: /
+    cloudProvider: aws
+    clusterDNS: 100.64.0.10
+    clusterDomain: cluster.local
+    enableDebuggingHandlers: true
+    evictionHard: memory.available<100Mi,nodefs.available<10%,nodefs.inodesFree<5%,imagefs.available<10%,imagefs.inodesFree<5%
+    hostnameOverride: '@aws'
+    kubeconfigPath: /var/lib/kubelet/kubeconfig
+    logLevel: 2
+    nodeLabels:
+      kubernetes.io/role: node
+      node-role.kubernetes.io/node: ""
+    nonMasqueradeCIDR: 100.64.0.0/10
+    podManifestPath: /etc/kubernetes/manifests
+  channels:
+  - memfs://clusters.example.com/containerd.example.com/addons/bootstrap-channel.yaml
+
+  __EOF_KUBE_ENV
+
+  download-release
+  echo "== nodeup node config done =="

tests/integration/update_cluster/containerd-custom/in-v1alpha2.yaml

+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: containerd.example.com
+spec:
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/containerd.example.com
+  containerRuntime: containerd
+  containerd:
+    packages:
+      urlAmd64: https://github.com/containerd/containerd/releases/download/v1.3.9/cri-containerd-cni-1.3.9-linux-amd64.tar.gz
+      hashAmd64: "0000000000000000000000000000000000000000000000000000000000000000"
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: us-test-1a
+    name: events
+  iam: {}
+  kubelet:
+    anonymousAuth: false
+  kubernetesVersion: v1.19.0
+  masterInternalName: api.internal.containerd.example.com
+  masterPublicName: api.containerd.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: nodes
+  labels:
+    kops.k8s.io/cluster: containerd.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.14-debian-stretch-amd64-hvm-ebs-2019-08-16
+  machineType: t2.medium
+  maxSize: 2
+  minSize: 2
+  role: Node
+  subnets:
+  - us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: master-us-test-1a
+  labels:
+    kops.k8s.io/cluster: containerd.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.14-debian-stretch-amd64-hvm-ebs-2019-08-16
+  machineType: m3.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a
+
+

tests/integration/update_cluster/containerd/id_rsa.pub

+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQCtWu40XQo8dczLsCq0OWV+hxm9uV3WxeH9Kgh4sMzQxNtoU1pvW0XdjpkBesRKGoolfWeCLXWxpyQb1IaiMkKoz7MdhQ/6UKjMjP66aFWWp3pwD0uj0HuJ7tq4gKHKRYGTaZIRWpzUiANBrjugVgA+Sd7E/mYwc/DMXkIyRZbvhQ==