ceph: mgr modules requests should timeout

Requests to enable mgr modules or call mgr modules should
timeout rather than hang. For example, if creating a self
signed cert request hangs, the mgr should continue with other
actions in the orchestration
Signed-off-by: travisn <tnielsen@redhat.com>

pkg/operator/ceph/cluster/mgr/dashboard.go

@@ -231,7 +231,7 @@ func (c *Cluster) setLoginCredentials(password string) error {
 		args := []string{"dashboard", "set-login-credentials", dashboardUsername, password}
 		cmd := client.NewCephCommand(c.context, c.Namespace, args)
 		cmd.Debug = true
-		return cmd.Run()
+		return cmd.RunWithTimeout(client.CmdExecuteTimeout)
 	}, c.exitCode, 5, invalidArgErrorCode, dashboardInitWaitTime)
 	if err != nil {
 		return fmt.Errorf("failed to set login creds on mgr. %+v", err)

pkg/operator/ceph/cluster/mgr/orchestrator.go

@@ -49,11 +49,10 @@ func (c *Cluster) configureOrchestratorModules() error {
 	// retry a few times in the case that the mgr module is not ready to accept commands
 	_, err := client.ExecuteCephCommandWithRetry(func() ([]byte, error) {
 		args := []string{"orchestrator", "set", "backend", "rook"}
-		return client.NewCephCommand(c.context, c.Namespace, args).Run()
+		return client.NewCephCommand(c.context, c.Namespace, args).RunWithTimeout(client.CmdExecuteTimeout)
 	}, c.exitCode, 5, invalidArgErrorCode, orchestratorInitWaitTime)
 	if err != nil {
 		return fmt.Errorf("failed to set rook as the orchestrator backend. %+v", err)
-
 	}
 
 	return nil

pkg/operator/ceph/cluster/mgr/orchestrator_test.go

@@ -18,6 +18,7 @@ package mgr
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/rook/rook/pkg/clusterd"
 	cephconfig "github.com/rook/rook/pkg/daemon/ceph/config"
@@ -32,6 +33,7 @@ func TestOrchestratorModules(t *testing.T) {
 	orchestratorModuleEnabled := false
 	rookModuleEnabled := false
 	rookBackendSet := false
+	backendErrorCount := 0
 	executor.MockExecuteCommandWithOutputFile = func(debug bool, actionName, command, outputFile string, args ...string) (string, error) {
 		logger.Infof("Command: %s %v", command, args)
 		if args[0] == "mgr" && args[1] == "module" && args[2] == "enable" {
@@ -44,7 +46,15 @@ func TestOrchestratorModules(t *testing.T) {
 				return "", nil
 			}
 		}
+		return "", fmt.Errorf("unexpected ceph command '%v'", args)
+	}
+	executor.MockExecuteCommandWithOutputFileTimeout = func(debug bool, timeout time.Duration, actionName, command, outputFile string, args ...string) (string, error) {
+		logger.Infof("Command: %s %v", command, args)
 		if args[0] == "orchestrator" && args[1] == "set" && args[2] == "backend" && args[3] == "rook" {
+			if backendErrorCount < 5 {
+				backendErrorCount++
+				return "", fmt.Errorf("test simulation failure")
+			}
 			rookBackendSet = true
 			return "", nil
 		}
@@ -56,6 +66,10 @@ func TestOrchestratorModules(t *testing.T) {
 	}
 
 	c := &Cluster{clusterInfo: clusterInfo, context: context}
+	c.exitCode = func(err error) (int, bool) {
+		return invalidArgErrorCode, true
+	}
+	orchestratorInitWaitTime = 0
 
 	// the modules are skipped on mimic
 	c.clusterInfo.CephVersion = cephver.Mimic
@@ -66,9 +80,17 @@ func TestOrchestratorModules(t *testing.T) {
 	assert.False(t, rookBackendSet)
 
 	// the modules are configured on nautilus
+	// the rook module will fail to be set
 	c.clusterInfo.CephVersion = cephver.Nautilus
 	err = c.configureOrchestratorModules()
-	assert.Nil(t, err)
+	assert.Error(t, err)
+	assert.True(t, orchestratorModuleEnabled)
+	assert.True(t, rookModuleEnabled)
+	assert.False(t, rookBackendSet)
+
+	// the rook module will succeed
+	err = c.configureOrchestratorModules()
+	assert.NoError(t, err)
 	assert.True(t, orchestratorModuleEnabled)
 	assert.True(t, rookModuleEnabled)
 	assert.True(t, rookBackendSet)