allow executing tests specific to a storage provider

Since most PRs are focused on a single storage provider,
the tests for other storage providers do not need to be
run. The CI will allow skipping tests for other storage
providers by specifying keywords in the PR description.
Similar to [skip ci], you can now specify [test storage-provider].
For example, [test ceph] will only trigger the test
suites for ceph and none of the other storage providers.
Signed-off-by: travisn <tnielsen@redhat.com>

INSTALL.md

@@ -165,3 +165,21 @@ This option should **not** be used if there are any changes to `.go` files, CI s
 
 In case there are known CI failures that are blocking a merge, project maintainers can add this flag
 to unblock the CI and merge the PR. This should be used with extreme care so regressions are not introduced.
+
+### [test storage-provider]
+
+Jenkins will only run the tests for an individual storage provider if specified. There is a keyword for each
+storage provider that has integration tests:
+- `[test cassandra]`
+- `[test ceph]`
+- `[test cockroachdb]`
+- `[test edgefs]`
+- `[test nfs]`
+
+Since the majority of effort for a PR generally focuses on a single storage provider it will save time
+in the CI if you only trigger the tests for your provider. If this key phrase is not found in the PR,
+all tests will be executed.
+
+These options should **not** be used if there are any changes to code which is shared with other
+storage providers. If there is any risk of affecting another storage provider, all tests should
+be executed in the CI.

Jenkinsfile

@@ -27,12 +27,22 @@ pipeline {
                     if (body.contains("[skip tests]")) {
                          env.shouldTest = "false"
                     }
-                    if (body.contains("[smoke only]")) {
-                          env.smokeOnly = "true"
-                    }
                     if (body.contains("[all logs]")) {
                           env.getLogs = "all"
                     }
+                    // extract which specific storage provider to test
+                    if (body.contains("[test cassandra]")) {
+                        env.testProvider = "cassandra"
+                    } else if (body.contains("[test ceph]")) {
+                        env.testProvider = "ceph"
+                    } else if (body.contains("[test cockroachdb]")) {
+                        env.testProvider = "cockroachdb"
+                    } else if (body.contains("[test edgefs]")) {
+                        env.testProvider = "edgefs"
+                    } else if (body.contains("[test nfs]")) {
+                        env.testProvider = "nfs"
+                    }
+                    echo ("integration test provider: ${env.testProvider}")
                 }
             }
         }
@@ -159,24 +169,14 @@ def RunIntegrationTest(k, v) {
                           export KUBECONFIG=$HOME/admin.conf
                           tests/scripts/helm.sh up'''
                     try{
-                        if ("${env.smokeOnly}" == "true") {
-                            echo "Running Smoke Tests"
-                            sh '''#!/bin/bash
-                                  set -o pipefail
-                                  export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \
-                                      KUBECONFIG=$HOME/admin.conf
-                                  kubectl config view
-                                  _output/tests/linux_amd64/integration -test.v -test.timeout 1800s -test.run SmokeSuite --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log'''
-                        }
-                        else {
                         echo "Running full regression"
                         sh '''#!/bin/bash
                               set -o pipefail
                               export PATH="/tmp/rook-tests-scripts-helm/linux-amd64:$PATH" \
-                                  KUBECONFIG=$HOME/admin.conf
+                                  KUBECONFIG=$HOME/admin.conf \
+                                  STORAGE_PROVIDER_TESTS='''+"${env.testProvider}"+'''
                               kubectl config view
                               _output/tests/linux_amd64/integration -test.v -test.timeout 7200s --host_type '''+"${k}"+''' --logs '''+"${env.getLogs}"+''' --helm /tmp/rook-tests-scripts-helm/linux-amd64/helm 2>&1 | tee _output/tests/integrationTests.log'''
-                         }
                     }
                     finally{
                         sh "journalctl -u kubelet > _output/tests/kubelet_${v}.log"

PendingReleaseNotes.md

@@ -9,6 +9,7 @@ an example usage
 - Added K8s 1.15 to the test matrix and removed K8s 1.10 from the test matrix.
 - OwnerReferences are created with the fully qualified `apiVersion` such that the references will work properly on OpenShift.
 - Linear disk device can now be used for Ceph OSDs.
+- The integration tests can be triggered for specific storage providers rather than always running all tests. See the [dev guide](INSTALL.md#test-storage-provider) for more details.
 
 ### Ceph
 

tests/framework/installer/installer.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 	"testing"
 
 	"github.com/coreos/pkg/capnslog"
@@ -33,6 +34,12 @@ const (
 	VersionMaster = "master"
 	// Version tag for Rook v0.9
 	Version1_0 = "v1.0.1"
+	// test suite names
+	CassandraTestSuite   = "cassandra"
+	CephTestSuite        = "ceph"
+	CockroachDBTestSuite = "cockroachdb"
+	EdgeFSTestSuite      = "edgefs"
+	NFSTestSuite         = "nfs"
 )
 
 var (
@@ -53,6 +60,21 @@ type TestSuite interface {
 	Teardown()
 }
 
+func SkipTestSuite(name string) bool {
+	testsToRun := os.Getenv("STORAGE_PROVIDER_TESTS")
+	if testsToRun == "" {
+		// run all test suites
+		return false
+	}
+	if strings.EqualFold(testsToRun, name) {
+		// this suite was requested
+		return false
+	}
+
+	logger.Infof("skipping test suite since only %s should be tested rather than %s", testsToRun, name)
+	return true
+}
+
 func init() {
 	// this default will only work if running kubernetes on the local machine
 	baseTestDir, _ = os.Getwd()

tests/integration/block_create_test.go

@@ -33,6 +33,10 @@ import (
 // Test K8s Block Image Creation Scenarios. These tests work when platform is set to Kubernetes
 
 func TestBlockCreateSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(BlockCreateSuite)
 	defer func(s *BlockCreateSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/cassandra_test.go

@@ -52,6 +52,10 @@ type CassandraSuite struct {
 
 // TestCassandraSuite initiates the CassandraSuite
 func TestCassandraSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CassandraTestSuite) {
+		t.Skip()
+	}
+
 	s := new(CassandraSuite)
 	defer func(s *CassandraSuite) {
 		r := recover()

tests/integration/cockroachdb_test.go

@@ -40,6 +40,10 @@ import (
 //   - 25% cache, 25% maxSQLMemory
 // ************************************************
 func TestCockroachDBSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CockroachDBTestSuite) {
+		t.Skip()
+	}
+
 	s := new(CockroachDBSuite)
 	defer func(s *CockroachDBSuite) {
 		HandlePanics(recover(), s, s.T)

tests/integration/edgefs_test.go

@@ -45,6 +45,10 @@ type EdgefsSuite struct {
 }
 
 func TestEdgefsSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.EdgeFSTestSuite) {
+		t.Skip()
+	}
+
 	s := new(EdgefsSuite)
 	defer func(s *EdgefsSuite) {
 		HandlePanics(recover(), s, s.T)

tests/integration/helm_test.go

@@ -46,6 +46,10 @@ var (
 // - Create the object store via the CRD
 // ***************************************************
 func TestHelmSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(HelmSuite)
 	defer func(s *HelmSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/multi_cluster_test.go

@@ -45,6 +45,10 @@ import (
 // - Create the object store via the CRD
 // *************************************************************
 func TestMultiClusterDeploySuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(MultiClusterDeploySuite)
 	defer func(s *MultiClusterDeploySuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/nfs_test.go

@@ -38,6 +38,10 @@ import (
 //   - Mount a NFS export and write data to it and verify
 // *******************************************************
 func TestNfsSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.NFSTestSuite) {
+		t.Skip()
+	}
+
 	s := new(NfsSuite)
 	defer func(s *NfsSuite) {
 		HandlePanics(recover(), s, s.T)

tests/integration/smoke_test.go

@@ -50,6 +50,10 @@ import (
 // - PUT/GET objects
 // ************************************************
 func TestSmokeSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(SmokeSuite)
 	defer func(s *SmokeSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/upgrade_test.go

@@ -40,6 +40,10 @@ import (
 // - One mon in the cluster
 // ************************************************
 func TestUpgradeSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(UpgradeSuite)
 	defer func(s *UpgradeSuite) {
 		HandlePanics(recover(), s.op, s.T)

tests/integration/zblock_mount_unmount_test.go

@@ -60,6 +60,10 @@ import (
 // The error is "the server does not allow this method on the requested resource (post cephclusters.ceph.rook.io)".
 // Everything appears to have been cleaned up successfully in this test, so it is still unclear what is causing the issue between tests.
 func TestBlockMountUnMountSuite(t *testing.T) {
+	if installer.SkipTestSuite(installer.CephTestSuite) {
+		t.Skip()
+	}
+
 	s := new(BlockMountUnMountSuite)
 	defer func(s *BlockMountUnMountSuite) {
 		HandlePanics(recover(), s.op, s.T)