Make it easier to run webhook tests locally (#476)

* make it easy to run webhook tests locally

* modify tests so they run locally

* follow the logs

* add instructions

* make it easy to run webhook tests locally

* modify tests so they run locally

* follow the logs

* add instructions

* use universal date command

* fix sed command for portability

* fix date command

* make entire image configurable

* fix instructions

docs/contributing.md

@@ -32,12 +32,26 @@ We label issues with the ["good first issue" tag](https://github.com/FairwindsOp
 
 The following commands are all required to pass as part of Polaris testing:
 
-```
+```bash
 go list ./... | grep -v vendor | xargs golint -set_exit_status
 go list ./... | grep -v vendor | xargs go vet
 go test ./pkg/... -v -coverprofile cover.out
 ```
 
+### Webhook tests
+```bash
+kind create cluster --wait=90s --image kindest/node:v1.15.11 --name polaris-test
+docker build -t quay.io/fairwinds/polaris:debug . # or use your own registry
+docker push quay.io/fairwinds/polaris:debug
+helm repo add jetstack https://charts.jetstack.io
+kubectl create ns cert-manager
+helm install cert-manager jetstack/cert-manager --namespace cert-manager --version 0.16.1 --set "installCRDs=true" --wait
+POLARIS_IMAGE=quay.io/fairwinds/polaris:debug ./test/webhook_test.sh
+```
+to avoid the final cleanup for debugging purposes, you can run
+```bash
+SKIP_FINAL_CLEANUP=true IMAGE_TAG=debug ./test/webhook_test.sh
+```
 ## Creating a New Issue
 
 If you've encountered an issue that is not already reported, please create a [new issue](https://github.com/FairwindsOps/polaris/issues), choose `Bug Report`, `Feature Request` or `Misc.` and follow the instructions in the template. 

test/webhook_cases/failing_test.daemonset.v1beta2.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
   name: fluentd-elasticsearch
-  namespace: kube-system
   labels:
     k8s-app: fluentd-logging
 spec:

test/webhook_cases/failing_test.daemonset.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: fluentd-elasticsearch
-  namespace: kube-system
   labels:
     k8s-app: fluentd-logging
 spec:
@@ -45,4 +44,4 @@ spec:
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
-        
\ No newline at end of file
+        

test/webhook_cases/passing_test.daemonset.v1beta2.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1beta2
 kind: DaemonSet
 metadata:
   name: fluentd-elasticsearch
-  namespace: kube-system
   labels:
     k8s-app: fluentd-logging
 spec:

test/webhook_cases/passing_test.daemonset.yaml

@@ -2,7 +2,6 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: fluentd-elasticsearch
-  namespace: kube-system
   labels:
     k8s-app: fluentd-logging
 spec:
@@ -45,4 +44,4 @@ spec:
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
-        
\ No newline at end of file
+        

test/webhook_test.sh

 #!/bin/bash
 set -e
 
-#sed is replacing the polaris version with this commit sha so we are testing exactly this verison.
-sed -r "s|'(quay.io/fairwinds/polaris:).+'|'\1${CIRCLE_SHA1}'|" ./deploy/webhook.yaml > ./deploy/webhook-test.yaml
-
 # Testing to ensure that the webhook starts up, allows a correct deployment to pass,
-# and prevents a incorrectly formatted deployment. 
+# and prevents a incorrectly formatted deployment.
+
+function get_timeout() {
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    date -v+4M +%s
+  else
+    date -d "+4 minutes" +%s
+  fi
+}
+
 function check_webhook_is_ready() {
     # Get the epoch time in one minute from now
     local timeout_epoch
 
     # Reset another 4 minutes to wait for webhook
-    timeout_epoch=$(date -d "+4 minutes" +%s)
+    timeout_epoch=$(get_timeout)
 
     # loop until this fails (desired condition is we cannot apply this yaml doc, which means the webhook is working
     echo "Waiting for webhook to be ready"
@@ -34,19 +40,23 @@ function check_timeout() {
         clean_up
         exit 1
     fi
-
 }
 
 # Clean up all your stuff
 function clean_up() {
+    echo -e "\n\nCleaning up (you may see some errors)...\n\n"
+    kubectl delete ns scale-test || true
+    kubectl delete ns polaris || true
+    kubectl delete ns tests || true
     # Clean up files you've installed (helps with local testing)
     for filename in test/webhook_cases/*.yaml; do
         # || true to avoid issues when we cannot delete
-        kubectl delete -f $filename &>/dev/null ||true
+        kubectl delete -f $filename ||true
     done
     # Uninstall webhook and webhook config
-    kubectl delete validatingwebhookconfigurations polaris-webhook --wait=false &>/dev/null
-    kubectl -n polaris delete deploy -l app=polaris --wait=false &>/dev/null
+    kubectl delete validatingwebhookconfigurations polaris-webhook --wait=false
+    kubectl -n polaris delete deploy -l app=polaris --wait=false
+    echo -e "\n\nDone cleaning up\n\n"
 }
 
 function grab_logs() {
@@ -56,39 +66,56 @@ function grab_logs() {
     kubectl -n polaris logs -l app=polaris
 }
 
-# Install a bad deployment
+#sed is replacing the polaris version with this commit sha so we are testing exactly this verison.
+if [ -z "${POLARIS_IMAGE}" ]; then
+  POLARIS_IMAGE="quay.io/fairwinds/polaris:$CIRCLE_SHA1"
+fi
+echo "using image $POLARIS_IMAGE"
+sed -E "s|'(quay.io/fairwinds/polaris:).+'|'${POLARIS_IMAGE}'|" ./deploy/webhook.yaml > ./deploy/webhook-test.yaml
+
+clean_up || true
+
+echo -e "Setting up..."
 kubectl create ns scale-test
-kubectl apply -n scale-test -f ./test/webhook_cases/failing_test.deployment.yaml
+kubectl create ns polaris
+kubectl create ns tests
 
-# Install the webhook 
-kubectl apply -f ./deploy/webhook-test.yaml &> /dev/null
+# Install a bad deployment
+kubectl apply -n scale-test -f ./test/webhook_cases/failing_test.deployment.yaml
 
+# Install the webhook
+kubectl apply -n polaris -f ./deploy/webhook-test.yaml
 
 # wait for the webhook to come online
 check_webhook_is_ready
-sleep 30
+sleep 5
+
+kubectl logs -n polaris $(kubectl get po -oname -n polaris | grep webhook) --follow &
 
 # Webhook started, setting all tests as passed initially.
 ALL_TESTS_PASSED=1
 
 # Run tests against correctly configured objects
 for filename in test/webhook_cases/passing_test.*.yaml; do
+    echo -e "\n\n"
     echo $filename
-    if ! kubectl apply -f $filename &> /dev/null; then
+    if ! kubectl apply -n tests -f $filename; then
         ALL_TESTS_PASSED=0
-        echo "Test Failed: Polaris prevented a deployment with no configuration issues." 
-        kubectl logs -n polaris $(kubectl get po -oname -n polaris | grep webhook)
+        echo -e "****Test Failed: Polaris prevented a deployment with no configuration issues****"
     fi
+    kubectl delete -n tests -f $filename || true
 done
 
 # Run tests against incorrectly configured objects
 for filename in test/webhook_cases/failing_test.*.yaml; do
+    echo -e "\n\n"
     echo $filename
-    if kubectl apply -f $filename &> /dev/null; then
+    if kubectl apply -n tests -f $filename; then
         ALL_TESTS_PASSED=0
-        echo "Test Failed: Polaris should have prevented this deployment due to configuration issues."
+        echo -e "****Test Failed: Polaris should have prevented this deployment due to configuration issues.****"
         kubectl logs -n polaris $(kubectl get po -oname -n polaris | grep webhook)
     fi
+    kubectl delete -n tests -f $filename || true
 done
 
 kubectl -n scale-test scale deployment nginx-deployment --replicas=2
@@ -100,7 +127,9 @@ if [ $pod_count != 2 ]; then
   echo "Existing deployment was unable to scale after webhook installed: found $pod_count pods"
 fi
 
-clean_up
+if [ -z $SKIP_FINAL_CLEANUP ]; then
+  clean_up
+fi
 
 #Verify that all the tests passed.
 if [ $ALL_TESTS_PASSED -eq 1 ]; then