Add fix command to mutate and update IaC (#746)

* added fix command

* update fix command to walk through the folder to find all files

* added ability to add comment

* fix comment prefix

* trim whitespaces to the line

* refactor update mutated file

* remove filepath as is not needed anymore

* remove filepath as is not needed anymore

* remove timestamp and status if creation is null

* added comments and fix tests

* remove hardcoded mutation in config

* revert comment deletion

* separate mutated to success files

* read multiple resources in a file and update both

* Remove mutation in config.yaml

checks/cpuLimitsMissing.yaml

@@ -26,6 +26,12 @@ schema:
               not:
                 const: ''
 mutations:
+  - op: add
+    path: /resources/limits
+    value: {}
   - op: add
     path: /resources/limits/cpu
-    value: 100m
\ No newline at end of file
+    value: 100m
+comments:
+  - find: "cpu: 100m"
+    comment: "TODO: Set this to the amount of CPU you want to reserve for your workload"
\ No newline at end of file

checks/cpuRequestsMissing.yaml

@@ -26,6 +26,12 @@ schema:
               not:
                 const: ''
 mutations:
+  - op: add
+    path: /resources/requests
+    value: {}
   - op: add
     path: /resources/requests/cpu
-    value: 100m
\ No newline at end of file
+    value: 100m
+comments:
+  - find: "cpu: 100m"
+    comment: "TODO: Set this to the amount of CPU you want to reserve for your workload"
\ No newline at end of file

checks/livenessProbeMissing.yaml

@@ -22,4 +22,7 @@ schema:
 mutations:
   - op: add
     path: /livenessProbe
-    value: {"exec": { "command": [ "cat", "/tmp/healthy" ] }, "initialDelaySeconds": 5, "periodSeconds": 5 }
\ No newline at end of file
+    value: {"exec": { "command": [ "cat", "/tmp/healthy" ] }, "initialDelaySeconds": 5, "periodSeconds": 5 }
+comments:
+  - find: "livenessProbe:"
+    comment: "TODO: Change livenessProbe setting to reflect your health endpoints"
\ No newline at end of file

checks/memoryLimitsMissing.yaml

@@ -31,4 +31,7 @@ mutations:
     value: {}
   - op: add
     path: /resources/limits/memory
-    value: "512Mi"
\ No newline at end of file
+    value: "512Mi"
+comments:
+  - find: "memory: 512Mi"
+    comment: "TODO: Set this to the amount of Memory you want to reserve for your workload"
\ No newline at end of file

checks/memoryRequestsMissing.yaml

@@ -31,4 +31,7 @@ mutations:
     value: {}
   - op: add
     path: /resources/requests/memory
-    value: "512Mi"
\ No newline at end of file
+    value: "512Mi"
+comments:
+  - find: "memory: 512Mi"
+    comment: "TODO: Set this to the amount of Memory you want to reserve for your workload"
\ No newline at end of file

checks/readinessProbeMissing.yaml

@@ -22,4 +22,7 @@ schema:
 mutations:
   - op: add
     path: /readinessProbe
-    value: {"exec": { "command": [ "cat", "/tmp/healthy" ] }, "initialDelaySeconds": 5, "periodSeconds": 5 }
\ No newline at end of file
+    value: {"exec": { "command": [ "cat", "/tmp/healthy" ] }, "initialDelaySeconds": 5, "periodSeconds": 5 }
+comments:
+  - find: "readinessProbe:"
+    comment: "TODO: Change livenessProbe setting to reflect your health endpoints"
\ No newline at end of file

cmd/polaris/fix.go

+// Copyright 2020 FairwindsOps Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/fairwindsops/polaris/pkg/kube"
+	"github.com/fairwindsops/polaris/pkg/mutation"
+	"github.com/fairwindsops/polaris/pkg/validator"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	yamlV3 "gopkg.in/yaml.v3"
+	"sigs.k8s.io/yaml"
+)
+
+var filesPath string
+
+func init() {
+	rootCmd.AddCommand(fixCommand)
+	fixCommand.PersistentFlags().StringVar(&filesPath, "files-path", "", "mutate and fix one or more YAML files in a specified folder")
+}
+
+var fixCommand = &cobra.Command{
+	Use:   "fix",
+	Short: "Fix Infrastructure as code files.",
+	Long:  `Fix Infrastructure as code files.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		logrus.Debug("Setting up controller manager")
+
+		if filesPath == "" {
+			logrus.Error("Please specify a sub-command.")
+			err := cmd.Help()
+			panic(err)
+		}
+
+		baseDir := filesPath + "/"
+		yamlFiles, err := getYamlFiles(baseDir)
+		if err != nil {
+			panic(err)
+		}
+		var contentStr string
+		isFirstResource := true
+		for _, fullFilePath := range yamlFiles {
+
+			yamlFile, err := ioutil.ReadFile(fullFilePath)
+			if err != nil {
+				panic(err)
+			}
+
+			dec := yamlV3.NewDecoder(bytes.NewReader(yamlFile))
+
+			for {
+				data := map[string]interface{}{}
+				err := dec.Decode(&data)
+				// check it was parsed
+				if data == nil {
+					continue
+				}
+				// break the loop in case of EOF
+				if errors.Is(err, io.EOF) {
+					break
+				}
+				if err != nil {
+					panic(err)
+				}
+				yamlContent, err := yamlV3.Marshal(data)
+				if err != nil {
+					panic(err)
+				}
+				kubeResources := kube.CreateResourceProviderFromYaml(string(yamlContent))
+				results, err := validator.ApplyAllSchemaChecksToResourceProvider(&config, kubeResources)
+				if err != nil {
+					panic(err)
+				}
+				comments, allMutations := mutation.GetMutationsAndCommentsFromResults(results)
+				updatedYamlContent := string(yamlContent)
+				if len(allMutations) > 0 {
+					for _, resources := range kubeResources.Resources {
+						key := fmt.Sprintf("%s/%s/%s", resources[0].Kind, resources[0].Resource.GetName(), resources[0].Resource.GetNamespace())
+						mutations := allMutations[key]
+						mutated, err := mutation.ApplyAllSchemaMutations(&config, kubeResources, resources[0], mutations)
+						if err != nil {
+							panic(err)
+						}
+						mutatedYamlContent, err := yaml.JSONToYAML(mutated.OriginalObjectJSON)
+						if err != nil {
+							panic(err)
+						}
+						updatedYamlContent = mutation.UpdateMutatedContentWithComments(string(mutatedYamlContent), comments)
+					}
+				}
+				if isFirstResource {
+					contentStr = updatedYamlContent
+					isFirstResource = false
+				} else {
+					contentStr += "\n"
+					contentStr += "---"
+					contentStr += "\n"
+					contentStr += updatedYamlContent
+				}
+			}
+
+			if contentStr != "" {
+				err = ioutil.WriteFile(fullFilePath, []byte(contentStr), 0644)
+				if err != nil {
+					logrus.Errorf("Error writing output to file: %v", err)
+					os.Exit(1)
+				}
+			}
+		}
+
+	},
+}
+
+func getYamlFiles(rootpath string) ([]string, error) {
+	var list []string
+	err := filepath.Walk(rootpath, func(path string, info os.FileInfo, err error) error {
+		if info.IsDir() {
+			return nil
+		}
+		if filepath.Ext(path) == ".yaml" {
+			list = append(list, path)
+		}
+		return nil
+	})
+	return list, err
+}

examples/config.yaml

@@ -27,8 +27,7 @@ checks:
   hostNetworkSet: danger
   hostPortSet: warning
   tlsSettingsMissing: warning
-
-
+ 
 exemptions:
   - namespace: kube-system
     controllerNames:

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/thoas/go-funk v0.9.1
 	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	k8s.io/api v0.22.2
 	k8s.io/apimachinery v0.22.2

pkg/config/schema.go

@@ -49,6 +49,12 @@ var HandledTargets = []TargetKind{
 	TargetPod,
 }
 
+// MutationComment is the comments added to a mutated file
+type MutationComment struct {
+	Find    string `yaml:"find" json:"find"`
+	Comment string `yaml:"comment" json:"comment"`
+}
+
 // SchemaCheck is a Polaris check that runs using JSON Schema
 type SchemaCheck struct {
 	ID                      string                            `yaml:"id" json:"id"`
@@ -66,6 +72,7 @@ type SchemaCheck struct {
 	AdditionalSchemaStrings map[string]string                 `yaml:"additionalSchemaStrings" json:"additionalSchemaStrings"`
 	AdditionalValidators    map[string]jsonschema.RootSchema  `yaml:"-" json:"-"`
 	Mutations               []map[string]interface{}          `yaml:"mutations" json:"mutations"`
+	Comments                []MutationComment                 `yaml:"comments" json:"comments"`
 }
 
 type resourceMinimum string

pkg/kube/resource.go

@@ -40,6 +40,10 @@ type GenericResource struct {
 
 // NewGenericResourceFromUnstructured creates a workload from an unstructured.Unstructured
 func NewGenericResourceFromUnstructured(unst unstructured.Unstructured, podSpecMap interface{}) (GenericResource, error) {
+	if unst.GetCreationTimestamp().Time.IsZero() {
+		unstructured.RemoveNestedField(unst.Object, "metadata", "creationTimestamp")
+		unstructured.RemoveNestedField(unst.Object, "status")
+	}
 	workload := GenericResource{
 		Kind:     unst.GetKind(),
 		Resource: unst,

pkg/kube/resources.go

@@ -225,6 +225,13 @@ func CreateResourceProviderFromPath(directory string) (*ResourceProvider, error)
 	return &resources, nil
 }
 
+// CreateResourceProviderFromYaml returns a new ResourceProvider using the yaml
+func CreateResourceProviderFromYaml(yamlContent string) *ResourceProvider {
+	resources := newResourceProvider("unknown", "Content", "unknown")
+	resources.addResourcesFromYaml(string(yamlContent))
+	return &resources
+}
+
 // CreateResourceProviderFromCluster creates a new ResourceProvider using live data from a cluster
 func CreateResourceProviderFromCluster(ctx context.Context, c conf.Configuration) (*ResourceProvider, error) {
 	kubeConf, configError := config.GetConfigWithContext(c.KubeContext)

pkg/mutation/mutate.go

 package mutation
 
 import (
+	"bufio"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	jsonpatch "github.com/evanphx/json-patch/v5"
 	"github.com/fairwindsops/polaris/pkg/config"
 	"github.com/fairwindsops/polaris/pkg/kube"
 	"github.com/fairwindsops/polaris/pkg/validator"
+	"github.com/thoas/go-funk"
 )
 
 // ApplyAllSchemaMutations applies available mutation to a single resource
@@ -35,9 +38,10 @@ func ApplyAllSchemaMutations(conf *config.Configuration, resourceProvider *kube.
 	return mutated, nil
 }
 
-// GetMutationsFromResults returns all mutations from results
-func GetMutationsFromResults(conf *config.Configuration, results []validator.Result) map[string][]map[string]interface{} {
+// GetMutationsAndCommentsFromResults returns all mutations from results
+func GetMutationsAndCommentsFromResults(results []validator.Result) ([]config.MutationComment, map[string][]map[string]interface{}) {
 	allMutationsFromResults := make(map[string][]map[string]interface{})
+	comments := []config.MutationComment{}
 	for _, result := range results {
 		key := fmt.Sprintf("%s/%s/%s", result.Kind, result.Name, result.Namespace)
 
@@ -49,6 +53,9 @@ func GetMutationsFromResults(conf *config.Configuration, results []validator.Res
 				}
 				allMutationsFromResults[key] = append(mutations, resultMessage.Mutations...)
 			}
+			if len(resultMessage.Comments) > 0 {
+				comments = append(comments, resultMessage.Comments...)
+			}
 		}
 
 		for _, resultMessage := range result.PodResult.Results {
@@ -59,6 +66,9 @@ func GetMutationsFromResults(conf *config.Configuration, results []validator.Res
 				}
 				allMutationsFromResults[key] = append(mutations, resultMessage.Mutations...)
 			}
+			if len(resultMessage.Comments) > 0 {
+				comments = append(comments, resultMessage.Comments...)
+			}
 		}
 
 		for _, containerResult := range result.PodResult.ContainerResults {
@@ -70,9 +80,36 @@ func GetMutationsFromResults(conf *config.Configuration, results []validator.Res
 					}
 					allMutationsFromResults[key] = append(mutations, resultMessage.Mutations...)
 				}
+				if len(resultMessage.Comments) > 0 {
+					comments = append(comments, resultMessage.Comments...)
+				}
 			}
 		}
 
 	}
-	return allMutationsFromResults
+	return comments, allMutationsFromResults
+}
+
+// UpdateMutatedContentWithComments Updates mutated object with comments
+func UpdateMutatedContentWithComments(yamlContent string, comments []config.MutationComment) string {
+	var lines []string
+	scanner := bufio.NewScanner(strings.NewReader(yamlContent))
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		line := scanner.Text()
+		lines = append(lines, line)
+	}
+	commentMap := funk.Map(comments, func(c config.MutationComment) (string, string) {
+		return c.Find, c.Comment
+	}).(map[string]string)
+	fileContent := ""
+	for _, line := range lines {
+		comment, ok := commentMap[strings.TrimSpace(line)]
+		if ok {
+			line += (" #" + comment)
+		}
+		fileContent += line
+		fileContent += "\n"
+	}
+	return fileContent
 }

pkg/validator/output.go

@@ -79,6 +79,7 @@ type ResultMessage struct {
 	Severity  config.Severity
 	Category  string
 	Mutations []map[string]interface{}
+	Comments  []config.MutationComment
 }
 
 // ResultSet contiains the results for a set of checks

pkg/validator/schema.go

@@ -319,6 +319,7 @@ func applySchemaCheck(conf *config.Configuration, checkID string, test schemaTes
 				return mutationCopy
 			}).([]map[string]interface{})
 			result.Mutations = mutations
+			result.Comments = check.Comments
 		}
 	}
 	return &result, nil

test/checks/cpuLimitsMissing/mutated.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/name: nginx
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources:
+      limits:
+        cpu: 100m #TODO: Set this to the amount of CPU you want to reserve for your workload
+        memory: 128Mi

test/checks/cpuRequestsMissing/mutated.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/name: nginx
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources:
+      requests:
+        cpu: 100m #TODO: Set this to the amount of CPU you want to reserve for your workload
+        memory: 128Mi

test/checks/deploymentMissingReplicas/mutated.yaml

+apiVersion: v1
+kind: Deployment
+metadata:
+  labels:
+    env: test
+  name: nginx
+spec:
+  replicas: 3
+  template:
+    spec:
+      containers: []

test/checks/deploymentMissingReplicas/success.yaml

@@ -8,5 +8,4 @@ spec:
   replicas: 3
   template:
     spec:
-      containers: []
-
+      containers: []
\ No newline at end of file

test/checks/hostIPCSet/mutated.yaml

+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app.kubernetes.io/name: nginx
+  name: nginx
+spec:
+  containers:
+  - image: nginx
+    name: nginx
+    resources: {}