feat: improve output structure for non-interactive modes

bl/evaluation/evaluator.go

@@ -27,6 +27,10 @@ func New(c CLIClient) *Evaluator {
 
 type FileNameRuleMapper map[string]map[int]*Rule
 
+type ResultType struct {
+	EvaluationResults               *EvaluationResults
+	NonInteractiveEvaluationResults *NonInteractiveEvaluationResults
+}
 type EvaluationResults struct {
 	FileNameRuleMapper FileNameRuleMapper
 	Summary            struct {
@@ -78,10 +82,10 @@ func (e *Evaluator) UpdateFailedK8sValidation(invalidFiles []*validation.Invalid
 	return err
 }
 
-func (e *Evaluator) Evaluate(filesConfigurations []*extractor.FileConfigurations, evaluationId int) (*EvaluationResults, error) {
+func (e *Evaluator) Evaluate(filesConfigurations []*extractor.FileConfigurations, evaluationId int, nonInteractiveMode bool, rulesCount int, policyName string) (ResultType, error) {
 
 	if len(filesConfigurations) == 0 {
-		return &EvaluationResults{}, nil
+		return ResultType{}, nil
 	}
 
 	res, err := e.cliClient.RequestEvaluation(&cliClient.EvaluationRequest{
@@ -89,11 +93,51 @@ func (e *Evaluator) Evaluate(filesConfigurations []*extractor.FileConfigurations
 		Files:        filesConfigurations,
 	})
 	if err != nil {
-		return nil, err
+		return ResultType{}, err
+	}
+	resultType := ResultType{}
+	resultType.EvaluationResults = e.formatEvaluationResults(res.Results, len(filesConfigurations))
+	if nonInteractiveMode {
+		resultType.NonInteractiveEvaluationResults = e.formatNonInteractiveEvaluationResults(resultType.EvaluationResults, res.Results, policyName, rulesCount)
+	}
+	return resultType, nil
+}
+
+// This method creates a NonInteractiveEvaluationResults structure
+// from EvaluationResults.
+func (e *Evaluator) formatNonInteractiveEvaluationResults(evaluationResults *EvaluationResults, listEvaluationResult []*cliClient.EvaluationResult, policyName string, totalRulesInPolicy int) *NonInteractiveEvaluationResults {
+	fileNameRuleMapper := evaluationResults.FileNameRuleMapper
+	ruleMapper := make(map[int]string)
+	for _, result := range listEvaluationResult {
+		ruleId := getRuleId(result)
+		ruleMapper[ruleId] = result.Rule.Identifier
+	}
+	nonInteractiveEvaluationResults := NonInteractiveEvaluationResults{}
+
+	for fileName, rules := range fileNameRuleMapper {
+		formattedEvaluationResults := FormattedEvaluationResults{}
+		formattedEvaluationResults.FileName = fileName
+
+		for _, rule := range rules {
+			ruleRObject := RulerObject{Identifier: ruleMapper[rule.ID], Name: rule.Name, MessageOnFailure: rule.FailSuggestion, OccurrencesDetails: rule.OccurrencesDetails}
+			formattedEvaluationResults.RuleRresults = append(
+				formattedEvaluationResults.RuleRresults,
+				&ruleRObject,
+			)
+		}
+		nonInteractiveEvaluationResults.FormattedEvaluationResults = append(
+			nonInteractiveEvaluationResults.FormattedEvaluationResults,
+			&formattedEvaluationResults,
+		)
+	}
+	nonInteractiveEvaluationResults.PolicySummary = PolicySummary{
+		PolicyName:         policyName,
+		TotalRulesInPolicy: totalRulesInPolicy,
+		TotalRulesFailed:   evaluationResults.Summary.TotalFailedRules,
+		TotalPassedCount:   evaluationResults.Summary.TotalPassedCount,
 	}
 
-	results := e.formatEvaluationResults(res.Results, len(filesConfigurations))
-	return results, nil
+	return &nonInteractiveEvaluationResults
 }
 
 func (e *Evaluator) formatEvaluationResults(evaluationResults []*cliClient.EvaluationResult, filesCount int) *EvaluationResults {

bl/evaluation/evaluator_test.go

@@ -107,12 +107,12 @@ func TestEvaluate(t *testing.T) {
 			}
 
 			// TODO: define and check the rest of the values
-			results, _ := evaluator.Evaluate(tt.args.validFilesConfigurations, tt.args.evaluationId)
+			results, _ := evaluator.Evaluate(tt.args.validFilesConfigurations, tt.args.evaluationId, tt.args.nonInteractiveMode, tt.args.rulesCount, tt.args.policyName)
 
 			if tt.expected.isRequestEvaluationCalled {
 				mockedCliClient.AssertCalled(t, "RequestEvaluation", mock.Anything)
-				assert.Equal(t, tt.expected.response.Summary, results.Summary)
-				assert.Equal(t, tt.expected.response.FileNameRuleMapper, results.FileNameRuleMapper)
+				assert.Equal(t, tt.expected.response.EvaluationResults.Summary, results.EvaluationResults.Summary)
+				assert.Equal(t, tt.expected.response.EvaluationResults.FileNameRuleMapper, results.EvaluationResults.FileNameRuleMapper)
 			} else {
 				mockedCliClient.AssertNotCalled(t, "RequestEvaluation")
 			}
@@ -124,11 +124,13 @@ type evaluateArgs struct {
 	validFilesConfigurations []*extractor.FileConfigurations
 	evaluationId             int
 	osInfo                   *OSInfo
+	nonInteractiveMode       bool
 	rulesCount               int
+	policyName               string
 }
 
 type evaluateExpected struct {
-	response                  *EvaluationResults
+	response                  ResultType
 	err                       error
 	isRequestEvaluationCalled bool
 	isCreateEvaluationCalled  bool
@@ -155,6 +157,9 @@ func request_evaluation_all_valid() *evaluateTestCase {
 				PlatformVersion: "1.2.3",
 				KernelVersion:   "4.5.6",
 			},
+			nonInteractiveMode: false,
+			rulesCount:         21,
+			policyName:         "Default",
 		},
 		mock: &evaluatorMock{
 			cliClient: &cliClientMockTestCase{
@@ -188,16 +193,18 @@ func request_evaluation_all_valid() *evaluateTestCase {
 			},
 		},
 		expected: &evaluateExpected{
-			response: &EvaluationResults{
-				FileNameRuleMapper: make(map[string]map[int]*Rule),
-				Summary: struct {
-					TotalFailedRules int
-					FilesCount       int
-					TotalPassedCount int
-				}{
-					TotalFailedRules: 0,
-					FilesCount:       1,
-					TotalPassedCount: 1,
+			response: ResultType{
+				EvaluationResults: &EvaluationResults{
+					FileNameRuleMapper: make(map[string]map[int]*Rule),
+					Summary: struct {
+						TotalFailedRules int
+						FilesCount       int
+						TotalPassedCount int
+					}{
+						TotalFailedRules: 0,
+						FilesCount:       1,
+						TotalPassedCount: 1,
+					},
 				},
 			},
 			err:                       nil,
@@ -217,6 +224,9 @@ func request_evaluation_all_invalid() *evaluateTestCase {
 				PlatformVersion: "1.2.3",
 				KernelVersion:   "4.5.6",
 			},
+			nonInteractiveMode: false,
+			rulesCount:         21,
+			policyName:         "Default",
 		},
 		mock: &evaluatorMock{
 			cliClient: &cliClientMockTestCase{
@@ -250,16 +260,18 @@ func request_evaluation_all_invalid() *evaluateTestCase {
 			},
 		},
 		expected: &evaluateExpected{
-			response: &EvaluationResults{
-				FileNameRuleMapper: make(map[string]map[int]*Rule),
-				Summary: struct {
-					TotalFailedRules int
-					FilesCount       int
-					TotalPassedCount int
-				}{
-					TotalFailedRules: 0,
-					FilesCount:       1,
-					TotalPassedCount: 0,
+			response: ResultType{
+				EvaluationResults: &EvaluationResults{
+					FileNameRuleMapper: make(map[string]map[int]*Rule),
+					Summary: struct {
+						TotalFailedRules int
+						FilesCount       int
+						TotalPassedCount int
+					}{
+						TotalFailedRules: 0,
+						FilesCount:       1,
+						TotalPassedCount: 0,
+					},
 				},
 			},
 			err:                       nil,

bl/evaluation/non_interactive_evaluator.go

+package evaluation
+
+import (
+	"github.com/datreeio/datree/bl/validation"
+)
+
+type FormattedOutput struct {
+	PolicyValidationResults []*FormattedEvaluationResults
+	PolicySummary           PolicySummary
+	EvaluationSummary       struct {
+		ConfigsCount              int
+		FilesCount                int
+		PassedYamlValidationCount int
+		PassedK8sValidationCount  int
+		PassedPolicyCheckCount    int
+	}
+	YamlValidationResults []*validation.InvalidYamlFile
+	K8sValidationResults  []*validation.InvalidK8sFile
+}
+
+type NonInteractiveEvaluationResults struct {
+	FormattedEvaluationResults []*FormattedEvaluationResults
+	PolicySummary              PolicySummary
+}
+
+type FormattedEvaluationResults struct {
+	FileName     string
+	RuleRresults []*RulerObject
+}
+
+type RulerObject struct {
+	Identifier         string
+	Name               string
+	MessageOnFailure   string
+	OccurrencesDetails []OccurrenceDetails
+}
+
+type PolicySummary struct {
+	PolicyName         string
+	TotalRulesInPolicy int
+	TotalRulesFailed   int
+	TotalPassedCount   int
+}

bl/evaluation/printer.go

@@ -21,23 +21,41 @@ type Printer interface {
 	PrintEvaluationSummary(summary printer.EvaluationSummary, k8sVersion string)
 }
 
-type FormattedOutput struct {
-	EvaluationResults *EvaluationResults
-	EvaluationSummary printer.EvaluationSummary
-	InvalidYamlFiles  []*validation.InvalidYamlFile
-	InvalidK8sFiles   []*validation.InvalidK8sFile
-}
+func PrintResults(results ResultType, invalidYamlFiles []*validation.InvalidYamlFile, invalidK8sFiles []*validation.InvalidK8sFile, evaluationSummary printer.EvaluationSummary, loginURL string, outputFormat string, printer Printer, k8sVersion string, policyName string) error {
+	if outputFormat == "json" || outputFormat == "yaml" || outputFormat == "xml" {
+		nonInteractiveEvaluationResults := results.NonInteractiveEvaluationResults
+		if nonInteractiveEvaluationResults == nil {
+			nonInteractiveEvaluationResults = &NonInteractiveEvaluationResults{}
+		}
+		formattedOutput := FormattedOutput{
+			PolicyValidationResults: nonInteractiveEvaluationResults.FormattedEvaluationResults,
+			PolicySummary:           nonInteractiveEvaluationResults.PolicySummary,
+			EvaluationSummary: struct {
+				ConfigsCount              int
+				FilesCount                int
+				PassedYamlValidationCount int
+				PassedK8sValidationCount  int
+				PassedPolicyCheckCount    int
+			}{
+				ConfigsCount:              evaluationSummary.ConfigsCount,
+				FilesCount:                evaluationSummary.FilesCount,
+				PassedYamlValidationCount: evaluationSummary.PassedYamlValidationCount,
+				PassedK8sValidationCount:  evaluationSummary.PassedK8sValidationCount,
+				PassedPolicyCheckCount:    evaluationSummary.PassedPolicyCheckCount,
+			},
+			YamlValidationResults: invalidYamlFiles,
+			K8sValidationResults:  invalidK8sFiles,
+		}
 
-func PrintResults(results *EvaluationResults, invalidYamlFiles []*validation.InvalidYamlFile, invalidK8sFiles []*validation.InvalidK8sFile, evaluationSummary printer.EvaluationSummary, loginURL string, outputFormat string, printer Printer, k8sVersion string, policyName string) error {
-	switch {
-	case outputFormat == "json":
-		return jsonOutput(&FormattedOutput{EvaluationResults: results, EvaluationSummary: evaluationSummary, InvalidYamlFiles: invalidYamlFiles, InvalidK8sFiles: invalidK8sFiles})
-	case outputFormat == "yaml":
-		return yamlOutput(&FormattedOutput{EvaluationResults: results, EvaluationSummary: evaluationSummary, InvalidYamlFiles: invalidYamlFiles, InvalidK8sFiles: invalidK8sFiles})
-	case outputFormat == "xml":
-		return xmlOutput(&FormattedOutput{EvaluationResults: results, EvaluationSummary: evaluationSummary, InvalidYamlFiles: invalidYamlFiles, InvalidK8sFiles: invalidK8sFiles})
-	default:
-		return textOutput(results, invalidYamlFiles, invalidK8sFiles, evaluationSummary, loginURL, printer, k8sVersion, policyName)
+		if outputFormat == "json" {
+			return jsonOutput(&formattedOutput)
+		} else if outputFormat == "yaml" {
+			return yamlOutput(&formattedOutput)
+		} else {
+			return xmlOutput(&formattedOutput)
+		}
+	} else {
+		return textOutput(results.EvaluationResults, invalidYamlFiles, invalidK8sFiles, evaluationSummary, loginURL, printer, k8sVersion, policyName)
 	}
 }
 

bl/evaluation/printer_test.go

@@ -31,7 +31,7 @@ func (c *mockPrinter) PrintEvaluationSummary(summary printer.EvaluationSummary,
 }
 
 type printResultsTestCaseArgs struct {
-	results           *EvaluationResults
+	results           ResultType
 	invalidYamlFiles  []*validation.InvalidYamlFile
 	invalidK8sFiles   []*validation.InvalidK8sFile
 	evaluationSummary printer.EvaluationSummary
@@ -76,7 +76,7 @@ func TestPrintResults(t *testing.T) {
 				mockedPrinter.AssertNotCalled(t, "PrintWarnings")
 			} else {
 				pwd, _ := os.Getwd()
-				warnings, _ := parseToPrinterWarnings(tt.args.results, tt.args.invalidYamlFiles, tt.args.invalidK8sFiles, pwd, "1.18.0")
+				warnings, _ := parseToPrinterWarnings(tt.args.results.EvaluationResults, tt.args.invalidYamlFiles, tt.args.invalidK8sFiles, pwd, "1.18.0")
 				mockedPrinter.AssertCalled(t, "PrintWarnings", warnings)
 			}
 		})
@@ -127,75 +127,79 @@ func readOutput(outputFormat string, formattedOutput FormattedOutput) string {
 }
 
 func createFormattedOutput() FormattedOutput {
-	evaluationResults := &EvaluationResults{
-		Summary: struct {
-			TotalFailedRules int
-			FilesCount       int
-			TotalPassedCount int
-		}{
-			TotalFailedRules: 4,
-			FilesCount:       1,
-			TotalPassedCount: 0,
+	evaluationResults := &NonInteractiveEvaluationResults{
+		PolicySummary: PolicySummary{
+			PolicyName:         "Default",
+			TotalRulesInPolicy: 21,
+			TotalRulesFailed:   4,
+			TotalPassedCount:   0,
 		},
-		FileNameRuleMapper: FileNameRuleMapper{"File1": map[int]*Rule{
-			1: &Rule{
-				ID:             1,
-				Name:           "Ensure each container image has a pinned (tag) version",
-				FailSuggestion: "Incorrect value for key `image` - specify an image version to avoid unpleasant \"version surprises\" in the future",
-				OccurrencesDetails: []OccurrenceDetails{{
-					MetadataName: "rss-site",
-					Kind:         "Deployment",
-				}},
-			},
-			4: &Rule{
-				ID:             4,
-				Name:           "Ensure each container has a configured memory limit",
-				FailSuggestion: "Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization",
-				OccurrencesDetails: []OccurrenceDetails{{
-					MetadataName: "rss-site",
-					Kind:         "Deployment",
-				}},
-			},
-			9: &Rule{
-				ID:             9,
-				Name:           "Ensure workload has valid label values",
-				FailSuggestion: "Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it",
+		FormattedEvaluationResults: []*FormattedEvaluationResults{&FormattedEvaluationResults{
+			FileName: "File1",
+			RuleRresults: []*RulerObject{&RulerObject{
+				Identifier:       "CONTAINERS_MISSING_IMAGE_VALUE_VERSION",
+				Name:             "Ensure each container image has a pinned (tag) version",
+				MessageOnFailure: "Incorrect value for key `image` - specify an image version to avoid unpleasant \"version surprises\" in the future",
 				OccurrencesDetails: []OccurrenceDetails{{
 					MetadataName: "rss-site",
 					Kind:         "Deployment",
 				}},
 			},
-			11: &Rule{
-				ID:             11,
-				Name:           "Ensure each container has a configured liveness probe",
-				FailSuggestion: "Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks",
-				OccurrencesDetails: []OccurrenceDetails{{
-					MetadataName: "rss-site",
-					Kind:         "Deployment",
-				}},
+				&RulerObject{
+					Identifier:       "CONTAINERS_MISSING_MEMORY_LIMIT_KEY",
+					Name:             "Ensure each container has a configured memory limit",
+					MessageOnFailure: "Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization",
+					OccurrencesDetails: []OccurrenceDetails{{
+						MetadataName: "rss-site",
+						Kind:         "Deployment",
+					}},
+				},
+				&RulerObject{
+					Identifier:       "WORKLOAD_INVALID_LABELS_VALUE",
+					Name:             "Ensure workload has valid label values",
+					MessageOnFailure: "Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it",
+					OccurrencesDetails: []OccurrenceDetails{{
+						MetadataName: "rss-site",
+						Kind:         "Deployment",
+					}},
+				},
+				&RulerObject{
+					Identifier:       "CONTAINERS_MISSING_LIVENESSPROBE_KEY",
+					Name:             "Ensure each container has a configured liveness probe",
+					MessageOnFailure: "Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks",
+					OccurrencesDetails: []OccurrenceDetails{{
+						MetadataName: "rss-site",
+						Kind:         "Deployment",
+					}},
+				},
 			},
 		}},
 	}
 
-	evaluationSummary := &printer.EvaluationSummary{
-		ConfigsCount:              1,
-		RulesCount:                21,
-		FilesCount:                1,
-		PassedYamlValidationCount: 1,
-		PassedK8sValidationCount:  1,
-		PassedPolicyCheckCount:    0,
-	}
 	return FormattedOutput{
-		EvaluationResults: evaluationResults,
-		EvaluationSummary: *evaluationSummary,
+		PolicyValidationResults: evaluationResults.FormattedEvaluationResults,
+		PolicySummary:           evaluationResults.PolicySummary,
+		EvaluationSummary: struct {
+			ConfigsCount              int
+			FilesCount                int
+			PassedYamlValidationCount int
+			PassedK8sValidationCount  int
+			PassedPolicyCheckCount    int
+		}{
+			ConfigsCount:              1,
+			FilesCount:                1,
+			PassedYamlValidationCount: 1,
+			PassedK8sValidationCount:  1,
+			PassedPolicyCheckCount:    0,
+		},
 	}
 }
 
 func getExpectedOutputs() expectedOutputs {
 	return expectedOutputs{
-		json: "{\"EvaluationResults\":{\"FileNameRuleMapper\":{\"File1\":{\"1\":{\"ID\":1,\"Name\":\"Ensure each container image has a pinned (tag) version\",\"FailSuggestion\":\"Incorrect value for key `image` - specify an image version to avoid unpleasant \\\"version surprises\\\" in the future\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]},\"11\":{\"ID\":11,\"Name\":\"Ensure each container has a configured liveness probe\",\"FailSuggestion\":\"Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]},\"4\":{\"ID\":4,\"Name\":\"Ensure each container has a configured memory limit\",\"FailSuggestion\":\"Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]},\"9\":{\"ID\":9,\"Name\":\"Ensure workload has valid label values\",\"FailSuggestion\":\"Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]}}},\"Summary\":{\"TotalFailedRules\":4,\"FilesCount\":1,\"TotalPassedCount\":0}},\"EvaluationSummary\":{\"ConfigsCount\":1,\"RulesCount\":21,\"FilesCount\":1,\"PassedYamlValidationCount\":1,\"PassedK8sValidationCount\":1,\"PassedPolicyCheckCount\":0},\"InvalidYamlFiles\":null,\"InvalidK8sFiles\":null}\n",
-		yaml: "evaluationresults:\n  filenamerulemapper:\n    File1:\n      1:\n        id: 1\n        name: Ensure each container image has a pinned (tag) version\n        failsuggestion: Incorrect value for key `image` - specify an image version\n          to avoid unpleasant \"version surprises\" in the future\n        occurrencesdetails:\n        - metadataname: rss-site\n          kind: Deployment\n      4:\n        id: 4\n        name: Ensure each container has a configured memory limit\n        failsuggestion: Missing property object `limits.memory` - value should be\n          within the accepted boundaries recommended by the organization\n        occurrencesdetails:\n        - metadataname: rss-site\n          kind: Deployment\n      9:\n        id: 9\n        name: Ensure workload has valid label values\n        failsuggestion: Incorrect value for key(s) under `labels` - the vales syntax\n          is not valid so the Kubernetes engine will not accept it\n        occurrencesdetails:\n        - metadataname: rss-site\n          kind: Deployment\n      11:\n        id: 11\n        name: Ensure each container has a configured liveness probe\n        failsuggestion: Missing property object `livenessProbe` - add a properly configured\n          livenessProbe to catch possible deadlocks\n        occurrencesdetails:\n        - metadataname: rss-site\n          kind: Deployment\n  summary:\n    totalfailedrules: 4\n    filescount: 1\n    totalpassedcount: 0\nevaluationsummary:\n  configscount: 1\n  rulescount: 21\n  filescount: 1\n  passedyamlvalidationcount: 1\n  passedk8svalidationcount: 1\n  passedpolicycheckcount: 0\ninvalidyamlfiles: []\ninvalidk8sfiles: []\n\n",
-		xml:  "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<FormattedOutput>\n\t<EvaluationResults>\n\t\t<FileNameRuleMapper>\n\t\t\t<File filename=\"File1\">\n\t\t\t\t<Rule>\n\t\t\t\t\t<ID>1</ID>\n\t\t\t\t\t<Name>Ensure each container image has a pinned (tag) version</Name>\n\t\t\t\t\t<FailSuggestion>Incorrect value for key `image` - specify an image version to avoid unpleasant &#34;version surprises&#34; in the future</FailSuggestion>\n\t\t\t\t\t<OccurrencesDetails>\n\t\t\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t\t\t</OccurrencesDetails>\n\t\t\t\t</Rule>\n\t\t\t</File>\n\t\t\t<File filename=\"File1\">\n\t\t\t\t<Rule>\n\t\t\t\t\t<ID>4</ID>\n\t\t\t\t\t<Name>Ensure each container has a configured memory limit</Name>\n\t\t\t\t\t<FailSuggestion>Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization</FailSuggestion>\n\t\t\t\t\t<OccurrencesDetails>\n\t\t\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t\t\t</OccurrencesDetails>\n\t\t\t\t</Rule>\n\t\t\t</File>\n\t\t\t<File filename=\"File1\">\n\t\t\t\t<Rule>\n\t\t\t\t\t<ID>9</ID>\n\t\t\t\t\t<Name>Ensure workload has valid label values</Name>\n\t\t\t\t\t<FailSuggestion>Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it</FailSuggestion>\n\t\t\t\t\t<OccurrencesDetails>\n\t\t\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t\t\t</OccurrencesDetails>\n\t\t\t\t</Rule>\n\t\t\t</File>\n\t\t\t<File filename=\"File1\">\n\t\t\t\t<Rule>\n\t\t\t\t\t<ID>11</ID>\n\t\t\t\t\t<Name>Ensure each container has a configured liveness probe</Name>\n\t\t\t\t\t<FailSuggestion>Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks</FailSuggestion>\n\t\t\t\t\t<OccurrencesDetails>\n\t\t\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t\t\t</OccurrencesDetails>\n\t\t\t\t</Rule>\n\t\t\t</File>\n\t\t</FileNameRuleMapper>\n\t\t<Summary>\n\t\t\t<TotalFailedRules>4</TotalFailedRules>\n\t\t\t<FilesCount>1</FilesCount>\n\t\t\t<TotalPassedCount>0</TotalPassedCount>\n\t\t</Summary>\n\t</EvaluationResults>\n\t<EvaluationSummary>\n\t\t<ConfigsCount>1</ConfigsCount>\n\t\t<RulesCount>21</RulesCount>\n\t\t<FilesCount>1</FilesCount>\n\t\t<PassedYamlValidationCount>1</PassedYamlValidationCount>\n\t\t<PassedK8sValidationCount>1</PassedK8sValidationCount>\n\t\t<PassedPolicyCheckCount>0</PassedPolicyCheckCount>\n\t</EvaluationSummary>\n</FormattedOutput>\n",
+		json: "{\"PolicyValidationResults\":[{\"FileName\":\"File1\",\"RuleRresults\":[{\"Identifier\":\"CONTAINERS_MISSING_IMAGE_VALUE_VERSION\",\"Name\":\"Ensure each container image has a pinned (tag) version\",\"MessageOnFailure\":\"Incorrect value for key `image` - specify an image version to avoid unpleasant \\\"version surprises\\\" in the future\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]},{\"Identifier\":\"CONTAINERS_MISSING_MEMORY_LIMIT_KEY\",\"Name\":\"Ensure each container has a configured memory limit\",\"MessageOnFailure\":\"Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]},{\"Identifier\":\"WORKLOAD_INVALID_LABELS_VALUE\",\"Name\":\"Ensure workload has valid label values\",\"MessageOnFailure\":\"Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]},{\"Identifier\":\"CONTAINERS_MISSING_LIVENESSPROBE_KEY\",\"Name\":\"Ensure each container has a configured liveness probe\",\"MessageOnFailure\":\"Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks\",\"OccurrencesDetails\":[{\"MetadataName\":\"rss-site\",\"Kind\":\"Deployment\"}]}]}],\"PolicySummary\":{\"PolicyName\":\"Default\",\"TotalRulesInPolicy\":21,\"TotalRulesFailed\":4,\"TotalPassedCount\":0},\"EvaluationSummary\":{\"ConfigsCount\":1,\"FilesCount\":1,\"PassedYamlValidationCount\":1,\"PassedK8sValidationCount\":1,\"PassedPolicyCheckCount\":0},\"YamlValidationResults\":null,\"K8sValidationResults\":null}\n",
+		yaml: "policyvalidationresults:\n- filename: File1\n  rulerresults:\n  - identifier: CONTAINERS_MISSING_IMAGE_VALUE_VERSION\n    name: Ensure each container image has a pinned (tag) version\n    messageonfailure: Incorrect value for key `image` - specify an image version to\n      avoid unpleasant \"version surprises\" in the future\n    occurrencesdetails:\n    - metadataname: rss-site\n      kind: Deployment\n  - identifier: CONTAINERS_MISSING_MEMORY_LIMIT_KEY\n    name: Ensure each container has a configured memory limit\n    messageonfailure: Missing property object `limits.memory` - value should be within\n      the accepted boundaries recommended by the organization\n    occurrencesdetails:\n    - metadataname: rss-site\n      kind: Deployment\n  - identifier: WORKLOAD_INVALID_LABELS_VALUE\n    name: Ensure workload has valid label values\n    messageonfailure: Incorrect value for key(s) under `labels` - the vales syntax\n      is not valid so the Kubernetes engine will not accept it\n    occurrencesdetails:\n    - metadataname: rss-site\n      kind: Deployment\n  - identifier: CONTAINERS_MISSING_LIVENESSPROBE_KEY\n    name: Ensure each container has a configured liveness probe\n    messageonfailure: Missing property object `livenessProbe` - add a properly configured\n      livenessProbe to catch possible deadlocks\n    occurrencesdetails:\n    - metadataname: rss-site\n      kind: Deployment\npolicysummary:\n  policyname: Default\n  totalrulesinpolicy: 21\n  totalrulesfailed: 4\n  totalpassedcount: 0\nevaluationsummary:\n  configscount: 1\n  filescount: 1\n  passedyamlvalidationcount: 1\n  passedk8svalidationcount: 1\n  passedpolicycheckcount: 0\nyamlvalidationresults: []\nk8svalidationresults: []\n\n",
+		xml:  "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<FormattedOutput>\n\t<PolicyValidationResults>\n\t\t<FileName>File1</FileName>\n\t\t<RuleRresults>\n\t\t\t<Identifier>CONTAINERS_MISSING_IMAGE_VALUE_VERSION</Identifier>\n\t\t\t<Name>Ensure each container image has a pinned (tag) version</Name>\n\t\t\t<MessageOnFailure>Incorrect value for key `image` - specify an image version to avoid unpleasant &#34;version surprises&#34; in the future</MessageOnFailure>\n\t\t\t<OccurrencesDetails>\n\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t</OccurrencesDetails>\n\t\t</RuleRresults>\n\t\t<RuleRresults>\n\t\t\t<Identifier>CONTAINERS_MISSING_MEMORY_LIMIT_KEY</Identifier>\n\t\t\t<Name>Ensure each container has a configured memory limit</Name>\n\t\t\t<MessageOnFailure>Missing property object `limits.memory` - value should be within the accepted boundaries recommended by the organization</MessageOnFailure>\n\t\t\t<OccurrencesDetails>\n\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t</OccurrencesDetails>\n\t\t</RuleRresults>\n\t\t<RuleRresults>\n\t\t\t<Identifier>WORKLOAD_INVALID_LABELS_VALUE</Identifier>\n\t\t\t<Name>Ensure workload has valid label values</Name>\n\t\t\t<MessageOnFailure>Incorrect value for key(s) under `labels` - the vales syntax is not valid so the Kubernetes engine will not accept it</MessageOnFailure>\n\t\t\t<OccurrencesDetails>\n\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t</OccurrencesDetails>\n\t\t</RuleRresults>\n\t\t<RuleRresults>\n\t\t\t<Identifier>CONTAINERS_MISSING_LIVENESSPROBE_KEY</Identifier>\n\t\t\t<Name>Ensure each container has a configured liveness probe</Name>\n\t\t\t<MessageOnFailure>Missing property object `livenessProbe` - add a properly configured livenessProbe to catch possible deadlocks</MessageOnFailure>\n\t\t\t<OccurrencesDetails>\n\t\t\t\t<MetadataName>rss-site</MetadataName>\n\t\t\t\t<Kind>Deployment</Kind>\n\t\t\t</OccurrencesDetails>\n\t\t</RuleRresults>\n\t</PolicyValidationResults>\n\t<PolicySummary>\n\t\t<PolicyName>Default</PolicyName>\n\t\t<TotalRulesInPolicy>21</TotalRulesInPolicy>\n\t\t<TotalRulesFailed>4</TotalRulesFailed>\n\t\t<TotalPassedCount>0</TotalPassedCount>\n\t</PolicySummary>\n\t<EvaluationSummary>\n\t\t<ConfigsCount>1</ConfigsCount>\n\t\t<FilesCount>1</FilesCount>\n\t\t<PassedYamlValidationCount>1</PassedYamlValidationCount>\n\t\t<PassedK8sValidationCount>1</PassedK8sValidationCount>\n\t\t<PassedPolicyCheckCount>0</PassedPolicyCheckCount>\n\t</EvaluationSummary>\n</FormattedOutput>\n",
 	}
 }
 
@@ -203,16 +207,27 @@ func print_resultst(outputFormat string) *printResultsTestCase {
 	return &printResultsTestCase{
 		name: "Print Results Text",
 		args: &printResultsTestCaseArgs{
-			results: &EvaluationResults{
-				FileNameRuleMapper: map[string]map[int]*Rule{},
-				Summary: struct {
-					TotalFailedRules int
-					FilesCount       int
-					TotalPassedCount int
-				}{
-					TotalFailedRules: 0,
-					FilesCount:       0,
-					TotalPassedCount: 0,
+			results: ResultType{
+				EvaluationResults: &EvaluationResults{
+					FileNameRuleMapper: map[string]map[int]*Rule{},
+					Summary: struct {
+						TotalFailedRules int
+						FilesCount       int
+						TotalPassedCount int
+					}{
+						TotalFailedRules: 0,
+						FilesCount:       0,
+						TotalPassedCount: 0,
+					},
+				},
+				NonInteractiveEvaluationResults: &NonInteractiveEvaluationResults{
+					PolicySummary: PolicySummary{
+						PolicyName:         "Default",
+						TotalRulesInPolicy: 0,
+						TotalRulesFailed:   0,
+						TotalPassedCount:   0,
+					},
+					FormattedEvaluationResults: []*FormattedEvaluationResults{},
 				},
 			},
 			invalidYamlFiles:  []*validation.InvalidYamlFile{},

cmd/test/main.go

@@ -21,7 +21,7 @@ import (
 )
 
 type Evaluator interface {
-	Evaluate(filesConfigurationsChan []*extractor.FileConfigurations, evaluationId int) (*evaluation.EvaluationResults, error)
+	Evaluate(filesConfigurations []*extractor.FileConfigurations, evaluationId int, nonInteractiveMode bool, rulesCount int, policyName string) (evaluation.ResultType, error)
 	CreateEvaluation(cliId string, cliVersion string, k8sVersion string, policyName string) (*cliClient.CreateEvaluationResponse, error)
 	UpdateFailedYamlValidation(invalidFiles []*validation.InvalidYamlFile, evaluationId int, stopEvaluation bool) error
 	UpdateFailedK8sValidation(invalidFiles []*validation.InvalidK8sFile, evaluationId int, stopEvaluation bool) error
@@ -207,8 +207,8 @@ func test(ctx *TestCommandContext, paths []string, flags TestCommandFlags) error
 	}
 
 	passedPolicyCheckCount := 0
-	if results != nil {
-		passedPolicyCheckCount = results.Summary.TotalPassedCount
+	if results.EvaluationResults != nil {
+		passedPolicyCheckCount = results.EvaluationResults.Summary.TotalPassedCount
 	}
 
 	passedYamlValidationCount := filesPathsLen - validationManager.InvalidYamlFilesCount()
@@ -243,14 +243,14 @@ func test(ctx *TestCommandContext, paths []string, flags TestCommandFlags) error
 
 	if err != nil {
 		invocationFailedErr = err
-	} else if validationManager.InvalidYamlFilesCount() > 0 || validationManager.InvalidK8sFilesCount() > 0 || results.Summary.TotalFailedRules > 0 {
+	} else if validationManager.InvalidYamlFilesCount() > 0 || validationManager.InvalidK8sFilesCount() > 0 || results.EvaluationResults.Summary.TotalFailedRules > 0 {
 		invocationFailedErr = fmt.Errorf("")
 	}
 
 	return invocationFailedErr
 }
 
-func evaluate(ctx *TestCommandContext, filesPaths []string, flags TestCommandFlags, cliId string) (*ValidationManager, *cliClient.CreateEvaluationResponse, *evaluation.EvaluationResults, error) {
+func evaluate(ctx *TestCommandContext, filesPaths []string, flags TestCommandFlags, cliId string) (*ValidationManager, *cliClient.CreateEvaluationResponse, evaluation.ResultType, error) {
 	isInteractiveMode := (flags.Output != "json") && (flags.Output != "yaml") && (flags.Output != "xml")
 
 	if isInteractiveMode {
@@ -281,7 +281,7 @@ func evaluate(ctx *TestCommandContext, filesPaths []string, flags TestCommandFla
 
 	createEvaluationResponse, err := ctx.Evaluator.CreateEvaluation(cliId, ctx.CliVersion, flags.K8sVersion, flags.PolicyName)
 	if err != nil {
-		return validationManager, nil, nil, err
+		return validationManager, nil, evaluation.ResultType{}, err
 	}
 
 	ctx.K8sValidator.InitClient(createEvaluationResponse.K8sVersion, flags.IgnoreMissingSchemas, flags.SchemaLocations)
@@ -304,7 +304,7 @@ func evaluate(ctx *TestCommandContext, filesPaths []string, flags TestCommandFla
 	if validationManager.InvalidYamlFilesCount() > 0 {
 		err = ctx.Evaluator.UpdateFailedYamlValidation(validationManager.InvalidYamlFiles(), createEvaluationResponse.EvaluationId, noValidYamlFiles)
 		if err != nil {
-			return validationManager, createEvaluationResponse, nil, err
+			return validationManager, createEvaluationResponse, evaluation.ResultType{}, err
 		}
 	}
 
@@ -316,16 +316,16 @@ func evaluate(ctx *TestCommandContext, filesPaths []string, flags TestCommandFla
 	if validationManager.InvalidK8sFilesCount() > 0 {
 		err = ctx.Evaluator.UpdateFailedK8sValidation(validationManager.InvalidK8sFiles(), createEvaluationResponse.EvaluationId, noValidK8sFiles)
 		if err != nil {
-			return validationManager, createEvaluationResponse, nil, err
+			return validationManager, createEvaluationResponse, evaluation.ResultType{}, err
 		}
 	}
 
 	validationManager.AggregateValidK8sFiles(validK8sFilesConfigurationsChan)
 
-	results, err := ctx.Evaluator.Evaluate(validationManager.ValidK8sFilesConfigurations(), createEvaluationResponse.EvaluationId)
+	results, err := ctx.Evaluator.Evaluate(validationManager.ValidK8sFilesConfigurations(), createEvaluationResponse.EvaluationId, !isInteractiveMode, createEvaluationResponse.RulesCount, createEvaluationResponse.PolicyName)
 
 	if err != nil {
-		return validationManager, createEvaluationResponse, nil, err
+		return validationManager, createEvaluationResponse, results, err
 	}
 
 	return validationManager, createEvaluationResponse, results, nil

cmd/test/main_test.go

@@ -18,9 +18,9 @@ type mockEvaluator struct {
 	mock.Mock
 }
 
-func (m *mockEvaluator) Evaluate(filesConfigurationsChan []*extractor.FileConfigurations, evaluationId int) (*evaluation.EvaluationResults, error) {
-	args := m.Called(filesConfigurationsChan, evaluationId)
-	return args.Get(0).(*evaluation.EvaluationResults), args.Error(1)
+func (m *mockEvaluator) Evaluate(filesConfigurationsChan []*extractor.FileConfigurations, evaluationId int, nonInteractiveMode bool, rulesCount int, policyName string) (evaluation.ResultType, error) {
+	args := m.Called(filesConfigurationsChan, evaluationId, nonInteractiveMode, rulesCount, policyName)
+	return args.Get(0).(evaluation.ResultType), args.Error(1)
 }
 
 func (m *mockEvaluator) CreateEvaluation(cliId string, cliVersion string, k8sVersion string, policyName string) (*cliClient.CreateEvaluationResponse, error) {
@@ -123,8 +123,9 @@ func (lc *LocalConfigMock) GetLocalConfiguration() (*localConfig.ConfigContent,
 
 func TestTestCommand(t *testing.T) {
 	evaluationId := 444
+	resultType := evaluation.ResultType{}
 
-	evaluationResults := &evaluation.EvaluationResults{
+	resultType.EvaluationResults = &evaluation.EvaluationResults{
 		FileNameRuleMapper: map[string]map[int]*evaluation.Rule{}, Summary: struct {
 			TotalFailedRules int
 			FilesCount       int
@@ -133,7 +134,7 @@ func TestTestCommand(t *testing.T) {
 	}
 
 	mockedEvaluator := &mockEvaluator{}
-	mockedEvaluator.On("Evaluate", mock.Anything, mock.Anything, mock.Anything).Return(evaluationResults, nil)
+	mockedEvaluator.On("Evaluate", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resultType, nil)
 	mockedEvaluator.On("CreateEvaluation", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&cliClient.CreateEvaluationResponse{EvaluationId: evaluationId, K8sVersion: "1.18.0", RulesCount: 21}, nil)
 	mockedEvaluator.On("UpdateFailedYamlValidation", mock.Anything, mock.Anything, mock.Anything).Return(nil)
 	mockedEvaluator.On("UpdateFailedK8sValidation", mock.Anything, mock.Anything, mock.Anything).Return(nil)
@@ -188,28 +189,28 @@ func test_testCommand_no_flags(t *testing.T, evaluator *mockEvaluator, k8sValida
 
 	k8sValidator.AssertCalled(t, "ValidateResources", mock.Anything, 100)
 	evaluator.AssertCalled(t, "CreateEvaluation", "134kh", "", "1.18.0", "Default")
-	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId)
+	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId, false, 21, "Default")
 }
 
 func test_testCommand_json_output(t *testing.T, evaluator *mockEvaluator, k8sValidator *K8sValidatorMock, filesConfigurations []*extractor.FileConfigurations, evaluationId int, ctx *TestCommandContext) {
 	test(ctx, []string{"8/*"}, TestCommandFlags{Output: "json"})
 
 	k8sValidator.AssertCalled(t, "ValidateResources", mock.Anything, 100)
-	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId)
+	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId, true, 21, "Default")
 }
 
 func test_testCommand_yaml_output(t *testing.T, evaluator *mockEvaluator, k8sValidator *K8sValidatorMock, filesConfigurations []*extractor.FileConfigurations, evaluationId int, ctx *TestCommandContext) {
 	test(ctx, []string{"8/*"}, TestCommandFlags{Output: "yaml"})
 
 	k8sValidator.AssertCalled(t, "ValidateResources", mock.Anything, 100)
-	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId)
+	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId, true, 21, "Default")
 }
 
 func test_testCommand_xml_output(t *testing.T, evaluator *mockEvaluator, k8sValidator *K8sValidatorMock, filesConfigurations []*extractor.FileConfigurations, evaluationId int, ctx *TestCommandContext) {
 	test(ctx, []string{"8/*"}, TestCommandFlags{Output: "xml"})
 
 	k8sValidator.AssertCalled(t, "ValidateResources", mock.Anything, 100)
-	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId)
+	evaluator.AssertCalled(t, "Evaluate", filesConfigurations, evaluationId, true, 21, "Default")
 }
 
 func test_testCommand_only_k8s_files(t *testing.T, k8sValidator *K8sValidatorMock, filesConfigurations []*extractor.FileConfigurations, evaluationId int, ctx *TestCommandContext) {