From 7c159fffc0a4f0d29cbb71f6e1772b517293c69c Mon Sep 17 00:00:00 2001
From: RoyUP9 <87927115+RoyUP9@users.noreply.github.com>
Date: Tue, 12 Jul 2022 10:19:24 +0300
Subject: [PATCH] Added redact using insertion filter (#1196)
---
acceptanceTests/cypress.config.js | 1 -
.../cypress/e2e/tests/RegexMasking.js | 7 -
acceptanceTests/tap_test.go | 58 +----
cli/cmd/tap.go | 1 -
cli/cmd/tapRunner.go | 17 +-
cli/config/configStructs/tapConfig.go | 103 ++++++---
performance_analysis/run_tapper_benchmark.sh | 2 +-
tap/api/options.go | 4 +-
tap/extensions/http/handlers.go | 4 -
tap/extensions/http/sensitive_data_cleaner.go | 205 ------------------
10 files changed, 78 insertions(+), 324 deletions(-)
delete mode 100644 acceptanceTests/cypress/e2e/tests/RegexMasking.js
diff --git a/acceptanceTests/cypress.config.js b/acceptanceTests/cypress.config.js
index aaa620128..506441092 100644
--- a/acceptanceTests/cypress.config.js
+++ b/acceptanceTests/cypress.config.js
@@ -11,7 +11,6 @@ module.exports = defineConfig({
testUrl: 'http://localhost:8899/',
redactHeaderContent: 'User-Header[REDACTED]',
redactBodyContent: '{ "User": "[REDACTED]" }',
- regexMaskingBodyContent: '[REDACTED]',
greenFilterColor: 'rgb(210, 250, 210)',
redFilterColor: 'rgb(250, 214, 220)',
bodyJsonClass: '.hljs',
diff --git a/acceptanceTests/cypress/e2e/tests/RegexMasking.js b/acceptanceTests/cypress/e2e/tests/RegexMasking.js
deleted file mode 100644
index 9f3f20fac..000000000
--- a/acceptanceTests/cypress/e2e/tests/RegexMasking.js
+++ /dev/null
@@ -1,7 +0,0 @@
-import {isValueExistsInElement} from "../testHelpers/TrafficHelper";
-
-it('Loading Mizu', function () {
- cy.visit(Cypress.env('testUrl'));
-});
-
-isValueExistsInElement(true, Cypress.env('regexMaskingBodyContent'), Cypress.env('bodyJsonClass'));
diff --git a/acceptanceTests/tap_test.go b/acceptanceTests/tap_test.go
index b71210bbb..a48069b4c 100644
--- a/acceptanceTests/tap_test.go
+++ b/acceptanceTests/tap_test.go
@@ -2,10 +2,8 @@ package acceptanceTests
import (
"archive/zip"
- "bytes"
"fmt"
"io/ioutil"
- "net/http"
"os/exec"
"path"
"strings"
@@ -343,7 +341,7 @@ func TestTapRedact(t *testing.T) {
tapNamespace := GetDefaultTapNamespace()
tapCmdArgs = append(tapCmdArgs, tapNamespace...)
- tapCmdArgs = append(tapCmdArgs, "--redact")
+ tapCmdArgs = append(tapCmdArgs, "--redact", "--set", "tap.redact-patterns.request-headers=User-Header", "--set", "tap.redact-patterns.request-body=User")
tapCmd := exec.Command(cliPath, tapCmdArgs...)
t.Logf("running command: %v", tapCmd.String())
@@ -429,60 +427,6 @@ func TestTapNoRedact(t *testing.T) {
RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/NoRedact.js\"")
}
-func TestTapRegexMasking(t *testing.T) {
- if testing.Short() {
- t.Skip("ignored acceptance test")
- }
-
- cliPath, cliPathErr := GetCliPath()
- if cliPathErr != nil {
- t.Errorf("failed to get cli path, err: %v", cliPathErr)
- return
- }
-
- tapCmdArgs := GetDefaultTapCommandArgs()
-
- tapNamespace := GetDefaultTapNamespace()
- tapCmdArgs = append(tapCmdArgs, tapNamespace...)
-
- tapCmdArgs = append(tapCmdArgs, "--redact")
-
- tapCmdArgs = append(tapCmdArgs, "-r", "Mizu")
-
- tapCmd := exec.Command(cliPath, tapCmdArgs...)
- t.Logf("running command: %v", tapCmd.String())
-
- t.Cleanup(func() {
- if err := CleanupCommand(tapCmd); err != nil {
- t.Logf("failed to cleanup tap command, err: %v", err)
- }
- })
-
- if err := tapCmd.Start(); err != nil {
- t.Errorf("failed to start tap command, err: %v", err)
- return
- }
-
- apiServerUrl := GetApiServerUrl(DefaultApiServerPort)
-
- if err := WaitTapPodsReady(apiServerUrl); err != nil {
- t.Errorf("failed to start tap pods on time, err: %v", err)
- return
- }
-
- proxyUrl := GetProxyUrl(DefaultNamespaceName, DefaultServiceName)
- for i := 0; i < DefaultEntriesCount; i++ {
- response, requestErr := http.Post(fmt.Sprintf("%v/post", proxyUrl), "text/plain", bytes.NewBufferString("Mizu"))
- if _, requestErr = ExecuteHttpRequest(response, requestErr); requestErr != nil {
- t.Errorf("failed to send proxy request, err: %v", requestErr)
- return
- }
- }
-
- RunCypressTests(t, "npx cypress run --spec \"cypress/e2e/tests/RegexMasking.js\"")
-
-}
-
func TestTapIgnoredUserAgents(t *testing.T) {
if testing.Short() {
t.Skip("ignored acceptance test")
diff --git a/cli/cmd/tap.go b/cli/cmd/tap.go
index fa6ce500c..c9efa7308 100644
--- a/cli/cmd/tap.go
+++ b/cli/cmd/tap.go
@@ -48,7 +48,6 @@ func init() {
tapCmd.Flags().Uint16P(configStructs.GuiPortTapName, "p", defaultTapConfig.GuiPort, "Provide a custom port for the web interface webserver")
tapCmd.Flags().StringSliceP(configStructs.NamespacesTapName, "n", defaultTapConfig.Namespaces, "Namespaces selector")
tapCmd.Flags().BoolP(configStructs.AllNamespacesTapName, "A", defaultTapConfig.AllNamespaces, "Tap all namespaces")
- tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
tapCmd.Flags().Bool(configStructs.EnableRedactionTapName, defaultTapConfig.EnableRedaction, "Enables redaction of potentially sensitive request/response headers and body values")
tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
tapCmd.Flags().String(configStructs.InsertionFilterName, defaultTapConfig.InsertionFilter, "Set the insertion filter. Accepts string or a file path.")
diff --git a/cli/cmd/tapRunner.go b/cli/cmd/tapRunner.go
index 5c6d87b79..2b26bfd73 100644
--- a/cli/cmd/tapRunner.go
+++ b/cli/cmd/tapRunner.go
@@ -230,23 +230,8 @@ func getErrorDisplayTextForK8sTapManagerError(err kubernetes.K8sTapManagerError)
}
func getMizuApiFilteringOptions() (*api.TrafficFilteringOptions, error) {
- var compiledRegexSlice []*api.SerializableRegexp
-
- if config.Config.Tap.PlainTextFilterRegexes != nil && len(config.Config.Tap.PlainTextFilterRegexes) > 0 {
- compiledRegexSlice = make([]*api.SerializableRegexp, 0)
- for _, regexStr := range config.Config.Tap.PlainTextFilterRegexes {
- compiledRegex, err := api.CompileRegexToSerializableRegexp(regexStr)
- if err != nil {
- return nil, err
- }
- compiledRegexSlice = append(compiledRegexSlice, compiledRegex)
- }
- }
-
return &api.TrafficFilteringOptions{
- PlainTextMaskingRegexes: compiledRegexSlice,
- IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
- EnableRedaction: config.Config.Tap.EnableRedaction,
+ IgnoredUserAgents: config.Config.Tap.IgnoredUserAgents,
}, nil
}
diff --git a/cli/config/configStructs/tapConfig.go b/cli/config/configStructs/tapConfig.go
index ad43d3ab1..79615e0ec 100644
--- a/cli/config/configStructs/tapConfig.go
+++ b/cli/config/configStructs/tapConfig.go
@@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"regexp"
+ "strings"
"github.com/up9inc/mizu/cli/uiUtils"
"github.com/up9inc/mizu/shared"
@@ -15,38 +16,43 @@ import (
)
const (
- GuiPortTapName = "gui-port"
- NamespacesTapName = "namespaces"
- AllNamespacesTapName = "all-namespaces"
- PlainTextFilterRegexesTapName = "regex-masking"
- EnableRedactionTapName = "redact"
- HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
- InsertionFilterName = "insertion-filter"
- DryRunTapName = "dry-run"
- ServiceMeshName = "service-mesh"
- TlsName = "tls"
- ProfilerName = "profiler"
- MaxLiveStreamsName = "max-live-streams"
+ GuiPortTapName = "gui-port"
+ NamespacesTapName = "namespaces"
+ AllNamespacesTapName = "all-namespaces"
+ EnableRedactionTapName = "redact"
+ HumanMaxEntriesDBSizeTapName = "max-entries-db-size"
+ InsertionFilterName = "insertion-filter"
+ DryRunTapName = "dry-run"
+ ServiceMeshName = "service-mesh"
+ TlsName = "tls"
+ ProfilerName = "profiler"
+ MaxLiveStreamsName = "max-live-streams"
)
type TapConfig struct {
- PodRegexStr string `yaml:"regex" default:".*"`
- GuiPort uint16 `yaml:"gui-port" default:"8899"`
- ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"`
- Namespaces []string `yaml:"namespaces"`
- AllNamespaces bool `yaml:"all-namespaces" default:"false"`
- PlainTextFilterRegexes []string `yaml:"regex-masking"`
- IgnoredUserAgents []string `yaml:"ignored-user-agents"`
- EnableRedaction bool `yaml:"redact" default:"false"`
- HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"`
- InsertionFilter string `yaml:"insertion-filter" default:""`
- DryRun bool `yaml:"dry-run" default:"false"`
- ApiServerResources shared.Resources `yaml:"api-server-resources"`
- TapperResources shared.Resources `yaml:"tapper-resources"`
- ServiceMesh bool `yaml:"service-mesh" default:"false"`
- Tls bool `yaml:"tls" default:"false"`
- Profiler bool `yaml:"profiler" default:"false"`
- MaxLiveStreams int `yaml:"max-live-streams" default:"500"`
+ PodRegexStr string `yaml:"regex" default:".*"`
+ GuiPort uint16 `yaml:"gui-port" default:"8899"`
+ ProxyHost string `yaml:"proxy-host" default:"127.0.0.1"`
+ Namespaces []string `yaml:"namespaces"`
+ AllNamespaces bool `yaml:"all-namespaces" default:"false"`
+ IgnoredUserAgents []string `yaml:"ignored-user-agents"`
+ EnableRedaction bool `yaml:"redact" default:"false"`
+ RedactPatterns struct {
+ RequestHeaders []string `yaml:"request-headers"`
+ ResponseHeaders []string `yaml:"response-headers"`
+ RequestBody []string `yaml:"request-body"`
+ ResponseBody []string `yaml:"response-body"`
+ RequestQueryParams []string `yaml:"request-query-params"`
+ } `yaml:"redact-patterns"`
+ HumanMaxEntriesDBSize string `yaml:"max-entries-db-size" default:"200MB"`
+ InsertionFilter string `yaml:"insertion-filter" default:""`
+ DryRun bool `yaml:"dry-run" default:"false"`
+ ApiServerResources shared.Resources `yaml:"api-server-resources"`
+ TapperResources shared.Resources `yaml:"tapper-resources"`
+ ServiceMesh bool `yaml:"service-mesh" default:"false"`
+ Tls bool `yaml:"tls" default:"false"`
+ Profiler bool `yaml:"profiler" default:"false"`
+ MaxLiveStreams int `yaml:"max-live-streams" default:"500"`
}
func (config *TapConfig) PodRegex() *regexp.Regexp {
@@ -71,9 +77,48 @@ func (config *TapConfig) GetInsertionFilter() string {
}
}
}
+
+ redactFilter := getRedactFilter(config)
+ if insertionFilter != "" && redactFilter != "" {
+ return fmt.Sprintf("(%s) and (%s)", insertionFilter, redactFilter)
+ } else if insertionFilter == "" && redactFilter != "" {
+ return redactFilter
+ }
+
return insertionFilter
}
+func getRedactFilter(config *TapConfig) string {
+ if !config.EnableRedaction {
+ return ""
+ }
+
+ var redactValues []string
+ for _, requestHeader := range config.RedactPatterns.RequestHeaders {
+ redactValues = append(redactValues, fmt.Sprintf("request.headers['%s']", requestHeader))
+ }
+ for _, responseHeader := range config.RedactPatterns.ResponseHeaders {
+ redactValues = append(redactValues, fmt.Sprintf("response.headers['%s']", responseHeader))
+ }
+
+ for _, requestBody := range config.RedactPatterns.RequestBody {
+ redactValues = append(redactValues, fmt.Sprintf("request.postData.text.json()...%s", requestBody))
+ }
+ for _, responseBody := range config.RedactPatterns.ResponseBody {
+ redactValues = append(redactValues, fmt.Sprintf("response.content.text.json()...%s", responseBody))
+ }
+
+ for _, requestQueryParams := range config.RedactPatterns.RequestQueryParams {
+ redactValues = append(redactValues, fmt.Sprintf("request.queryString['%s']", requestQueryParams))
+ }
+
+ if len(redactValues) == 0 {
+ return ""
+ }
+
+ return fmt.Sprintf("redact(\"%s\")", strings.Join(redactValues, "\",\""))
+}
+
func (config *TapConfig) Validate() error {
_, compileErr := regexp.Compile(config.PodRegexStr)
if compileErr != nil {
diff --git a/performance_analysis/run_tapper_benchmark.sh b/performance_analysis/run_tapper_benchmark.sh
index d2b7e6ccd..bebcd7825 100755
--- a/performance_analysis/run_tapper_benchmark.sh
+++ b/performance_analysis/run_tapper_benchmark.sh
@@ -57,7 +57,7 @@ log "Writing output to $MIZU_BENCHMARK_OUTPUT_DIR"
cd $MIZU_HOME || exit 1
export HOST_MODE=0
-export SENSITIVE_DATA_FILTERING_OPTIONS='{"EnableRedaction": false}'
+export SENSITIVE_DATA_FILTERING_OPTIONS='{}'
export MIZU_DEBUG_DISABLE_PCAP=false
export MIZU_DEBUG_DISABLE_TCP_REASSEMBLY=false
export MIZU_DEBUG_DISABLE_TCP_STREAM=false
diff --git a/tap/api/options.go b/tap/api/options.go
index 78c9ad095..1f2ce2579 100644
--- a/tap/api/options.go
+++ b/tap/api/options.go
@@ -1,7 +1,5 @@
package api
type TrafficFilteringOptions struct {
- IgnoredUserAgents []string
- PlainTextMaskingRegexes []*SerializableRegexp
- EnableRedaction bool
+ IgnoredUserAgents []string
}
diff --git a/tap/extensions/http/handlers.go b/tap/extensions/http/handlers.go
index 870cf511a..4560e4a5b 100644
--- a/tap/extensions/http/handlers.go
+++ b/tap/extensions/http/handlers.go
@@ -18,10 +18,6 @@ func filterAndEmit(item *api.OutputChannelItem, emitter api.Emitter, options *ap
return
}
- if options.EnableRedaction {
- FilterSensitiveData(item, options)
- }
-
replaceForwardedFor(item)
emitter.Emit(item)
diff --git a/tap/extensions/http/sensitive_data_cleaner.go b/tap/extensions/http/sensitive_data_cleaner.go
index 17b0fb435..0e8885f8e 100644
--- a/tap/extensions/http/sensitive_data_cleaner.go
+++ b/tap/extensions/http/sensitive_data_cleaner.go
@@ -1,30 +1,14 @@
package http
import (
- "bytes"
- "encoding/json"
- "encoding/xml"
- "errors"
- "fmt"
- "io/ioutil"
"net/http"
- "net/url"
"strings"
- "github.com/beevik/etree"
"github.com/up9inc/mizu/tap/api"
)
-const maskedFieldPlaceholderValue = "[REDACTED]"
const userAgent = "user-agent"
-//these values MUST be all lower case and contain no `-` or `_` characters
-var personallyIdentifiableDataFields = []string{"token", "authorization", "authentication", "cookie", "userid", "password",
- "username", "user", "key", "passcode", "pass", "auth", "authtoken", "jwt",
- "bearer", "clientid", "clientsecret", "redirecturi", "phonenumber",
- "zip", "zipcode", "address", "country", "firstname", "lastname",
- "middlename", "fname", "lname", "birthdate"}
-
func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) bool {
if item.Protocol.Name != "http" {
return false
@@ -48,192 +32,3 @@ func IsIgnoredUserAgent(item *api.OutputChannelItem, options *api.TrafficFilteri
return false
}
-
-func FilterSensitiveData(item *api.OutputChannelItem, options *api.TrafficFilteringOptions) {
- request := item.Pair.Request.Payload.(HTTPPayload).Data.(*http.Request)
- response := item.Pair.Response.Payload.(HTTPPayload).Data.(*http.Response)
-
- filterHeaders(&request.Header)
- filterHeaders(&response.Header)
- filterUrl(request.URL)
- filterRequestBody(request, options)
- filterResponseBody(response, options)
-}
-
-func filterRequestBody(request *http.Request, options *api.TrafficFilteringOptions) {
- contenType := getContentTypeHeaderValue(request.Header)
- body, err := ioutil.ReadAll(request.Body)
- if err != nil {
- return
- }
- filteredBody, err := filterHttpBody(body, contenType, options)
- if err == nil {
- request.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
- } else {
- request.Body = ioutil.NopCloser(bytes.NewBuffer(body))
- }
-}
-
-func filterResponseBody(response *http.Response, options *api.TrafficFilteringOptions) {
- contentType := getContentTypeHeaderValue(response.Header)
- body, err := ioutil.ReadAll(response.Body)
- if err != nil {
- return
- }
- filteredBody, err := filterHttpBody(body, contentType, options)
- if err == nil {
- response.Body = ioutil.NopCloser(bytes.NewBuffer(filteredBody))
- } else {
- response.Body = ioutil.NopCloser(bytes.NewBuffer(body))
- }
-}
-
-func filterHeaders(headers *http.Header) {
- for key := range *headers {
- if strings.ToLower(key) == userAgent {
- continue
- }
-
- if strings.ToLower(key) == "cookie" {
- headers.Del(key)
- } else if isFieldNameSensitive(key) {
- headers.Set(key, maskedFieldPlaceholderValue)
- }
- }
-}
-
-func getContentTypeHeaderValue(headers http.Header) string {
- for key := range headers {
- if strings.ToLower(key) == "content-type" {
- return headers.Get(key)
- }
- }
- return ""
-}
-
-func isFieldNameSensitive(fieldName string) bool {
- if fieldName == ":authority" {
- return false
- }
-
- name := strings.ToLower(fieldName)
- name = strings.ReplaceAll(name, "_", "")
- name = strings.ReplaceAll(name, "-", "")
- name = strings.ReplaceAll(name, " ", "")
-
- for _, sensitiveField := range personallyIdentifiableDataFields {
- if strings.Contains(name, sensitiveField) {
- return true
- }
- }
-
- return false
-}
-
-func filterHttpBody(bytes []byte, contentType string, options *api.TrafficFilteringOptions) ([]byte, error) {
- mimeType := strings.Split(contentType, ";")[0]
- switch strings.ToLower(mimeType) {
- case "application/json":
- return filterJsonBody(bytes)
- case "text/html":
- fallthrough
- case "application/xhtml+xml":
- fallthrough
- case "text/xml":
- fallthrough
- case "application/xml":
- return filterXmlEtree(bytes)
- case "text/plain":
- if options != nil && options.PlainTextMaskingRegexes != nil {
- return filterPlainText(bytes, options), nil
- }
- }
- return bytes, nil
-}
-
-func filterPlainText(bytes []byte, options *api.TrafficFilteringOptions) []byte {
- for _, regex := range options.PlainTextMaskingRegexes {
- bytes = regex.ReplaceAll(bytes, []byte(maskedFieldPlaceholderValue))
- }
- return bytes
-}
-
-func filterXmlEtree(bytes []byte) ([]byte, error) {
- if !IsValidXML(bytes) {
- return nil, errors.New("Invalid XML")
- }
- xmlDoc := etree.NewDocument()
- err := xmlDoc.ReadFromBytes(bytes)
- if err != nil {
- return nil, err
- } else {
- filterXmlElement(xmlDoc.Root())
- }
- return xmlDoc.WriteToBytes()
-}
-
-func IsValidXML(data []byte) bool {
- return xml.Unmarshal(data, new(interface{})) == nil
-}
-
-func filterXmlElement(element *etree.Element) {
- for i, attribute := range element.Attr {
- if isFieldNameSensitive(attribute.Key) {
- element.Attr[i].Value = maskedFieldPlaceholderValue
- }
- }
- if element.ChildElements() == nil || len(element.ChildElements()) == 0 {
- if isFieldNameSensitive(element.Tag) {
- element.SetText(maskedFieldPlaceholderValue)
- }
- } else {
- for _, element := range element.ChildElements() {
- filterXmlElement(element)
- }
- }
-}
-
-func filterJsonBody(bytes []byte) ([]byte, error) {
- var bodyJsonMap map[string]interface{}
- err := json.Unmarshal(bytes, &bodyJsonMap)
- if err != nil {
- return nil, err
- }
- filterJsonMap(bodyJsonMap)
- return json.Marshal(bodyJsonMap)
-}
-
-func filterJsonMap(jsonMap map[string]interface{}) {
- for key, value := range jsonMap {
- // Do not replace nil values with maskedFieldPlaceholderValue
- if value == nil {
- continue
- }
-
- nestedMap, isNested := value.(map[string]interface{})
- if isNested {
- filterJsonMap(nestedMap)
- } else {
- if isFieldNameSensitive(key) {
- jsonMap[key] = maskedFieldPlaceholderValue
- }
- }
- }
-}
-
-func filterUrl(url *url.URL) {
- if len(url.RawQuery) > 0 {
- newQueryArgs := make([]string, 0)
- for urlQueryParamName, urlQueryParamValues := range url.Query() {
- newValues := urlQueryParamValues
- if isFieldNameSensitive(urlQueryParamName) {
- newValues = []string{maskedFieldPlaceholderValue}
- }
- for _, paramValue := range newValues {
- newQueryArgs = append(newQueryArgs, fmt.Sprintf("%s=%s", urlQueryParamName, paramValue))
- }
- }
-
- url.RawQuery = strings.Join(newQueryArgs, "&")
- }
-}
--
GitLab