Remove fetch command (and direction) (#264)

acceptanceTests/tap_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"os/exec"
 	"strings"
@@ -12,7 +11,7 @@ import (
 	"time"
 )
 
-func TestTapAndFetch(t *testing.T) {
+func TestTap(t *testing.T) {
 	if testing.Short() {
 		t.Skip("ignored acceptance test")
 	}
@@ -93,37 +92,6 @@ func TestTapAndFetch(t *testing.T) {
 				t.Errorf("%v", err)
 				return
 			}
-
-			fetchCmdArgs := getDefaultFetchCommandArgs()
-			fetchCmd := exec.Command(cliPath, fetchCmdArgs...)
-			t.Logf("running command: %v", fetchCmd.String())
-
-			if err := fetchCmd.Start(); err != nil {
-				t.Errorf("failed to start fetch command, err: %v", err)
-				return
-			}
-
-			harCheckFunc := func() error {
-				harBytes, readFileErr := ioutil.ReadFile("./unknown_source.har")
-				if readFileErr != nil {
-					return fmt.Errorf("failed to read har file, err: %v", readFileErr)
-				}
-
-				harEntries, err := getEntriesFromHarBytes(harBytes)
-				if err != nil {
-					return fmt.Errorf("failed to get entries from har, err: %v", err)
-				}
-
-				if len(harEntries) == 0 {
-					return fmt.Errorf("unexpected har entries result - Expected more than 0 entries")
-				}
-
-				return nil
-			}
-			if err := retriesExecute(shortRetriesCount, harCheckFunc); err != nil {
-				t.Errorf("%v", err)
-				return
-			}
 		})
 	}
 }

acceptanceTests/testsUtils.go

@@ -76,13 +76,6 @@ func getDefaultTapNamespace() []string {
 	return []string{"-n", "mizu-tests"}
 }
 
-func getDefaultFetchCommandArgs() []string {
-	fetchCommand := "fetch"
-	defaultCmdArgs := getDefaultCommandArgs()
-
-	return append([]string{fetchCommand}, defaultCmdArgs...)
-}
-
 func getDefaultConfigCommandArgs() []string {
 	configCommand := "config"
 	defaultCmdArgs := getDefaultCommandArgs()
@@ -179,19 +172,6 @@ func cleanupCommand(cmd *exec.Cmd) error {
 	return nil
 }
 
-func getEntriesFromHarBytes(harBytes []byte) ([]interface{}, error) {
-	harInterface, convertErr := jsonBytesToInterface(harBytes)
-	if convertErr != nil {
-		return nil, convertErr
-	}
-
-	har := harInterface.(map[string]interface{})
-	harLog := har["log"].(map[string]interface{})
-	harEntries := harLog["entries"].([]interface{})
-
-	return harEntries, nil
-}
-
 func getPods(tapStatusInterface interface{}) ([]map[string]interface{}, error) {
 	tapStatus := tapStatusInterface.(map[string]interface{})
 	podsInterface := tapStatus["pods"].([]interface{})

agent/README.md

 # mizu agent
 Agent for MIZU (API server and tapper)
 Basic APIs:
-* /fetch - retrieve traffic data
 * /stats - retrieve statistics of collected data
 * /viewer - web ui
 

agent/pkg/controllers/entries_controller.go

@@ -3,6 +3,7 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/google/martian/har"
 	"mizuserver/pkg/database"
 	"mizuserver/pkg/models"
 	"mizuserver/pkg/providers"
@@ -10,11 +11,9 @@ import (
 	"mizuserver/pkg/utils"
 	"mizuserver/pkg/validation"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/gin-gonic/gin"
-	"github.com/google/martian/har"
 	"github.com/romana/rlog"
 
 	tapApi "github.com/up9inc/mizu/tap/api"
@@ -64,93 +63,6 @@ func GetEntries(c *gin.Context) {
 	c.JSON(http.StatusOK, baseEntries)
 }
 
-func GetHARs(c *gin.Context) {
-	entriesFilter := &models.HarFetchRequestQuery{}
-	order := database.OrderDesc
-	if err := c.BindQuery(entriesFilter); err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-	err := validation.Validate(entriesFilter)
-	if err != nil {
-		c.JSON(http.StatusBadRequest, err)
-	}
-
-	var timestampFrom, timestampTo int64
-
-	if entriesFilter.From < 0 {
-		timestampFrom = 0
-	} else {
-		timestampFrom = entriesFilter.From
-	}
-	if entriesFilter.To <= 0 {
-		timestampTo = time.Now().UnixNano() / int64(time.Millisecond)
-	} else {
-		timestampTo = entriesFilter.To
-	}
-
-	var entries []tapApi.MizuEntry
-	database.GetEntriesTable().
-		Where(fmt.Sprintf("timestamp BETWEEN %v AND %v", timestampFrom, timestampTo)).
-		Order(fmt.Sprintf("timestamp %s", order)).
-		Find(&entries)
-
-	if len(entries) > 0 {
-		// the entries always order from oldest to newest - we should reverse
-		utils.ReverseSlice(entries)
-	}
-
-	harsObject := map[string]*models.ExtendedHAR{}
-
-	for _, entryData := range entries {
-		var harEntry har.Entry
-		_ = json.Unmarshal([]byte(entryData.Entry), &harEntry)
-		if entryData.ResolvedDestination != "" {
-			harEntry.Request.URL = utils.SetHostname(harEntry.Request.URL, entryData.ResolvedDestination)
-		}
-
-		var fileName string
-		sourceOfEntry := entryData.ResolvedSource
-		if sourceOfEntry != "" {
-			// naively assumes the proper service source is http
-			sourceOfEntry = fmt.Sprintf("http://%s", sourceOfEntry)
-			//replace / from the file name because they end up creating a corrupted folder
-			fileName = fmt.Sprintf("%s.har", strings.ReplaceAll(sourceOfEntry, "/", "_"))
-		} else {
-			fileName = "unknown_source.har"
-		}
-		if harOfSource, ok := harsObject[fileName]; ok {
-			harOfSource.Log.Entries = append(harOfSource.Log.Entries, &harEntry)
-		} else {
-			var entriesHar []*har.Entry
-			entriesHar = append(entriesHar, &harEntry)
-			harsObject[fileName] = &models.ExtendedHAR{
-				Log: &models.ExtendedLog{
-					Version: "1.2",
-					Creator: &models.ExtendedCreator{
-						Creator: &har.Creator{
-							Name:    "mizu",
-							Version: "0.0.2",
-						},
-					},
-					Entries: entriesHar,
-				},
-			}
-			// leave undefined when no source is present, otherwise modeler assumes source is empty string ""
-			if sourceOfEntry != "" {
-				harsObject[fileName].Log.Creator.Source = &sourceOfEntry
-			}
-		}
-	}
-
-	retObj := map[string][]byte{}
-	for k, v := range harsObject {
-		bytesData, _ := json.Marshal(v)
-		retObj[k] = bytesData
-	}
-	buffer := utils.ZipData(retObj)
-	c.Data(http.StatusOK, "application/octet-stream", buffer.Bytes())
-}
-
 func UploadEntries(c *gin.Context) {
 	rlog.Infof("Upload entries - started\n")
 

agent/pkg/routes/entries_routes.go

@@ -15,8 +15,6 @@ func EntriesRoutes(ginApp *gin.Engine) {
 	routeGroup.GET("/uploadEntries", controllers.UploadEntries)
 	routeGroup.GET("/resolving", controllers.GetCurrentResolvingInformation)
 
-	routeGroup.GET("/har", controllers.GetHARs)
-
 	routeGroup.GET("/resetDB", controllers.DeleteAllEntries)     // get single (full) entry
 	routeGroup.GET("/generalStats", controllers.GetGeneralStats) // get general stats about entries in DB
 

cli/apiserver/provider.go

 package apiserver
 
 import (
-	"archive/zip"
 	"bytes"
 	"encoding/json"
 	"fmt"
@@ -131,28 +130,6 @@ func (provider *apiServerProvider) GetGeneralStats() (map[string]interface{}, er
 	return generalStats, nil
 }
 
-func (provider *apiServerProvider) GetHars(fromTimestamp int, toTimestamp int) (*zip.Reader, error) {
-	if !provider.isReady {
-		return nil, fmt.Errorf("trying to reach api server when not initialized yet")
-	}
-	resp, err := http.Get(fmt.Sprintf("%s/api/har?from=%v&to=%v", provider.url, fromTimestamp, toTimestamp))
-	if err != nil {
-		return nil, fmt.Errorf("failed getting har from api server %w", err)
-	}
-
-	defer func() { _ = resp.Body.Close() }()
-
-	body, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed reading hars %w", err)
-	}
-
-	zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body)))
-	if err != nil {
-		return nil, fmt.Errorf("failed craeting zip reader %w", err)
-	}
-	return zipReader, nil
-}
 
 func (provider *apiServerProvider) GetVersion() (string, error) {
 	if !provider.isReady {

cli/cmd/fetch.go

-package cmd
-
-import (
-	"github.com/creasty/defaults"
-	"github.com/spf13/cobra"
-	"github.com/up9inc/mizu/cli/apiserver"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/logger"
-	"github.com/up9inc/mizu/cli/mizu/version"
-	"github.com/up9inc/mizu/cli/telemetry"
-	"github.com/up9inc/mizu/cli/uiUtils"
-)
-
-var fetchCmd = &cobra.Command{
-	Use:   "fetch",
-	Short: "Download recorded traffic to files",
-	RunE: func(cmd *cobra.Command, args []string) error {
-		go telemetry.ReportRun("fetch", config.Config.Fetch)
-
-		if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil {
-			logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, make sure one running")
-			return nil
-		}
-
-		if isCompatible, err := version.CheckVersionCompatibility(); err != nil {
-			return err
-		} else if !isCompatible {
-			return nil
-		}
-		RunMizuFetch()
-		return nil
-	},
-}
-
-func init() {
-	rootCmd.AddCommand(fetchCmd)
-
-	defaultFetchConfig := configStructs.FetchConfig{}
-	defaults.Set(&defaultFetchConfig)
-
-	fetchCmd.Flags().StringP(configStructs.DirectoryFetchName, "d", defaultFetchConfig.Directory, "Provide a custom directory for fetched entries")
-	fetchCmd.Flags().Int(configStructs.FromTimestampFetchName, defaultFetchConfig.FromTimestamp, "Custom start timestamp for fetched entries")
-	fetchCmd.Flags().Int(configStructs.ToTimestampFetchName, defaultFetchConfig.ToTimestamp, "Custom end timestamp fetched entries")
-	fetchCmd.Flags().Uint16P(configStructs.GuiPortFetchName, "p", defaultFetchConfig.GuiPort, "Provide a custom port for the web interface webserver")
-}

cli/cmd/fetchRunner.go

-package cmd
-
-import (
-	"github.com/up9inc/mizu/cli/apiserver"
-	"github.com/up9inc/mizu/cli/config"
-	"github.com/up9inc/mizu/cli/logger"
-	"github.com/up9inc/mizu/cli/mizu/fsUtils"
-	"github.com/up9inc/mizu/cli/uiUtils"
-)
-
-func RunMizuFetch() {
-	if err := apiserver.Provider.InitAndTestConnection(GetApiServerUrl()); err != nil {
-		logger.Log.Errorf(uiUtils.Error, "Couldn't connect to API server, check logs")
-	}
-
-	zipReader, err := apiserver.Provider.GetHars(config.Config.Fetch.FromTimestamp, config.Config.Fetch.ToTimestamp)
-	if err != nil {
-		logger.Log.Errorf("Failed fetch data from API server %v", err)
-		return
-	}
-
-	if err := fsUtils.Unzip(zipReader, config.Config.Fetch.Directory); err != nil {
-		logger.Log.Debugf("[ERROR] failed unzip %v", err)
-	}
-}

cli/cmd/tap.go

@@ -64,7 +64,6 @@ func init() {
 	tapCmd.Flags().StringSliceP(configStructs.PlainTextFilterRegexesTapName, "r", defaultTapConfig.PlainTextFilterRegexes, "List of regex expressions that are used to filter matching values from text/plain http bodies")
 	tapCmd.Flags().Bool(configStructs.DisableRedactionTapName, defaultTapConfig.DisableRedaction, "Disables redaction of potentially sensitive request/response headers and body values")
 	tapCmd.Flags().String(configStructs.HumanMaxEntriesDBSizeTapName, defaultTapConfig.HumanMaxEntriesDBSize, "Override the default max entries db size")
-	tapCmd.Flags().String(configStructs.DirectionTapName, defaultTapConfig.Direction, "Record traffic that goes in this direction (relative to the tapped pod): in/any")
 	tapCmd.Flags().Bool(configStructs.DryRunTapName, defaultTapConfig.DryRun, "Preview of all pods matching the regex, without tapping them")
 	tapCmd.Flags().String(configStructs.EnforcePolicyFile, defaultTapConfig.EnforcePolicyFile, "Yaml file with policy rules")
 }

cli/config/configStruct.go

@@ -18,7 +18,6 @@ const (
 
 type ConfigStruct struct {
 	Tap                    configStructs.TapConfig     `yaml:"tap"`
-	Fetch                  configStructs.FetchConfig   `yaml:"fetch"`
 	Version                configStructs.VersionConfig `yaml:"version"`
 	View                   configStructs.ViewConfig    `yaml:"view"`
 	Logs                   configStructs.LogsConfig    `yaml:"logs"`

cli/config/configStructs/fetchConfig.go

-package configStructs
-
-const (
-	DirectoryFetchName     = "directory"
-	FromTimestampFetchName = "from"
-	ToTimestampFetchName   = "to"
-	GuiPortFetchName       = "gui-port"
-)
-
-type FetchConfig struct {
-	Directory     string `yaml:"directory" default:"."`
-	FromTimestamp int    `yaml:"from" default:"0"`
-	ToTimestamp   int    `yaml:"to" default:"0"`
-	GuiPort       uint16 `yaml:"gui-port" default:"8899"`
-}

cli/config/configStructs/tapConfig.go

@@ -3,10 +3,8 @@ package configStructs
 import (
 	"errors"
 	"fmt"
-	"regexp"
-	"strings"
-
 	"github.com/up9inc/mizu/shared/units"
+	"regexp"
 )
 
 const (
@@ -17,7 +15,6 @@ const (
 	PlainTextFilterRegexesTapName = "regex-masking"
 	DisableRedactionTapName       = "no-redact"
 	HumanMaxEntriesDBSizeTapName  = "max-entries-db-size"
-	DirectionTapName              = "direction"
 	DryRunTapName                 = "dry-run"
 	EnforcePolicyFile             = "test-rules"
 )
@@ -34,7 +31,6 @@ type TapConfig struct {
 	HealthChecksUserAgentHeaders []string  `yaml:"ignored-user-agents"`
 	DisableRedaction             bool      `yaml:"no-redact" default:"false"`
 	HumanMaxEntriesDBSize        string    `yaml:"max-entries-db-size" default:"200MB"`
-	Direction                    string    `yaml:"direction" default:"in"`
 	DryRun                       bool      `yaml:"dry-run" default:"false"`
 	EnforcePolicyFile            string    `yaml:"test-rules"`
 	ApiServerResources           Resources `yaml:"api-server-resources"`
@@ -69,10 +65,5 @@ func (config *TapConfig) Validate() error {
 		return errors.New(fmt.Sprintf("Could not parse --%s value %s", HumanMaxEntriesDBSizeTapName, config.HumanMaxEntriesDBSize))
 	}
 
-	directionLowerCase := strings.ToLower(config.Direction)
-	if directionLowerCase != "any" && directionLowerCase != "in" {
-		return errors.New(fmt.Sprintf("%s is not a valid value for flag --%s. Acceptable values are in/any.", config.Direction, DirectionTapName))
-	}
-
 	return nil
 }