Refactor Mizu, define an extension API and add new protocols: AMQP, Kafka (#224)

* Separate HTTP related code into `extensions/http` as a Go plugin

* Move `extensions` folder into `tap` folder

* Move HTTP files into `tap/extensions/lib` for now

* Replace `orcaman/concurrent-map` with `sync.Map`

* Remove `grpc_assembler.go`

* Remove `github.com/up9inc/mizu/tap/extensions/http/lib`

* Add a build script to automatically build extensions from a known path and load them

* Start to define the extension API

* Implement the `run()` function for the TCP stream

* Add support of defining multiple ports to the extension API

* Set the extension name inside the extension

* Declare the `Dissect` function in the extension API

* Dissect HTTP request from inside the HTTP extension

* Make the distinction of outbound and inbound ports

* Dissect HTTP response from inside the HTTP extension

* Bring back the HTTP request-response pair matcher

* Return a `*api.RequestResponsePair` ...

cli/config/configStructs/tapConfig.go

@@ -53,15 +53,6 @@ func (config *TapConfig) PodRegex() *regexp.Regexp {
 	return podRegex
 }
 
-func (config *TapConfig) TapOutgoing() bool {
-	directionLowerCase := strings.ToLower(config.Direction)
-	if directionLowerCase == "any" {
-		return true
-	}
-
-	return false
-}
-
 func (config *TapConfig) MaxEntriesDBSizeBytes() int64 {
 	maxEntriesDBSizeBytes, _ := units.HumanReadableToBytes(config.HumanMaxEntriesDBSize)
 	return maxEntriesDBSizeBytes

cli/kubernetes/provider.go

@@ -7,15 +7,17 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/up9inc/mizu/cli/config/configStructs"
-	"github.com/up9inc/mizu/cli/logger"
 	"path/filepath"
 	"regexp"
 	"strconv"
 
+	"github.com/up9inc/mizu/cli/config/configStructs"
+	"github.com/up9inc/mizu/cli/logger"
+
+	"io"
+
 	"github.com/up9inc/mizu/cli/mizu"
 	"github.com/up9inc/mizu/shared"
-	"io"
 	core "k8s.io/api/core/v1"
 	rbac "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -55,21 +57,21 @@ func NewProvider(kubeConfigPath string) (*Provider, error) {
 	restClientConfig, err := kubernetesConfig.ClientConfig()
 	if err != nil {
 		if clientcmd.IsEmptyConfig(err) {
-			return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n" +
+			return nil, fmt.Errorf("couldn't find the kube config file, or file is empty (%s)\n"+
 				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
 		if clientcmd.IsConfigurationInvalid(err) {
-			return nil, fmt.Errorf("invalid kube config file (%s)\n" +
+			return nil, fmt.Errorf("invalid kube config file (%s)\n"+
 				"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 		}
 
-		return nil, fmt.Errorf("error while using kube config (%s)\n" +
+		return nil, fmt.Errorf("error while using kube config (%s)\n"+
 			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
 
 	clientSet, err := getClientSet(restClientConfig)
 	if err != nil {
-		return nil, fmt.Errorf("error while using kube config (%s)\n" +
+		return nil, fmt.Errorf("error while using kube config (%s)\n"+
 			"you can set alternative kube config file path by adding the kube-config-path field to the mizu config file, err:  %w", kubeConfigPath, err)
 	}
 
@@ -573,11 +575,11 @@ func (provider *Provider) CreateConfigMap(ctx context.Context, namespace string,
 	return nil
 }
 
-func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, tapOutgoing bool, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error {
-	logger.Log.Debugf("Applying %d tapper deamonsets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
+func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespace string, daemonSetName string, podImage string, tapperPodName string, apiServerPodIp string, nodeToTappedPodIPMap map[string][]string, serviceAccountName string, resources configStructs.Resources, imagePullPolicy core.PullPolicy) error {
+	logger.Log.Debugf("Applying %d tapper daemon sets, ns: %s, daemonSetName: %s, podImage: %s, tapperPodName: %s", len(nodeToTappedPodIPMap), namespace, daemonSetName, podImage, tapperPodName)
 
 	if len(nodeToTappedPodIPMap) == 0 {
-		return fmt.Errorf("Daemon set %s must tap at least 1 pod", daemonSetName)
+		return fmt.Errorf("daemon set %s must tap at least 1 pod", daemonSetName)
 	}
 
 	nodeToTappedPodIPMapJsonStr, err := json.Marshal(nodeToTappedPodIPMap)
@@ -590,9 +592,7 @@ func (provider *Provider) ApplyMizuTapperDaemonSet(ctx context.Context, namespac
 		"-i", "any",
 		"--tap",
 		"--api-server-address", fmt.Sprintf("ws://%s/wsTapper", apiServerPodIp),
-	}
-	if tapOutgoing {
-		mizuCmd = append(mizuCmd, "--anydirection")
+		"--nodefrag",
 	}
 
 	agentContainer := applyconfcore.Container()

shared/models.go

@@ -5,7 +5,7 @@ import (
 	"io/ioutil"
 	"strings"
 
-	yaml "gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v3"
 )
 
 type WebSocketMessageType string

tap/api/api.go

+package api
+
+import (
+	"bufio"
+	"plugin"
+	"sync"
+	"time"
+)
+
+type Protocol struct {
+	Name            string   `json:"name"`
+	LongName        string   `json:"long_name"`
+	Abbreviation    string   `json:"abbreviation"`
+	Version         string   `json:"version"`
+	BackgroundColor string   `json:"background_color"`
+	ForegroundColor string   `json:"foreground_color"`
+	FontSize        int8     `json:"font_size"`
+	ReferenceLink   string   `json:"reference_link"`
+	Ports           []string `json:"ports"`
+	Priority        uint8    `json:"priority"`
+}
+
+type Extension struct {
+	Protocol   Protocol
+	Path       string
+	Plug       *plugin.Plugin
+	Dissector  Dissector
+	MatcherMap *sync.Map
+}
+
+type ConnectionInfo struct {
+	ClientIP   string
+	ClientPort string
+	ServerIP   string
+	ServerPort string
+	IsOutgoing bool
+}
+
+type TcpID struct {
+	SrcIP   string
+	DstIP   string
+	SrcPort string
+	DstPort string
+	Ident   string
+}
+
+type CounterPair struct {
+	Request  uint
+	Response uint
+}
+
+type GenericMessage struct {
+	IsRequest   bool        `json:"is_request"`
+	CaptureTime time.Time   `json:"capture_time"`
+	Payload     interface{} `json:"payload"`
+}
+
+type RequestResponsePair struct {
+	Request  GenericMessage `json:"request"`
+	Response GenericMessage `json:"response"`
+}
+
+// `Protocol` is modified in the later stages of data propagation. Therefore it's not a pointer.
+type OutputChannelItem struct {
+	Protocol       Protocol
+	Timestamp      int64
+	ConnectionInfo *ConnectionInfo
+	Pair           *RequestResponsePair
+}
+
+type Dissector interface {
+	Register(*Extension)
+	Ping()
+	Dissect(b *bufio.Reader, isClient bool, tcpID *TcpID, counterPair *CounterPair, emitter Emitter) error
+	Analyze(item *OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *MizuEntry
+	Summarize(entry *MizuEntry) *BaseEntryDetails
+	Represent(entry *MizuEntry) (Protocol, []byte, error)
+}
+
+type Emitting struct {
+	OutputChannel chan *OutputChannelItem
+}
+
+type Emitter interface {
+	Emit(item *OutputChannelItem)
+}
+
+func (e *Emitting) Emit(item *OutputChannelItem) {
+	e.OutputChannel <- item
+}
+
+type MizuEntry struct {
+	ID                  uint `gorm:"primarykey"`
+	CreatedAt           time.Time
+	UpdatedAt           time.Time
+	ProtocolName        string `json:"protocol_key" gorm:"column:protocolKey"`
+	ProtocolVersion     string `json:"protocol_version" gorm:"column:protocolVersion"`
+	Entry               string `json:"entry,omitempty" gorm:"column:entry"`
+	EntryId             string `json:"entryId" gorm:"column:entryId"`
+	Url                 string `json:"url" gorm:"column:url"`
+	Method              string `json:"method" gorm:"column:method"`
+	Status              int    `json:"status" gorm:"column:status"`
+	RequestSenderIp     string `json:"requestSenderIp" gorm:"column:requestSenderIp"`
+	Service             string `json:"service" gorm:"column:service"`
+	Timestamp           int64  `json:"timestamp" gorm:"column:timestamp"`
+	Path                string `json:"path" gorm:"column:path"`
+	ResolvedSource      string `json:"resolvedSource,omitempty" gorm:"column:resolvedSource"`
+	ResolvedDestination string `json:"resolvedDestination,omitempty" gorm:"column:resolvedDestination"`
+	SourceIp            string `json:"sourceIp,omitempty" gorm:"column:sourceIp"`
+	DestinationIp       string `json:"destinationIp,omitempty" gorm:"column:destinationIp"`
+	SourcePort          string `json:"sourcePort,omitempty" gorm:"column:sourcePort"`
+	DestinationPort     string `json:"destinationPort,omitempty" gorm:"column:destinationPort"`
+	IsOutgoing          bool   `json:"isOutgoing,omitempty" gorm:"column:isOutgoing"`
+	EstimatedSizeBytes  int    `json:"-" gorm:"column:estimatedSizeBytes"`
+}
+
+type MizuEntryWrapper struct {
+	Protocol       Protocol  `json:"protocol"`
+	Representation string    `json:"representation"`
+	Data           MizuEntry `json:"data"`
+}
+
+type BaseEntryDetails struct {
+	Id              string          `json:"id,omitempty"`
+	Protocol        Protocol        `json:"protocol,omitempty"`
+	Url             string          `json:"url,omitempty"`
+	RequestSenderIp string          `json:"request_sender_ip,omitempty"`
+	Service         string          `json:"service,omitempty"`
+	Summary         string          `json:"summary,omitempty"`
+	StatusCode      int             `json:"status_code"`
+	Method          string          `json:"method,omitempty"`
+	Timestamp       int64           `json:"timestamp,omitempty"`
+	SourceIp        string          `json:"source_ip,omitempty"`
+	DestinationIp   string          `json:"destination_ip,omitempty"`
+	SourcePort      string          `json:"source_port,omitempty"`
+	DestinationPort string          `json:"destination_port,omitempty"`
+	IsOutgoing      bool            `json:"isOutgoing,omitempty"`
+	Latency         int64           `json:"latency,omitempty"`
+	Rules           ApplicableRules `json:"rules,omitempty"`
+}
+
+type ApplicableRules struct {
+	Latency       int64 `json:"latency,omitempty"`
+	Status        bool  `json:"status,omitempty"`
+	NumberOfRules int   `json:"numberOfRules,omitempty"`
+}
+
+type DataUnmarshaler interface {
+	UnmarshalData(*MizuEntry) error
+}
+
+func (bed *BaseEntryDetails) UnmarshalData(entry *MizuEntry) error {
+	entryUrl := entry.Url
+	service := entry.Service
+	bed.Id = entry.EntryId
+	bed.Url = entryUrl
+	bed.Service = service
+	bed.Summary = entry.Path
+	bed.StatusCode = entry.Status
+	bed.Method = entry.Method
+	bed.Timestamp = entry.Timestamp
+	bed.RequestSenderIp = entry.RequestSenderIp
+	bed.IsOutgoing = entry.IsOutgoing
+	return nil
+}

tap/api/go.mod

+module github.com/up9inc/mizu/tap/api
+
+go 1.16

tap/cleaner.go

 package tap
 
 import (
-	"github.com/romana/rlog"
 	"sync"
 	"time"
 
 	"github.com/google/gopacket/reassembly"
+	"github.com/romana/rlog"
+	"github.com/up9inc/mizu/tap/api"
 )
 
 type CleanerStats struct {
@@ -17,7 +18,6 @@ type CleanerStats struct {
 type Cleaner struct {
 	assembler         *reassembly.Assembler
 	assemblerMutex    *sync.Mutex
-	matcher           *requestResponseMatcher
 	cleanPeriod       time.Duration
 	connectionTimeout time.Duration
 	stats             CleanerStats
@@ -32,13 +32,15 @@ func (cl *Cleaner) clean() {
 	flushed, closed := cl.assembler.FlushCloseOlderThan(startCleanTime.Add(-cl.connectionTimeout))
 	cl.assemblerMutex.Unlock()
 
-	deleted := cl.matcher.deleteOlderThan(startCleanTime.Add(-cl.connectionTimeout))
+	for _, extension := range extensions {
+		deleted := deleteOlderThan(extension.MatcherMap, startCleanTime.Add(-cl.connectionTimeout))
+		cl.stats.deleted += deleted
+	}
 
 	cl.statsMutex.Lock()
 	rlog.Debugf("Assembler Stats after cleaning %s", cl.assembler.Dump())
 	cl.stats.flushed += flushed
 	cl.stats.closed += closed
-	cl.stats.deleted += deleted
 	cl.statsMutex.Unlock()
 }
 
@@ -70,3 +72,25 @@ func (cl *Cleaner) dumpStats() CleanerStats {
 
 	return stats
 }
+
+func deleteOlderThan(matcherMap *sync.Map, t time.Time) int {
+	numDeleted := 0
+
+	if matcherMap == nil {
+		return numDeleted
+	}
+
+	matcherMap.Range(func(key interface{}, value interface{}) bool {
+		message, _ := value.(*api.GenericMessage)
+		// TODO: Investigate the reason why `request` is `nil` in some rare occasion
+		if message != nil {
+			if message.CaptureTime.Before(t) {
+				matcherMap.Delete(key)
+				numDeleted++
+			}
+		}
+		return true
+	})
+
+	return numDeleted
+}

tap/extensions/amqp/go.mod

+module github.com/up9inc/mizu/tap/extensions/amqp
+
+go 1.16
+
+require (
+    github.com/up9inc/mizu/tap/api v0.0.0
+)
+
+replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/amqp/helpers.go

+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+var connectionMethodMap = map[int]string{
+	10: "connection start",
+	11: "connection start-ok",
+	20: "connection secure",
+	21: "connection secure-ok",
+	30: "connection tune",
+	31: "connection tune-ok",
+	40: "connection open",
+	41: "connection open-ok",
+	50: "connection close",
+	51: "connection close-ok",
+	60: "connection blocked",
+	61: "connection unblocked",
+}
+
+var channelMethodMap = map[int]string{
+	10: "channel open",
+	11: "channel open-ok",
+	20: "channel flow",
+	21: "channel flow-ok",
+	40: "channel close",
+	41: "channel close-ok",
+}
+
+var exchangeMethodMap = map[int]string{
+	10: "exchange declare",
+	11: "exchange declare-ok",
+	20: "exchange delete",
+	21: "exchange delete-ok",
+	30: "exchange bind",
+	31: "exchange bind-ok",
+	40: "exchange unbind",
+	51: "exchange unbind-ok",
+}
+
+var queueMethodMap = map[int]string{
+	10: "queue declare",
+	11: "queue declare-ok",
+	20: "queue bind",
+	21: "queue bind-ok",
+	50: "queue unbind",
+	51: "queue unbind-ok",
+	30: "queue purge",
+	31: "queue purge-ok",
+	40: "queue delete",
+	41: "queue delete-ok",
+}
+
+var basicMethodMap = map[int]string{
+	10:  "basic qos",
+	11:  "basic qos-ok",
+	20:  "basic consume",
+	21:  "basic consume-ok",
+	30:  "basic cancel",
+	31:  "basic cancel-ok",
+	40:  "basic publish",
+	50:  "basic return",
+	60:  "basic deliver",
+	70:  "basic get",
+	71:  "basic get-ok",
+	72:  "basic get-empty",
+	80:  "basic ack",
+	90:  "basic reject",
+	100: "basic recover-async",
+	110: "basic recover",
+	111: "basic recover-ok",
+	120: "basic nack",
+}
+
+var txMethodMap = map[int]string{
+	10: "tx select",
+	11: "tx select-ok",
+	20: "tx commit",
+	21: "tx commit-ok",
+	30: "tx rollback",
+	31: "tx rollback-ok",
+}
+
+type AMQPWrapper struct {
+	Method  string      `json:"method"`
+	Url     string      `json:"url"`
+	Details interface{} `json:"details"`
+}
+
+func emitAMQP(event interface{}, _type string, method string, connectionInfo *api.ConnectionInfo, emitter api.Emitter) {
+	request := &api.GenericMessage{
+		IsRequest:   true,
+		CaptureTime: time.Now(),
+		Payload: AMQPPayload{
+			Data: &AMQPWrapper{
+				Method:  method,
+				Url:     "",
+				Details: event,
+			},
+		},
+	}
+	item := &api.OutputChannelItem{
+		Protocol:       protocol,
+		Timestamp:      time.Now().UnixNano() / int64(time.Millisecond),
+		ConnectionInfo: connectionInfo,
+		Pair: &api.RequestResponsePair{
+			Request:  *request,
+			Response: api.GenericMessage{},
+		},
+	}
+	emitter.Emit(item)
+}
+
+func representProperties(properties map[string]interface{}, rep []interface{}) ([]interface{}, string, string) {
+	contentType := ""
+	contentEncoding := ""
+	deliveryMode := ""
+	priority := ""
+	correlationId := ""
+	replyTo := ""
+	expiration := ""
+	messageId := ""
+	timestamp := ""
+	_type := ""
+	userId := ""
+	appId := ""
+
+	if properties["ContentType"] != nil {
+		contentType = properties["ContentType"].(string)
+	}
+	if properties["ContentEncoding"] != nil {
+		contentEncoding = properties["ContentEncoding"].(string)
+	}
+	if properties["Delivery Mode"] != nil {
+		deliveryMode = fmt.Sprintf("%g", properties["DeliveryMode"].(float64))
+	}
+	if properties["Priority"] != nil {
+		priority = fmt.Sprintf("%g", properties["Priority"].(float64))
+	}
+	if properties["CorrelationId"] != nil {
+		correlationId = properties["CorrelationId"].(string)
+	}
+	if properties["ReplyTo"] != nil {
+		replyTo = properties["ReplyTo"].(string)
+	}
+	if properties["Expiration"] != nil {
+		expiration = properties["Expiration"].(string)
+	}
+	if properties["MessageId"] != nil {
+		messageId = properties["MessageId"].(string)
+	}
+	if properties["Timestamp"] != nil {
+		timestamp = properties["Timestamp"].(string)
+	}
+	if properties["Type"] != nil {
+		_type = properties["Type"].(string)
+	}
+	if properties["UserId"] != nil {
+		userId = properties["UserId"].(string)
+	}
+	if properties["AppId"] != nil {
+		appId = properties["AppId"].(string)
+	}
+
+	props, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Content Type",
+			"value": contentType,
+		},
+		{
+			"name":  "Content Encoding",
+			"value": contentEncoding,
+		},
+		{
+			"name":  "Delivery Mode",
+			"value": deliveryMode,
+		},
+		{
+			"name":  "Priority",
+			"value": priority,
+		},
+		{
+			"name":  "Correlation ID",
+			"value": correlationId,
+		},
+		{
+			"name":  "Reply To",
+			"value": replyTo,
+		},
+		{
+			"name":  "Expiration",
+			"value": expiration,
+		},
+		{
+			"name":  "Message ID",
+			"value": messageId,
+		},
+		{
+			"name":  "Timestamp",
+			"value": timestamp,
+		},
+		{
+			"name":  "Type",
+			"value": _type,
+		},
+		{
+			"name":  "User ID",
+			"value": userId,
+		},
+		{
+			"name":  "App ID",
+			"value": appId,
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Properties",
+		"data":  string(props),
+	})
+
+	return rep, contentType, contentEncoding
+}
+
+func representBasicPublish(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Exchange",
+			"value": event["Exchange"].(string),
+		},
+		{
+			"name":  "Routing Key",
+			"value": event["RoutingKey"].(string),
+		},
+		{
+			"name":  "Mandatory",
+			"value": strconv.FormatBool(event["Mandatory"].(bool)),
+		},
+		{
+			"name":  "Immediate",
+			"value": strconv.FormatBool(event["Immediate"].(bool)),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	properties := event["Properties"].(map[string]interface{})
+	rep, contentType, _ := representProperties(properties, rep)
+
+	if properties["Headers"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range properties["Headers"].(map[string]interface{}) {
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": value.(string),
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Headers",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	if event["Body"] != nil {
+		rep = append(rep, map[string]string{
+			"type":      "body",
+			"title":     "Body",
+			"encoding":  "base64",
+			"mime_type": contentType,
+			"data":      event["Body"].(string),
+		})
+	}
+
+	return rep
+}
+
+func representBasicDeliver(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	consumerTag := ""
+	deliveryTag := ""
+	redelivered := ""
+
+	if event["ConsumerTag"] != nil {
+		consumerTag = event["ConsumerTag"].(string)
+	}
+	if event["DeliveryTag"] != nil {
+		deliveryTag = fmt.Sprintf("%g", event["DeliveryTag"].(float64))
+	}
+	if event["Redelivered"] != nil {
+		redelivered = strconv.FormatBool(event["Redelivered"].(bool))
+	}
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Consumer Tag",
+			"value": consumerTag,
+		},
+		{
+			"name":  "Delivery Tag",
+			"value": deliveryTag,
+		},
+		{
+			"name":  "Redelivered",
+			"value": redelivered,
+		},
+		{
+			"name":  "Exchange",
+			"value": event["Exchange"].(string),
+		},
+		{
+			"name":  "Routing Key",
+			"value": event["RoutingKey"].(string),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	properties := event["Properties"].(map[string]interface{})
+	rep, contentType, _ := representProperties(properties, rep)
+
+	if properties["Headers"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range properties["Headers"].(map[string]interface{}) {
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": value.(string),
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Headers",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	if event["Body"] != nil {
+		rep = append(rep, map[string]string{
+			"type":      "body",
+			"title":     "Body",
+			"encoding":  "base64",
+			"mime_type": contentType,
+			"data":      event["Body"].(string),
+		})
+	}
+
+	return rep
+}
+
+func representQueueDeclare(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Queue",
+			"value": event["Queue"].(string),
+		},
+		{
+			"name":  "Passive",
+			"value": strconv.FormatBool(event["Passive"].(bool)),
+		},
+		{
+			"name":  "Durable",
+			"value": strconv.FormatBool(event["Durable"].(bool)),
+		},
+		{
+			"name":  "Exclusive",
+			"value": strconv.FormatBool(event["Exclusive"].(bool)),
+		},
+		{
+			"name":  "Auto Delete",
+			"value": strconv.FormatBool(event["AutoDelete"].(bool)),
+		},
+		{
+			"name":  "NoWait",
+			"value": strconv.FormatBool(event["NoWait"].(bool)),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	if event["Arguments"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range event["Arguments"].(map[string]interface{}) {
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": value.(string),
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Arguments",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}
+
+func representExchangeDeclare(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Exchange",
+			"value": event["Exchange"].(string),
+		},
+		{
+			"name":  "Type",
+			"value": event["Type"].(string),
+		},
+		{
+			"name":  "Passive",
+			"value": strconv.FormatBool(event["Passive"].(bool)),
+		},
+		{
+			"name":  "Durable",
+			"value": strconv.FormatBool(event["Durable"].(bool)),
+		},
+		{
+			"name":  "Auto Delete",
+			"value": strconv.FormatBool(event["AutoDelete"].(bool)),
+		},
+		{
+			"name":  "Internal",
+			"value": strconv.FormatBool(event["Internal"].(bool)),
+		},
+		{
+			"name":  "NoWait",
+			"value": strconv.FormatBool(event["NoWait"].(bool)),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	if event["Arguments"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range event["Arguments"].(map[string]interface{}) {
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": value.(string),
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Arguments",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}
+
+func representConnectionStart(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Version Major",
+			"value": fmt.Sprintf("%g", event["VersionMajor"].(float64)),
+		},
+		{
+			"name":  "Version Minor",
+			"value": fmt.Sprintf("%g", event["VersionMinor"].(float64)),
+		},
+		{
+			"name":  "Mechanisms",
+			"value": event["Mechanisms"].(string),
+		},
+		{
+			"name":  "Locales",
+			"value": event["Locales"].(string),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	if event["ServerProperties"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range event["ServerProperties"].(map[string]interface{}) {
+			var outcome string
+			switch value.(type) {
+			case string:
+				outcome = value.(string)
+				break
+			case map[string]interface{}:
+				x, _ := json.Marshal(value)
+				outcome = string(x)
+				break
+			default:
+				panic("Unknown data type for the server property!")
+			}
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": outcome,
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Server Properties",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}
+
+func representConnectionClose(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Reply Code",
+			"value": fmt.Sprintf("%g", event["ReplyCode"].(float64)),
+		},
+		{
+			"name":  "Reply Text",
+			"value": event["ReplyText"].(string),
+		},
+		{
+			"name":  "Class ID",
+			"value": fmt.Sprintf("%g", event["ClassId"].(float64)),
+		},
+		{
+			"name":  "Method ID",
+			"value": fmt.Sprintf("%g", event["MethodId"].(float64)),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	return rep
+}
+
+func representQueueBind(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Queue",
+			"value": event["Queue"].(string),
+		},
+		{
+			"name":  "Exchange",
+			"value": event["Exchange"].(string),
+		},
+		{
+			"name":  "RoutingKey",
+			"value": event["RoutingKey"].(string),
+		},
+		{
+			"name":  "NoWait",
+			"value": strconv.FormatBool(event["NoWait"].(bool)),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	if event["Arguments"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range event["Arguments"].(map[string]interface{}) {
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": value.(string),
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Arguments",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}
+
+func representBasicConsume(event map[string]interface{}) []interface{} {
+	rep := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Queue",
+			"value": event["Queue"].(string),
+		},
+		{
+			"name":  "Consumer Tag",
+			"value": event["ConsumerTag"].(string),
+		},
+		{
+			"name":  "No Local",
+			"value": strconv.FormatBool(event["NoLocal"].(bool)),
+		},
+		{
+			"name":  "No Ack",
+			"value": strconv.FormatBool(event["NoAck"].(bool)),
+		},
+		{
+			"name":  "Exclusive",
+			"value": strconv.FormatBool(event["Exclusive"].(bool)),
+		},
+		{
+			"name":  "NoWait",
+			"value": strconv.FormatBool(event["NoWait"].(bool)),
+		},
+	})
+	rep = append(rep, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	if event["Arguments"] != nil {
+		headers := make([]map[string]string, 0)
+		for name, value := range event["Arguments"].(map[string]interface{}) {
+			headers = append(headers, map[string]string{
+				"name":  name,
+				"value": value.(string),
+			})
+		}
+		headersMarshaled, _ := json.Marshal(headers)
+		rep = append(rep, map[string]string{
+			"type":  "table",
+			"title": "Arguments",
+			"data":  string(headersMarshaled),
+		})
+	}
+
+	return rep
+}

tap/extensions/amqp/main.go

+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"strconv"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+var protocol api.Protocol = api.Protocol{
+	Name:            "amqp",
+	LongName:        "Advanced Message Queuing Protocol 0-9-1",
+	Abbreviation:    "AMQP",
+	Version:         "0-9-1",
+	BackgroundColor: "#ff6600",
+	ForegroundColor: "#ffffff",
+	FontSize:        12,
+	ReferenceLink:   "https://www.rabbitmq.com/amqp-0-9-1-reference.html",
+	Ports:           []string{"5671", "5672"},
+	Priority:        1,
+}
+
+func init() {
+	log.Println("Initializing AMQP extension...")
+}
+
+type dissecting string
+
+func (d dissecting) Register(extension *api.Extension) {
+	extension.Protocol = protocol
+}
+
+func (d dissecting) Ping() {
+	log.Printf("pong %s\n", protocol.Name)
+}
+
+const amqpRequest string = "amqp_request"
+
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, emitter api.Emitter) error {
+	r := AmqpReader{b}
+
+	var remaining int
+	var header *HeaderFrame
+	var body []byte
+
+	connectionInfo := &api.ConnectionInfo{
+		ClientIP:   tcpID.SrcIP,
+		ClientPort: tcpID.SrcPort,
+		ServerIP:   tcpID.DstIP,
+		ServerPort: tcpID.DstPort,
+		IsOutgoing: true,
+	}
+
+	eventBasicPublish := &BasicPublish{
+		Exchange:   "",
+		RoutingKey: "",
+		Mandatory:  false,
+		Immediate:  false,
+		Body:       nil,
+		Properties: Properties{},
+	}
+
+	eventBasicDeliver := &BasicDeliver{
+		ConsumerTag: "",
+		DeliveryTag: 0,
+		Redelivered: false,
+		Exchange:    "",
+		RoutingKey:  "",
+		Properties:  Properties{},
+		Body:        nil,
+	}
+
+	var lastMethodFrameMessage Message
+
+	for {
+		frame, err := r.ReadFrame()
+		if err == io.EOF {
+			// We must read until we see an EOF... very important!
+			return errors.New("AMQP EOF")
+		} else if err != nil {
+			// TODO: Causes ignoring some methods. Return only in case of a certain error. But what?
+			return err
+		}
+
+		switch f := frame.(type) {
+		case *HeartbeatFrame:
+			// drop
+
+		case *HeaderFrame:
+			// start content state
+			header = f
+			remaining = int(header.Size)
+			switch lastMethodFrameMessage.(type) {
+			case *BasicPublish:
+				eventBasicPublish.Properties = header.Properties
+			case *BasicDeliver:
+				eventBasicDeliver.Properties = header.Properties
+			default:
+			}
+
+		case *BodyFrame:
+			// continue until terminated
+			body = append(body, f.Body...)
+			remaining -= len(f.Body)
+			switch lastMethodFrameMessage.(type) {
+			case *BasicPublish:
+				eventBasicPublish.Body = f.Body
+				emitAMQP(*eventBasicPublish, amqpRequest, basicMethodMap[40], connectionInfo, emitter)
+			case *BasicDeliver:
+				eventBasicDeliver.Body = f.Body
+				emitAMQP(*eventBasicDeliver, amqpRequest, basicMethodMap[60], connectionInfo, emitter)
+			default:
+			}
+
+		case *MethodFrame:
+			lastMethodFrameMessage = f.Method
+			switch m := f.Method.(type) {
+			case *BasicPublish:
+				eventBasicPublish.Exchange = m.Exchange
+				eventBasicPublish.RoutingKey = m.RoutingKey
+				eventBasicPublish.Mandatory = m.Mandatory
+				eventBasicPublish.Immediate = m.Immediate
+
+			case *QueueBind:
+				eventQueueBind := &QueueBind{
+					Queue:      m.Queue,
+					Exchange:   m.Exchange,
+					RoutingKey: m.RoutingKey,
+					NoWait:     m.NoWait,
+					Arguments:  m.Arguments,
+				}
+				emitAMQP(*eventQueueBind, amqpRequest, queueMethodMap[20], connectionInfo, emitter)
+
+			case *BasicConsume:
+				eventBasicConsume := &BasicConsume{
+					Queue:       m.Queue,
+					ConsumerTag: m.ConsumerTag,
+					NoLocal:     m.NoLocal,
+					NoAck:       m.NoAck,
+					Exclusive:   m.Exclusive,
+					NoWait:      m.NoWait,
+					Arguments:   m.Arguments,
+				}
+				emitAMQP(*eventBasicConsume, amqpRequest, basicMethodMap[20], connectionInfo, emitter)
+
+			case *BasicDeliver:
+				eventBasicDeliver.ConsumerTag = m.ConsumerTag
+				eventBasicDeliver.DeliveryTag = m.DeliveryTag
+				eventBasicDeliver.Redelivered = m.Redelivered
+				eventBasicDeliver.Exchange = m.Exchange
+				eventBasicDeliver.RoutingKey = m.RoutingKey
+
+			case *QueueDeclare:
+				eventQueueDeclare := &QueueDeclare{
+					Queue:      m.Queue,
+					Passive:    m.Passive,
+					Durable:    m.Durable,
+					AutoDelete: m.AutoDelete,
+					Exclusive:  m.Exclusive,
+					NoWait:     m.NoWait,
+					Arguments:  m.Arguments,
+				}
+				emitAMQP(*eventQueueDeclare, amqpRequest, queueMethodMap[10], connectionInfo, emitter)
+
+			case *ExchangeDeclare:
+				eventExchangeDeclare := &ExchangeDeclare{
+					Exchange:   m.Exchange,
+					Type:       m.Type,
+					Passive:    m.Passive,
+					Durable:    m.Durable,
+					AutoDelete: m.AutoDelete,
+					Internal:   m.Internal,
+					NoWait:     m.NoWait,
+					Arguments:  m.Arguments,
+				}
+				emitAMQP(*eventExchangeDeclare, amqpRequest, exchangeMethodMap[10], connectionInfo, emitter)
+
+			case *ConnectionStart:
+				eventConnectionStart := &ConnectionStart{
+					VersionMajor:     m.VersionMajor,
+					VersionMinor:     m.VersionMinor,
+					ServerProperties: m.ServerProperties,
+					Mechanisms:       m.Mechanisms,
+					Locales:          m.Locales,
+				}
+				emitAMQP(*eventConnectionStart, amqpRequest, connectionMethodMap[10], connectionInfo, emitter)
+
+			case *ConnectionClose:
+				eventConnectionClose := &ConnectionClose{
+					ReplyCode: m.ReplyCode,
+					ReplyText: m.ReplyText,
+					ClassId:   m.ClassId,
+					MethodId:  m.MethodId,
+				}
+				emitAMQP(*eventConnectionClose, amqpRequest, connectionMethodMap[50], connectionInfo, emitter)
+
+			default:
+
+			}
+
+		default:
+			// log.Printf("unexpected frame: %+v\n", f)
+		}
+	}
+}
+
+func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+	request := item.Pair.Request.Payload.(map[string]interface{})
+	reqDetails := request["details"].(map[string]interface{})
+	service := "amqp"
+	if resolvedDestination != "" {
+		service = resolvedDestination
+	} else if resolvedSource != "" {
+		service = resolvedSource
+	}
+
+	summary := ""
+	switch request["method"] {
+	case basicMethodMap[40]:
+		summary = reqDetails["Exchange"].(string)
+		break
+	case basicMethodMap[60]:
+		summary = reqDetails["Exchange"].(string)
+		break
+	case exchangeMethodMap[10]:
+		summary = reqDetails["Exchange"].(string)
+		break
+	case queueMethodMap[10]:
+		summary = reqDetails["Queue"].(string)
+		break
+	case connectionMethodMap[10]:
+		summary = fmt.Sprintf(
+			"%s.%s",
+			strconv.Itoa(int(reqDetails["VersionMajor"].(float64))),
+			strconv.Itoa(int(reqDetails["VersionMinor"].(float64))),
+		)
+		break
+	case connectionMethodMap[50]:
+		summary = reqDetails["ReplyText"].(string)
+		break
+	case queueMethodMap[20]:
+		summary = reqDetails["Queue"].(string)
+		break
+	case basicMethodMap[20]:
+		summary = reqDetails["Queue"].(string)
+		break
+	}
+
+	request["url"] = summary
+	entryBytes, _ := json.Marshal(item.Pair)
+	return &api.MizuEntry{
+		ProtocolName:        protocol.Name,
+		ProtocolVersion:     protocol.Version,
+		EntryId:             entryId,
+		Entry:               string(entryBytes),
+		Url:                 fmt.Sprintf("%s%s", service, summary),
+		Method:              request["method"].(string),
+		Status:              0,
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		Path:                summary,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+	}
+
+}
+
+func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
+	return &api.BaseEntryDetails{
+		Id:              entry.EntryId,
+		Protocol:        protocol,
+		Url:             entry.Url,
+		RequestSenderIp: entry.RequestSenderIp,
+		Service:         entry.Service,
+		Summary:         entry.Path,
+		StatusCode:      entry.Status,
+		Method:          entry.Method,
+		Timestamp:       entry.Timestamp,
+		SourceIp:        entry.SourceIp,
+		DestinationIp:   entry.DestinationIp,
+		SourcePort:      entry.SourcePort,
+		DestinationPort: entry.DestinationPort,
+		IsOutgoing:      entry.IsOutgoing,
+		Latency:         0,
+		Rules: api.ApplicableRules{
+			Latency: 0,
+			Status:  false,
+		},
+	}
+}
+
+func (d dissecting) Represent(entry *api.MizuEntry) (api.Protocol, []byte, error) {
+	var root map[string]interface{}
+	json.Unmarshal([]byte(entry.Entry), &root)
+	representation := make(map[string]interface{}, 0)
+	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
+	var repRequest []interface{}
+	details := request["details"].(map[string]interface{})
+	switch request["method"].(string) {
+	case basicMethodMap[40]:
+		repRequest = representBasicPublish(details)
+		break
+	case basicMethodMap[60]:
+		repRequest = representBasicDeliver(details)
+		break
+	case queueMethodMap[10]:
+		repRequest = representQueueDeclare(details)
+		break
+	case exchangeMethodMap[10]:
+		repRequest = representExchangeDeclare(details)
+		break
+	case connectionMethodMap[10]:
+		repRequest = representConnectionStart(details)
+		break
+	case connectionMethodMap[50]:
+		repRequest = representConnectionClose(details)
+		break
+	case queueMethodMap[20]:
+		repRequest = representQueueBind(details)
+		break
+	case basicMethodMap[20]:
+		repRequest = representBasicConsume(details)
+		break
+	}
+	representation["request"] = repRequest
+	object, err := json.Marshal(representation)
+	return protocol, object, err
+}
+
+var Dissector dissecting

tap/extensions/amqp/read.go

+// Copyright (c) 2012, Sean Treadway, SoundCloud Ltd.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// Source code and contact info at http://github.com/streadway/amqp
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"io"
+	"time"
+)
+
+/*
+Reads a frame from an input stream and returns an interface that can be cast into
+one of the following:
+
+   MethodFrame
+   PropertiesFrame
+   BodyFrame
+   HeartbeatFrame
+
+2.3.5  frame Details
+
+All frames consist of a header (7 octets), a payload of arbitrary size, and a
+'frame-end' octet that detects malformed frames:
+
+  0      1         3             7                  size+7 size+8
+  +------+---------+-------------+  +------------+  +-----------+
+  | type | channel |     size    |  |  payload   |  | frame-end |
+  +------+---------+-------------+  +------------+  +-----------+
+   octet   short         long         size octets       octet
+
+To read a frame, we:
+  1. Read the header and check the frame type and channel.
+	2. Depending on the frame type, we read the payload and process it.
+  3. Read the frame end octet.
+
+In realistic implementations where performance is a concern, we would use
+“read-ahead buffering” or
+
+“gathering reads” to avoid doing three separate system calls to read a frame.
+*/
+func (r *AmqpReader) ReadFrame() (frame frame, err error) {
+	var scratch [7]byte
+
+	if _, err = io.ReadFull(r.R, scratch[:7]); err != nil {
+		return
+	}
+
+	typ := uint8(scratch[0])
+	channel := binary.BigEndian.Uint16(scratch[1:3])
+	size := binary.BigEndian.Uint32(scratch[3:7])
+
+	if size > 1000000 {
+		return nil, ErrMaxSize
+	}
+
+	switch typ {
+	case frameMethod:
+		if frame, err = r.parseMethodFrame(channel, size); err != nil {
+			return
+		}
+
+	case frameHeader:
+		if frame, err = r.parseHeaderFrame(channel, size); err != nil {
+			return
+		}
+
+	case frameBody:
+		if frame, err = r.parseBodyFrame(channel, size); err != nil {
+			return nil, err
+		}
+
+	case frameHeartbeat:
+		if frame, err = r.parseHeartbeatFrame(channel, size); err != nil {
+			return
+		}
+
+	default:
+		return nil, ErrFrame
+	}
+
+	if _, err = io.ReadFull(r.R, scratch[:1]); err != nil {
+		return nil, err
+	}
+
+	if scratch[0] != frameEnd {
+		return nil, ErrFrame
+	}
+
+	return
+}
+
+func readShortstr(r io.Reader) (v string, err error) {
+	var length uint8
+	if err = binary.Read(r, binary.BigEndian, &length); err != nil {
+		return
+	}
+
+	bytes := make([]byte, length)
+	if _, err = io.ReadFull(r, bytes); err != nil {
+		return
+	}
+	return string(bytes), nil
+}
+
+func readLongstr(r io.Reader) (v string, err error) {
+	var length uint32
+	if err = binary.Read(r, binary.BigEndian, &length); err != nil {
+		return
+	}
+
+	// slices can't be longer than max int32 value
+	if length > (^uint32(0) >> 1) {
+		return
+	}
+
+	bytes := make([]byte, length)
+	if _, err = io.ReadFull(r, bytes); err != nil {
+		return
+	}
+	return string(bytes), nil
+}
+
+func readDecimal(r io.Reader) (v Decimal, err error) {
+	if err = binary.Read(r, binary.BigEndian, &v.Scale); err != nil {
+		return
+	}
+	if err = binary.Read(r, binary.BigEndian, &v.Value); err != nil {
+		return
+	}
+	return
+}
+
+func readFloat32(r io.Reader) (v float32, err error) {
+	if err = binary.Read(r, binary.BigEndian, &v); err != nil {
+		return
+	}
+	return
+}
+
+func readFloat64(r io.Reader) (v float64, err error) {
+	if err = binary.Read(r, binary.BigEndian, &v); err != nil {
+		return
+	}
+	return
+}
+
+func readTimestamp(r io.Reader) (v time.Time, err error) {
+	var sec int64
+	if err = binary.Read(r, binary.BigEndian, &sec); err != nil {
+		return
+	}
+	return time.Unix(sec, 0), nil
+}
+
+/*
+'A': []interface{}
+'D': Decimal
+'F': Table
+'I': int32
+'S': string
+'T': time.Time
+'V': nil
+'b': byte
+'d': float64
+'f': float32
+'l': int64
+'s': int16
+'t': bool
+'x': []byte
+*/
+func readField(r io.Reader) (v interface{}, err error) {
+	var typ byte
+	if err = binary.Read(r, binary.BigEndian, &typ); err != nil {
+		return
+	}
+
+	switch typ {
+	case 't':
+		var value uint8
+		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
+			return
+		}
+		return (value != 0), nil
+
+	case 'b':
+		var value [1]byte
+		if _, err = io.ReadFull(r, value[0:1]); err != nil {
+			return
+		}
+		return value[0], nil
+
+	case 's':
+		var value int16
+		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
+			return
+		}
+		return value, nil
+
+	case 'I':
+		var value int32
+		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
+			return
+		}
+		return value, nil
+
+	case 'l':
+		var value int64
+		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
+			return
+		}
+		return value, nil
+
+	case 'f':
+		var value float32
+		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
+			return
+		}
+		return value, nil
+
+	case 'd':
+		var value float64
+		if err = binary.Read(r, binary.BigEndian, &value); err != nil {
+			return
+		}
+		return value, nil
+
+	case 'D':
+		return readDecimal(r)
+
+	case 'S':
+		return readLongstr(r)
+
+	case 'A':
+		return readArray(r)
+
+	case 'T':
+		return readTimestamp(r)
+
+	case 'F':
+		return readTable(r)
+
+	case 'x':
+		var len int32
+		if err = binary.Read(r, binary.BigEndian, &len); err != nil {
+			return nil, err
+		}
+
+		value := make([]byte, len)
+		if _, err = io.ReadFull(r, value); err != nil {
+			return nil, err
+		}
+		return value, err
+
+	case 'V':
+		return nil, nil
+	}
+
+	return nil, ErrSyntax
+}
+
+/*
+	Field tables are long strings that contain packed name-value pairs.  The
+	name-value pairs are encoded as short string defining the name, and octet
+	defining the values type and then the value itself.   The valid field types for
+	tables are an extension of the native integer, bit, string, and timestamp
+	types, and are shown in the grammar.  Multi-octet integer fields are always
+	held in network byte order.
+*/
+func readTable(r io.Reader) (table Table, err error) {
+	var nested bytes.Buffer
+	var str string
+
+	if str, err = readLongstr(r); err != nil {
+		return
+	}
+
+	nested.Write([]byte(str))
+
+	table = make(Table)
+
+	for nested.Len() > 0 {
+		var key string
+		var value interface{}
+
+		if key, err = readShortstr(&nested); err != nil {
+			return
+		}
+
+		if value, err = readField(&nested); err != nil {
+			return
+		}
+
+		table[key] = value
+	}
+
+	return
+}
+
+func readArray(r io.Reader) ([]interface{}, error) {
+	var (
+		size uint32
+		err  error
+	)
+
+	if err = binary.Read(r, binary.BigEndian, &size); err != nil {
+		return nil, err
+	}
+
+	var (
+		lim   = &io.LimitedReader{R: r, N: int64(size)}
+		arr   = []interface{}{}
+		field interface{}
+	)
+
+	for {
+		if field, err = readField(lim); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		arr = append(arr, field)
+	}
+
+	return arr, nil
+}
+
+// Checks if this bit mask matches the flags bitset
+func hasProperty(mask uint16, prop int) bool {
+	return int(mask)&prop > 0
+}
+
+func (r *AmqpReader) parseHeaderFrame(channel uint16, size uint32) (frame frame, err error) {
+	hf := &HeaderFrame{
+		ChannelId: channel,
+	}
+
+	if err = binary.Read(r.R, binary.BigEndian, &hf.ClassId); err != nil {
+		return
+	}
+
+	if err = binary.Read(r.R, binary.BigEndian, &hf.weight); err != nil {
+		return
+	}
+
+	if err = binary.Read(r.R, binary.BigEndian, &hf.Size); err != nil {
+		return
+	}
+
+	var flags uint16
+
+	if err = binary.Read(r.R, binary.BigEndian, &flags); err != nil {
+		return
+	}
+
+	if hasProperty(flags, flagContentType) {
+		if hf.Properties.ContentType, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagContentEncoding) {
+		if hf.Properties.ContentEncoding, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagHeaders) {
+		if hf.Properties.Headers, err = readTable(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagDeliveryMode) {
+		if err = binary.Read(r.R, binary.BigEndian, &hf.Properties.DeliveryMode); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagPriority) {
+		if err = binary.Read(r.R, binary.BigEndian, &hf.Properties.Priority); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagCorrelationId) {
+		if hf.Properties.CorrelationId, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagReplyTo) {
+		if hf.Properties.ReplyTo, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagExpiration) {
+		if hf.Properties.Expiration, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagMessageId) {
+		if hf.Properties.MessageId, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagTimestamp) {
+		if hf.Properties.Timestamp, err = readTimestamp(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagType) {
+		if hf.Properties.Type, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagUserId) {
+		if hf.Properties.UserId, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagAppId) {
+		if hf.Properties.AppId, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+	if hasProperty(flags, flagReserved1) {
+		if hf.Properties.reserved1, err = readShortstr(r.R); err != nil {
+			return
+		}
+	}
+
+	return hf, nil
+}
+
+func (r *AmqpReader) parseBodyFrame(channel uint16, size uint32) (frame frame, err error) {
+	bf := &BodyFrame{
+		ChannelId: channel,
+		Body:      make([]byte, size),
+	}
+
+	if _, err = io.ReadFull(r.R, bf.Body); err != nil {
+		return nil, err
+	}
+
+	return bf, nil
+}
+
+var errHeartbeatPayload = errors.New("Heartbeats should not have a payload")
+
+func (r *AmqpReader) parseHeartbeatFrame(channel uint16, size uint32) (frame frame, err error) {
+	hf := &HeartbeatFrame{
+		ChannelId: channel,
+	}
+
+	if size > 0 {
+		return nil, errHeartbeatPayload
+	}
+
+	return hf, nil
+}

tap/extensions/amqp/structs.go

+package main
+
+import (
+	"encoding/json"
+)
+
+type AMQPPayload struct {
+	Data interface{}
+}
+
+type AMQPPayloader interface {
+	MarshalJSON() ([]byte, error)
+}
+
+func (h AMQPPayload) MarshalJSON() ([]byte, error) {
+	return json.Marshal(h.Data)
+}

tap/extensions/amqp/types.go

+// Copyright (c) 2012, Sean Treadway, SoundCloud Ltd.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// Source code and contact info at http://github.com/streadway/amqp
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"time"
+)
+
+// Constants for standard AMQP 0-9-1 exchange types.
+const (
+	ExchangeDirect  = "direct"
+	ExchangeFanout  = "fanout"
+	ExchangeTopic   = "topic"
+	ExchangeHeaders = "headers"
+)
+
+var (
+	// ErrClosed is returned when the channel or connection is not open
+	ErrClosed = &Error{Code: ChannelError, Reason: "channel/connection is not open"}
+
+	// ErrChannelMax is returned when Connection.Channel has been called enough
+	// times that all channel IDs have been exhausted in the client or the
+	// server.
+	ErrChannelMax = &Error{Code: ChannelError, Reason: "channel id space exhausted"}
+
+	// ErrSASL is returned from Dial when the authentication mechanism could not
+	// be negoated.
+	ErrSASL = &Error{Code: AccessRefused, Reason: "SASL could not negotiate a shared mechanism"}
+
+	// ErrCredentials is returned when the authenticated client is not authorized
+	// to any vhost.
+	ErrCredentials = &Error{Code: AccessRefused, Reason: "username or password not allowed"}
+
+	// ErrVhost is returned when the authenticated user is not permitted to
+	// access the requested Vhost.
+	ErrVhost = &Error{Code: AccessRefused, Reason: "no access to this vhost"}
+
+	// ErrSyntax is hard protocol error, indicating an unsupported protocol,
+	// implementation or encoding.
+	ErrSyntax = &Error{Code: SyntaxError, Reason: "invalid field or value inside of a frame"}
+
+	// ErrFrame is returned when the protocol frame cannot be read from the
+	// server, indicating an unsupported protocol or unsupported frame type.
+	ErrFrame = &Error{Code: FrameError, Reason: "frame could not be parsed"}
+
+	// ErrCommandInvalid is returned when the server sends an unexpected response
+	// to this requested message type. This indicates a bug in this client.
+	ErrCommandInvalid = &Error{Code: CommandInvalid, Reason: "unexpected command received"}
+
+	// ErrUnexpectedFrame is returned when something other than a method or
+	// heartbeat frame is delivered to the Connection, indicating a bug in the
+	// client.
+	ErrUnexpectedFrame = &Error{Code: UnexpectedFrame, Reason: "unexpected frame received"}
+
+	// ErrFieldType is returned when writing a message containing a Go type unsupported by AMQP.
+	ErrFieldType = &Error{Code: SyntaxError, Reason: "unsupported table field type"}
+
+	// ErrClosed is returned when the channel or connection is not open
+	ErrMaxSize = &Error{Code: MaxSizeError, Reason: "an AMQP message cannot be bigger than 1MB"}
+)
+
+// Error captures the code and reason a channel or connection has been closed
+// by the server.
+type Error struct {
+	Code    int    // constant code from the specification
+	Reason  string // description of the error
+	Server  bool   // true when initiated from the server, false when from this library
+	Recover bool   // true when this error can be recovered by retrying later or with different parameters
+}
+
+func newError(code uint16, text string) *Error {
+	return &Error{
+		Code:    int(code),
+		Reason:  text,
+		Recover: isSoftExceptionCode(int(code)),
+		Server:  true,
+	}
+}
+
+func (e Error) Error() string {
+	return fmt.Sprintf("Exception (%d) Reason: %q", e.Code, e.Reason)
+}
+
+// Used by header frames to capture routing and header information
+type Properties struct {
+	ContentType     string    // MIME content type
+	ContentEncoding string    // MIME content encoding
+	Headers         Table     // Application or header exchange table
+	DeliveryMode    uint8     // queue implementation use - Transient (1) or Persistent (2)
+	Priority        uint8     // queue implementation use - 0 to 9
+	CorrelationId   string    // application use - correlation identifier
+	ReplyTo         string    // application use - address to to reply to (ex: RPC)
+	Expiration      string    // implementation use - message expiration spec
+	MessageId       string    // application use - message identifier
+	Timestamp       time.Time // application use - message timestamp
+	Type            string    // application use - message type name
+	UserId          string    // application use - creating user id
+	AppId           string    // application use - creating application
+	reserved1       string    // was cluster-id - process for buffer consumption
+}
+
+// DeliveryMode.  Transient means higher throughput but messages will not be
+// restored on broker restart.  The delivery mode of publishings is unrelated
+// to the durability of the queues they reside on.  Transient messages will
+// not be restored to durable queues, persistent messages will be restored to
+// durable queues and lost on non-durable queues during server restart.
+//
+// This remains typed as uint8 to match Publishing.DeliveryMode.  Other
+// delivery modes specific to custom queue implementations are not enumerated
+// here.
+const (
+	Transient  uint8 = 1
+	Persistent uint8 = 2
+)
+
+// The property flags are an array of bits that indicate the presence or
+// absence of each property value in sequence.  The bits are ordered from most
+// high to low - bit 15 indicates the first property.
+const (
+	flagContentType     = 0x8000
+	flagContentEncoding = 0x4000
+	flagHeaders         = 0x2000
+	flagDeliveryMode    = 0x1000
+	flagPriority        = 0x0800
+	flagCorrelationId   = 0x0400
+	flagReplyTo         = 0x0200
+	flagExpiration      = 0x0100
+	flagMessageId       = 0x0080
+	flagTimestamp       = 0x0040
+	flagType            = 0x0020
+	flagUserId          = 0x0010
+	flagAppId           = 0x0008
+	flagReserved1       = 0x0004
+)
+
+// Queue captures the current server state of the queue on the server returned
+// from Channel.QueueDeclare or Channel.QueueInspect.
+type Queue struct {
+	Name      string // server confirmed or generated name
+	Messages  int    // count of messages not awaiting acknowledgment
+	Consumers int    // number of consumers receiving deliveries
+}
+
+// Publishing captures the client message sent to the server.  The fields
+// outside of the Headers table included in this struct mirror the underlying
+// fields in the content frame.  They use native types for convenience and
+// efficiency.
+type Publishing struct {
+	// Application or exchange specific fields,
+	// the headers exchange will inspect this field.
+	Headers Table
+
+	// Properties
+	ContentType     string    // MIME content type
+	ContentEncoding string    // MIME content encoding
+	DeliveryMode    uint8     // Transient (0 or 1) or Persistent (2)
+	Priority        uint8     // 0 to 9
+	CorrelationId   string    // correlation identifier
+	ReplyTo         string    // address to to reply to (ex: RPC)
+	Expiration      string    // message expiration spec
+	MessageId       string    // message identifier
+	Timestamp       time.Time // message timestamp
+	Type            string    // message type name
+	UserId          string    // creating user id - ex: "guest"
+	AppId           string    // creating application id
+
+	// The application specific payload of the message
+	Body []byte
+}
+
+// Blocking notifies the server's TCP flow control of the Connection.  When a
+// server hits a memory or disk alarm it will block all connections until the
+// resources are reclaimed.  Use NotifyBlock on the Connection to receive these
+// events.
+type Blocking struct {
+	Active bool   // TCP pushback active/inactive on server
+	Reason string // Server reason for activation
+}
+
+// Confirmation notifies the acknowledgment or negative acknowledgement of a
+// publishing identified by its delivery tag.  Use NotifyPublish on the Channel
+// to consume these events.
+type Confirmation struct {
+	DeliveryTag uint64 // A 1 based counter of publishings from when the channel was put in Confirm mode
+	Ack         bool   // True when the server successfully received the publishing
+}
+
+// Decimal matches the AMQP decimal type.  Scale is the number of decimal
+// digits Scale == 2, Value == 12345, Decimal == 123.45
+type Decimal struct {
+	Scale uint8
+	Value int32
+}
+
+// Table stores user supplied fields of the following types:
+//
+//   bool
+//   byte
+//   float32
+//   float64
+//   int
+//   int16
+//   int32
+//   int64
+//   nil
+//   string
+//   time.Time
+//   amqp.Decimal
+//   amqp.Table
+//   []byte
+//   []interface{} - containing above types
+//
+// Functions taking a table will immediately fail when the table contains a
+// value of an unsupported type.
+//
+// The caller must be specific in which precision of integer it wishes to
+// encode.
+//
+// Use a type assertion when reading values from a table for type conversion.
+//
+// RabbitMQ expects int32 for integer values.
+//
+type Table map[string]interface{}
+
+func validateField(f interface{}) error {
+	switch fv := f.(type) {
+	case nil, bool, byte, int, int16, int32, int64, float32, float64, string, []byte, Decimal, time.Time:
+		return nil
+
+	case []interface{}:
+		for _, v := range fv {
+			if err := validateField(v); err != nil {
+				return fmt.Errorf("in array %s", err)
+			}
+		}
+		return nil
+
+	case Table:
+		for k, v := range fv {
+			if err := validateField(v); err != nil {
+				return fmt.Errorf("table field %q %s", k, err)
+			}
+		}
+		return nil
+	}
+
+	return fmt.Errorf("value %T not supported", f)
+}
+
+// Validate returns and error if any Go types in the table are incompatible with AMQP types.
+func (t Table) Validate() error {
+	return validateField(t)
+}
+
+// Heap interface for maintaining delivery tags
+type tagSet []uint64
+
+func (set tagSet) Len() int              { return len(set) }
+func (set tagSet) Less(i, j int) bool    { return (set)[i] < (set)[j] }
+func (set tagSet) Swap(i, j int)         { (set)[i], (set)[j] = (set)[j], (set)[i] }
+func (set *tagSet) Push(tag interface{}) { *set = append(*set, tag.(uint64)) }
+func (set *tagSet) Pop() interface{} {
+	val := (*set)[len(*set)-1]
+	*set = (*set)[:len(*set)-1]
+	return val
+}
+
+type Message interface {
+	id() (uint16, uint16)
+	wait() bool
+	read(io.Reader) error
+	write(io.Writer) error
+}
+
+type messageWithContent interface {
+	Message
+	getContent() (Properties, []byte)
+	setContent(Properties, []byte)
+}
+
+/*
+The base interface implemented as:
+
+2.3.5  frame Details
+
+All frames consist of a header (7 octets), a payload of arbitrary size, and a 'frame-end' octet that detects
+malformed frames:
+
+  0      1         3             7                  size+7 size+8
+  +------+---------+-------------+  +------------+  +-----------+
+  | type | channel |     size    |  |  payload   |  | frame-end |
+  +------+---------+-------------+  +------------+  +-----------+
+   octet   short         long         size octets       octet
+
+To read a frame, we:
+
+ 1. Read the header and check the frame type and channel.
+ 2. Depending on the frame type, we read the payload and process it.
+ 3. Read the frame end octet.
+
+In realistic implementations where performance is a concern, we would use
+“read-ahead buffering” or “gathering reads” to avoid doing three separate
+system calls to read a frame.
+
+*/
+type frame interface {
+	write(io.Writer) error
+	channel() uint16
+}
+
+type AmqpReader struct {
+	R io.Reader
+}
+
+type writer struct {
+	w io.Writer
+}
+
+// Implements the frame interface for Connection RPC
+type protocolHeader struct{}
+
+func (protocolHeader) write(w io.Writer) error {
+	_, err := w.Write([]byte{'A', 'M', 'Q', 'P', 0, 0, 9, 1})
+	return err
+}
+
+func (protocolHeader) channel() uint16 {
+	panic("only valid as initial handshake")
+}
+
+/*
+Method frames carry the high-level protocol commands (which we call "methods").
+One method frame carries one command.  The method frame payload has this format:
+
+  0          2           4
+  +----------+-----------+-------------- - -
+  | class-id | method-id | arguments...
+  +----------+-----------+-------------- - -
+     short      short    ...
+
+To process a method frame, we:
+ 1. Read the method frame payload.
+ 2. Unpack it into a structure.  A given method always has the same structure,
+ so we can unpack the method rapidly.  3. Check that the method is allowed in
+ the current context.
+ 4. Check that the method arguments are valid.
+ 5. Execute the method.
+
+Method frame bodies are constructed as a list of AMQP data fields (bits,
+integers, strings and string tables).  The marshalling code is trivially
+generated directly from the protocol specifications, and can be very rapid.
+*/
+type MethodFrame struct {
+	ChannelId uint16
+	ClassId   uint16
+	MethodId  uint16
+	Method    Message
+}
+
+func (f *MethodFrame) channel() uint16 { return f.ChannelId }
+
+/*
+Heartbeating is a technique designed to undo one of TCP/IP's features, namely
+its ability to recover from a broken physical connection by closing only after
+a quite long time-out.  In some scenarios we need to know very rapidly if a
+peer is disconnected or not responding for other reasons (e.g. it is looping).
+Since heartbeating can be done at a low level, we implement this as a special
+type of frame that peers exchange at the transport level, rather than as a
+class method.
+*/
+type HeartbeatFrame struct {
+	ChannelId uint16
+}
+
+func (f *HeartbeatFrame) channel() uint16 { return f.ChannelId }
+
+/*
+Certain methods (such as Basic.Publish, Basic.Deliver, etc.) are formally
+defined as carrying content.  When a peer sends such a method frame, it always
+follows it with a content header and zero or more content body frames.
+
+A content header frame has this format:
+
+    0          2        4           12               14
+    +----------+--------+-----------+----------------+------------- - -
+    | class-id | weight | body size | property flags | property list...
+    +----------+--------+-----------+----------------+------------- - -
+      short     short    long long       short        remainder...
+
+We place content body in distinct frames (rather than including it in the
+method) so that AMQP may support "zero copy" techniques in which content is
+never marshalled or encoded.  We place the content properties in their own
+frame so that recipients can selectively discard contents they do not want to
+process
+*/
+type HeaderFrame struct {
+	ChannelId  uint16
+	ClassId    uint16
+	weight     uint16
+	Size       uint64
+	Properties Properties
+}
+
+func (f *HeaderFrame) channel() uint16 { return f.ChannelId }
+
+/*
+Content is the application data we carry from client-to-client via the AMQP
+server.  Content is, roughly speaking, a set of properties plus a binary data
+part.  The set of allowed properties are defined by the Basic class, and these
+form the "content header frame".  The data can be any size, and MAY be broken
+into several (or many) chunks, each forming a "content body frame".
+
+Looking at the frames for a specific channel, as they pass on the wire, we
+might see something like this:
+
+		[method]
+		[method] [header] [body] [body]
+		[method]
+		...
+*/
+type BodyFrame struct {
+	ChannelId uint16
+	Body      []byte
+}
+
+func (f *BodyFrame) channel() uint16 { return f.ChannelId }

tap/extensions/amqp/write.go

+// Copyright (c) 2012, Sean Treadway, SoundCloud Ltd.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+// Source code and contact info at http://github.com/streadway/amqp
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"io"
+	"math"
+	"time"
+)
+
+func (w *writer) WriteFrame(frame frame) (err error) {
+	if err = frame.write(w.w); err != nil {
+		return
+	}
+
+	if buf, ok := w.w.(*bufio.Writer); ok {
+		err = buf.Flush()
+	}
+
+	return
+}
+
+func (f *MethodFrame) write(w io.Writer) (err error) {
+	var payload bytes.Buffer
+
+	if f.Method == nil {
+		return errors.New("malformed frame: missing method")
+	}
+
+	class, method := f.Method.id()
+
+	if err = binary.Write(&payload, binary.BigEndian, class); err != nil {
+		return
+	}
+
+	if err = binary.Write(&payload, binary.BigEndian, method); err != nil {
+		return
+	}
+
+	if err = f.Method.write(&payload); err != nil {
+		return
+	}
+
+	return writeFrame(w, frameMethod, f.ChannelId, payload.Bytes())
+}
+
+// Heartbeat
+//
+// Payload is empty
+func (f *HeartbeatFrame) write(w io.Writer) (err error) {
+	return writeFrame(w, frameHeartbeat, f.ChannelId, []byte{})
+}
+
+// CONTENT HEADER
+// 0          2        4           12               14
+// +----------+--------+-----------+----------------+------------- - -
+// | class-id | weight | body size | property flags | property list...
+// +----------+--------+-----------+----------------+------------- - -
+//    short     short    long long       short        remainder...
+//
+func (f *HeaderFrame) write(w io.Writer) (err error) {
+	var payload bytes.Buffer
+	var zeroTime time.Time
+
+	if err = binary.Write(&payload, binary.BigEndian, f.ClassId); err != nil {
+		return
+	}
+
+	if err = binary.Write(&payload, binary.BigEndian, f.weight); err != nil {
+		return
+	}
+
+	if err = binary.Write(&payload, binary.BigEndian, f.Size); err != nil {
+		return
+	}
+
+	// First pass will build the mask to be serialized, second pass will serialize
+	// each of the fields that appear in the mask.
+
+	var mask uint16
+
+	if len(f.Properties.ContentType) > 0 {
+		mask = mask | flagContentType
+	}
+	if len(f.Properties.ContentEncoding) > 0 {
+		mask = mask | flagContentEncoding
+	}
+	if f.Properties.Headers != nil && len(f.Properties.Headers) > 0 {
+		mask = mask | flagHeaders
+	}
+	if f.Properties.DeliveryMode > 0 {
+		mask = mask | flagDeliveryMode
+	}
+	if f.Properties.Priority > 0 {
+		mask = mask | flagPriority
+	}
+	if len(f.Properties.CorrelationId) > 0 {
+		mask = mask | flagCorrelationId
+	}
+	if len(f.Properties.ReplyTo) > 0 {
+		mask = mask | flagReplyTo
+	}
+	if len(f.Properties.Expiration) > 0 {
+		mask = mask | flagExpiration
+	}
+	if len(f.Properties.MessageId) > 0 {
+		mask = mask | flagMessageId
+	}
+	if f.Properties.Timestamp != zeroTime {
+		mask = mask | flagTimestamp
+	}
+	if len(f.Properties.Type) > 0 {
+		mask = mask | flagType
+	}
+	if len(f.Properties.UserId) > 0 {
+		mask = mask | flagUserId
+	}
+	if len(f.Properties.AppId) > 0 {
+		mask = mask | flagAppId
+	}
+
+	if err = binary.Write(&payload, binary.BigEndian, mask); err != nil {
+		return
+	}
+
+	if hasProperty(mask, flagContentType) {
+		if err = writeShortstr(&payload, f.Properties.ContentType); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagContentEncoding) {
+		if err = writeShortstr(&payload, f.Properties.ContentEncoding); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagHeaders) {
+		if err = writeTable(&payload, f.Properties.Headers); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagDeliveryMode) {
+		if err = binary.Write(&payload, binary.BigEndian, f.Properties.DeliveryMode); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagPriority) {
+		if err = binary.Write(&payload, binary.BigEndian, f.Properties.Priority); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagCorrelationId) {
+		if err = writeShortstr(&payload, f.Properties.CorrelationId); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagReplyTo) {
+		if err = writeShortstr(&payload, f.Properties.ReplyTo); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagExpiration) {
+		if err = writeShortstr(&payload, f.Properties.Expiration); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagMessageId) {
+		if err = writeShortstr(&payload, f.Properties.MessageId); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagTimestamp) {
+		if err = binary.Write(&payload, binary.BigEndian, uint64(f.Properties.Timestamp.Unix())); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagType) {
+		if err = writeShortstr(&payload, f.Properties.Type); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagUserId) {
+		if err = writeShortstr(&payload, f.Properties.UserId); err != nil {
+			return
+		}
+	}
+	if hasProperty(mask, flagAppId) {
+		if err = writeShortstr(&payload, f.Properties.AppId); err != nil {
+			return
+		}
+	}
+
+	return writeFrame(w, frameHeader, f.ChannelId, payload.Bytes())
+}
+
+// Body
+//
+// Payload is one byterange from the full body who's size is declared in the
+// Header frame
+func (f *BodyFrame) write(w io.Writer) (err error) {
+	return writeFrame(w, frameBody, f.ChannelId, f.Body)
+}
+
+func writeFrame(w io.Writer, typ uint8, channel uint16, payload []byte) (err error) {
+	end := []byte{frameEnd}
+	size := uint(len(payload))
+
+	_, err = w.Write([]byte{
+		byte(typ),
+		byte((channel & 0xff00) >> 8),
+		byte((channel & 0x00ff) >> 0),
+		byte((size & 0xff000000) >> 24),
+		byte((size & 0x00ff0000) >> 16),
+		byte((size & 0x0000ff00) >> 8),
+		byte((size & 0x000000ff) >> 0),
+	})
+
+	if err != nil {
+		return
+	}
+
+	if _, err = w.Write(payload); err != nil {
+		return
+	}
+
+	if _, err = w.Write(end); err != nil {
+		return
+	}
+
+	return
+}
+
+func writeShortstr(w io.Writer, s string) (err error) {
+	b := []byte(s)
+
+	var length = uint8(len(b))
+
+	if err = binary.Write(w, binary.BigEndian, length); err != nil {
+		return
+	}
+
+	if _, err = w.Write(b[:length]); err != nil {
+		return
+	}
+
+	return
+}
+
+func writeLongstr(w io.Writer, s string) (err error) {
+	b := []byte(s)
+
+	var length = uint32(len(b))
+
+	if err = binary.Write(w, binary.BigEndian, length); err != nil {
+		return
+	}
+
+	if _, err = w.Write(b[:length]); err != nil {
+		return
+	}
+
+	return
+}
+
+/*
+'A': []interface{}
+'D': Decimal
+'F': Table
+'I': int32
+'S': string
+'T': time.Time
+'V': nil
+'b': byte
+'d': float64
+'f': float32
+'l': int64
+'s': int16
+'t': bool
+'x': []byte
+*/
+func writeField(w io.Writer, value interface{}) (err error) {
+	var buf [9]byte
+	var enc []byte
+
+	switch v := value.(type) {
+	case bool:
+		buf[0] = 't'
+		if v {
+			buf[1] = byte(1)
+		} else {
+			buf[1] = byte(0)
+		}
+		enc = buf[:2]
+
+	case byte:
+		buf[0] = 'b'
+		buf[1] = byte(v)
+		enc = buf[:2]
+
+	case int16:
+		buf[0] = 's'
+		binary.BigEndian.PutUint16(buf[1:3], uint16(v))
+		enc = buf[:3]
+
+	case int:
+		buf[0] = 'I'
+		binary.BigEndian.PutUint32(buf[1:5], uint32(v))
+		enc = buf[:5]
+
+	case int32:
+		buf[0] = 'I'
+		binary.BigEndian.PutUint32(buf[1:5], uint32(v))
+		enc = buf[:5]
+
+	case int64:
+		buf[0] = 'l'
+		binary.BigEndian.PutUint64(buf[1:9], uint64(v))
+		enc = buf[:9]
+
+	case float32:
+		buf[0] = 'f'
+		binary.BigEndian.PutUint32(buf[1:5], math.Float32bits(v))
+		enc = buf[:5]
+
+	case float64:
+		buf[0] = 'd'
+		binary.BigEndian.PutUint64(buf[1:9], math.Float64bits(v))
+		enc = buf[:9]
+
+	case Decimal:
+		buf[0] = 'D'
+		buf[1] = byte(v.Scale)
+		binary.BigEndian.PutUint32(buf[2:6], uint32(v.Value))
+		enc = buf[:6]
+
+	case string:
+		buf[0] = 'S'
+		binary.BigEndian.PutUint32(buf[1:5], uint32(len(v)))
+		enc = append(buf[:5], []byte(v)...)
+
+	case []interface{}: // field-array
+		buf[0] = 'A'
+
+		sec := new(bytes.Buffer)
+		for _, val := range v {
+			if err = writeField(sec, val); err != nil {
+				return
+			}
+		}
+
+		binary.BigEndian.PutUint32(buf[1:5], uint32(sec.Len()))
+		if _, err = w.Write(buf[:5]); err != nil {
+			return
+		}
+
+		if _, err = w.Write(sec.Bytes()); err != nil {
+			return
+		}
+
+		return
+
+	case time.Time:
+		buf[0] = 'T'
+		binary.BigEndian.PutUint64(buf[1:9], uint64(v.Unix()))
+		enc = buf[:9]
+
+	case Table:
+		if _, err = w.Write([]byte{'F'}); err != nil {
+			return
+		}
+		return writeTable(w, v)
+
+	case []byte:
+		buf[0] = 'x'
+		binary.BigEndian.PutUint32(buf[1:5], uint32(len(v)))
+		if _, err = w.Write(buf[0:5]); err != nil {
+			return
+		}
+		if _, err = w.Write(v); err != nil {
+			return
+		}
+		return
+
+	case nil:
+		buf[0] = 'V'
+		enc = buf[:1]
+
+	default:
+		return ErrFieldType
+	}
+
+	_, err = w.Write(enc)
+
+	return
+}
+
+func writeTable(w io.Writer, table Table) (err error) {
+	var buf bytes.Buffer
+
+	for key, val := range table {
+		if err = writeShortstr(&buf, key); err != nil {
+			return
+		}
+		if err = writeField(&buf, val); err != nil {
+			return
+		}
+	}
+
+	return writeLongstr(w, string(buf.Bytes()))
+}

tap/extensions/http/go.mod

+module github.com/up9inc/mizu/tap/extensions/http
+
+go 1.16
+
+require (
+	github.com/google/martian v2.1.0+incompatible
+	github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7
+	github.com/up9inc/mizu/tap/api v0.0.0
+	golang.org/x/net v0.0.0-20210224082022-3d97a244fca7
+	golang.org/x/text v0.3.5 // indirect
+)
+
+replace github.com/up9inc/mizu/tap/api v0.0.0 => ../../api

tap/extensions/http/go.sum

+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7 h1:jkvpcEatpwuMF5O5LVxTnehj6YZ/aEZN4NWD/Xml4pI=
+github.com/romana/rlog v0.0.0-20171115192701-f018bc92e7d7/go.mod h1:KTrHyWpO1sevuXPZwyeZc72ddWRFqNSKDFl7uVWKpg0=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM=
+golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

tap/grpc_assembler.go → tap/extensions/http/grpc_assembler.go

-package tap
+package main
 
 import (
 	"bufio"
@@ -17,17 +17,19 @@ import (
 )
 
 const frameHeaderLen = 9
+
 var clientPreface = []byte(http2.ClientPreface)
+
 const initialHeaderTableSize = 4096
 const protoHTTP2 = "HTTP/2.0"
 const protoMajorHTTP2 = 2
 const protoMinorHTTP2 = 0
 
-var maxHTTP2DataLen int = maxHTTP2DataLenDefault // value initialized during init
+var maxHTTP2DataLen = 1 * 1024 * 1024 // 1MB
 
 type messageFragment struct {
 	headers []hpack.HeaderField
-	data []byte
+	data    []byte
 }
 
 type fragmentsByStream map[uint32]*messageFragment
@@ -46,7 +48,7 @@ func (fbs *fragmentsByStream) appendFrame(streamID uint32, frame http2.Frame) {
 		if existingFragment, ok := (*fbs)[streamID]; ok {
 			existingDataLen := len(existingFragment.data)
 			// Never save more than maxHTTP2DataLen bytes
-			numBytesToAppend := int(math.Min(float64(maxHTTP2DataLen - existingDataLen), float64(newDataLen)))
+			numBytesToAppend := int(math.Min(float64(maxHTTP2DataLen-existingDataLen), float64(newDataLen)))
 
 			existingFragment.data = append(existingFragment.data, frame.Data()[:numBytesToAppend]...)
 		} else {
@@ -69,19 +71,19 @@ func (fbs *fragmentsByStream) pop(streamID uint32) ([]hpack.HeaderField, []byte)
 	return headers, data
 }
 
-func createGrpcAssembler(b *bufio.Reader) GrpcAssembler {
+func createGrpcAssembler(b *bufio.Reader) *GrpcAssembler {
 	var framerOutput bytes.Buffer
 	framer := http2.NewFramer(&framerOutput, b)
 	framer.ReadMetaHeaders = hpack.NewDecoder(initialHeaderTableSize, nil)
-	return GrpcAssembler{
+	return &GrpcAssembler{
 		fragmentsByStream: make(fragmentsByStream),
-		framer: framer,
+		framer:            framer,
 	}
 }
 
 type GrpcAssembler struct {
 	fragmentsByStream fragmentsByStream
-	framer *http2.Framer
+	framer            *http2.Framer
 }
 
 func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
@@ -118,26 +120,26 @@ func (ga *GrpcAssembler) readMessage() (uint32, interface{}, error) {
 	var messageHTTP1 interface{}
 	if _, ok := headersHTTP1[":method"]; ok {
 		messageHTTP1 = http.Request{
-			URL: &url.URL{},
-			Method: "POST",
-			Header: headersHTTP1,
-			Proto: protoHTTP2,
-			ProtoMajor: protoMajorHTTP2,
-			ProtoMinor: protoMinorHTTP2,
-			Body: io.NopCloser(strings.NewReader(dataString)),
+			URL:           &url.URL{},
+			Method:        "POST",
+			Header:        headersHTTP1,
+			Proto:         protoHTTP2,
+			ProtoMajor:    protoMajorHTTP2,
+			ProtoMinor:    protoMinorHTTP2,
+			Body:          io.NopCloser(strings.NewReader(dataString)),
 			ContentLength: int64(len(dataString)),
 		}
 	} else if _, ok := headersHTTP1[":status"]; ok {
 		messageHTTP1 = http.Response{
-			Header: headersHTTP1,
-			Proto: protoHTTP2,
-			ProtoMajor: protoMajorHTTP2,
-			ProtoMinor: protoMinorHTTP2,
-			Body: io.NopCloser(strings.NewReader(dataString)),
+			Header:        headersHTTP1,
+			Proto:         protoHTTP2,
+			ProtoMajor:    protoMajorHTTP2,
+			ProtoMinor:    protoMinorHTTP2,
+			Body:          io.NopCloser(strings.NewReader(dataString)),
 			ContentLength: int64(len(dataString)),
 		}
 	} else {
-		return 0, nil, errors.New("Failed to assemble stream: neither a request nor a message")
+		return 0, nil, errors.New("failed to assemble stream: neither a request nor a message")
 	}
 
 	return streamID, messageHTTP1, nil
@@ -225,7 +227,7 @@ func checkClientPreface(b *bufio.Reader) (bool, error) {
 func discardClientPreface(b *bufio.Reader) error {
 	if isClientPrefacePresent, err := checkClientPreface(b); err != nil {
 		return err
-	} else if !isClientPrefacePresent{
+	} else if !isClientPrefacePresent {
 		return errors.New("discardClientPreface: does not begin with client preface")
 	}
 

tap/extensions/http/handlers.go

+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/romana/rlog"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+func handleHTTP2Stream(grpcAssembler *GrpcAssembler, tcpID *api.TcpID, emitter api.Emitter) error {
+	streamID, messageHTTP1, err := grpcAssembler.readMessage()
+	if err != nil {
+		return err
+	}
+
+	var item *api.OutputChannelItem
+
+	switch messageHTTP1 := messageHTTP1.(type) {
+	case http.Request:
+		ident := fmt.Sprintf(
+			"%s->%s %s->%s %d",
+			tcpID.SrcIP,
+			tcpID.DstIP,
+			tcpID.SrcPort,
+			tcpID.DstPort,
+			streamID,
+		)
+		item = reqResMatcher.registerRequest(ident, &messageHTTP1, time.Now())
+		if item != nil {
+			item.ConnectionInfo = &api.ConnectionInfo{
+				ClientIP:   tcpID.SrcIP,
+				ClientPort: tcpID.SrcPort,
+				ServerIP:   tcpID.DstIP,
+				ServerPort: tcpID.DstPort,
+				IsOutgoing: true,
+			}
+		}
+	case http.Response:
+		ident := fmt.Sprintf(
+			"%s->%s %s->%s %d",
+			tcpID.DstIP,
+			tcpID.SrcIP,
+			tcpID.DstPort,
+			tcpID.SrcPort,
+			streamID,
+		)
+		item = reqResMatcher.registerResponse(ident, &messageHTTP1, time.Now())
+		if item != nil {
+			item.ConnectionInfo = &api.ConnectionInfo{
+				ClientIP:   tcpID.DstIP,
+				ClientPort: tcpID.DstPort,
+				ServerIP:   tcpID.SrcIP,
+				ServerPort: tcpID.SrcPort,
+				IsOutgoing: false,
+			}
+		}
+	}
+
+	if item != nil {
+		item.Protocol = http2Protocol
+		emitter.Emit(item)
+	}
+
+	return nil
+}
+
+func handleHTTP1ClientStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, emitter api.Emitter) error {
+	req, err := http.ReadRequest(b)
+	if err != nil {
+		// log.Println("Error reading stream:", err)
+		return err
+	}
+	counterPair.Request++
+
+	body, err := ioutil.ReadAll(req.Body)
+	req.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
+	s := len(body)
+	if err != nil {
+		rlog.Debugf("[HTTP-request-body] stream %s Got body err: %s", tcpID.Ident, err)
+	}
+	if err := req.Body.Close(); err != nil {
+		rlog.Debugf("[HTTP-request-body-close] stream %s Failed to close request body: %s", tcpID.Ident, err)
+	}
+	encoding := req.Header["Content-Encoding"]
+	rlog.Tracef(1, "HTTP/1 Request: %s %s %s (Body:%d) -> %s", tcpID.Ident, req.Method, req.URL, s, encoding)
+
+	ident := fmt.Sprintf(
+		"%s->%s %s->%s %d",
+		tcpID.SrcIP,
+		tcpID.DstIP,
+		tcpID.SrcPort,
+		tcpID.DstPort,
+		counterPair.Request,
+	)
+	item := reqResMatcher.registerRequest(ident, req, time.Now())
+	if item != nil {
+		item.ConnectionInfo = &api.ConnectionInfo{
+			ClientIP:   tcpID.SrcIP,
+			ClientPort: tcpID.SrcPort,
+			ServerIP:   tcpID.DstIP,
+			ServerPort: tcpID.DstPort,
+			IsOutgoing: true,
+		}
+		emitter.Emit(item)
+	}
+	return nil
+}
+
+func handleHTTP1ServerStream(b *bufio.Reader, tcpID *api.TcpID, counterPair *api.CounterPair, emitter api.Emitter) error {
+	res, err := http.ReadResponse(b, nil)
+	if err != nil {
+		// log.Println("Error reading stream:", err)
+		return err
+	}
+	counterPair.Response++
+
+	body, err := ioutil.ReadAll(res.Body)
+	res.Body = io.NopCloser(bytes.NewBuffer(body)) // rewind
+	s := len(body)
+	if err != nil {
+		rlog.Debugf("[HTTP-response-body] HTTP/%s: failed to get body(parsed len:%d): %s", tcpID.Ident, s, err)
+	}
+	if err := res.Body.Close(); err != nil {
+		rlog.Debugf("[HTTP-response-body-close] HTTP/%s: failed to close body(parsed len:%d): %s", tcpID.Ident, s, err)
+	}
+	sym := ","
+	if res.ContentLength > 0 && res.ContentLength != int64(s) {
+		sym = "!="
+	}
+	contentType, ok := res.Header["Content-Type"]
+	if !ok {
+		contentType = []string{http.DetectContentType(body)}
+	}
+	encoding := res.Header["Content-Encoding"]
+	rlog.Tracef(1, "HTTP/1 Response: %s %s (%d%s%d%s) -> %s", tcpID.Ident, res.Status, res.ContentLength, sym, s, contentType, encoding)
+
+	ident := fmt.Sprintf(
+		"%s->%s %s->%s %d",
+		tcpID.DstIP,
+		tcpID.SrcIP,
+		tcpID.DstPort,
+		tcpID.SrcPort,
+		counterPair.Response,
+	)
+	item := reqResMatcher.registerResponse(ident, res, time.Now())
+	if item != nil {
+		item.ConnectionInfo = &api.ConnectionInfo{
+			ClientIP:   tcpID.DstIP,
+			ClientPort: tcpID.DstPort,
+			ServerIP:   tcpID.SrcIP,
+			ServerPort: tcpID.SrcPort,
+			IsOutgoing: false,
+		}
+		emitter.Emit(item)
+	}
+	return nil
+}

tap/extensions/http/main.go

+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/url"
+
+	"github.com/romana/rlog"
+
+	"github.com/up9inc/mizu/tap/api"
+)
+
+var protocol api.Protocol = api.Protocol{
+	Name:            "http",
+	LongName:        "Hypertext Transfer Protocol -- HTTP/1.1",
+	Abbreviation:    "HTTP",
+	Version:         "1.1",
+	BackgroundColor: "#205cf5",
+	ForegroundColor: "#ffffff",
+	FontSize:        12,
+	ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc2616",
+	Ports:           []string{"80", "8080", "50051"},
+	Priority:        0,
+}
+
+var http2Protocol api.Protocol = api.Protocol{
+	Name:            "http",
+	LongName:        "Hypertext Transfer Protocol Version 2 (HTTP/2) (gRPC)",
+	Abbreviation:    "HTTP/2",
+	Version:         "2.0",
+	BackgroundColor: "#244c5a",
+	ForegroundColor: "#ffffff",
+	FontSize:        11,
+	ReferenceLink:   "https://datatracker.ietf.org/doc/html/rfc7540",
+	Ports:           []string{"80", "8080"},
+	Priority:        0,
+}
+
+const (
+	TypeHttpRequest = iota
+	TypeHttpResponse
+)
+
+func init() {
+	log.Println("Initializing HTTP extension...")
+}
+
+type dissecting string
+
+func (d dissecting) Register(extension *api.Extension) {
+	extension.Protocol = protocol
+	extension.MatcherMap = reqResMatcher.openMessagesMap
+}
+
+func (d dissecting) Ping() {
+	log.Printf("pong %s\n", protocol.Name)
+}
+
+func (d dissecting) Dissect(b *bufio.Reader, isClient bool, tcpID *api.TcpID, counterPair *api.CounterPair, emitter api.Emitter) error {
+	ident := fmt.Sprintf("%s->%s:%s->%s", tcpID.SrcIP, tcpID.DstIP, tcpID.SrcPort, tcpID.DstPort)
+	isHTTP2, err := checkIsHTTP2Connection(b, isClient)
+	if err != nil {
+		rlog.Debugf("[HTTP/2-Prepare-Connection] stream %s Failed to check if client is HTTP/2: %s (%v,%+v)", ident, err, err, err)
+		// Do something?
+	}
+
+	var grpcAssembler *GrpcAssembler
+	if isHTTP2 {
+		err := prepareHTTP2Connection(b, isClient)
+		if err != nil {
+			rlog.Debugf("[HTTP/2-Prepare-Connection-After-Check] stream %s error: %s (%v,%+v)", ident, err, err, err)
+		}
+		grpcAssembler = createGrpcAssembler(b)
+	}
+
+	success := false
+	for {
+		if isHTTP2 {
+			err = handleHTTP2Stream(grpcAssembler, tcpID, emitter)
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			} else if err != nil {
+				rlog.Debugf("[HTTP/2] stream %s error: %s (%v,%+v)", ident, err, err, err)
+				continue
+			}
+			success = true
+		} else if isClient {
+			err = handleHTTP1ClientStream(b, tcpID, counterPair, emitter)
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			} else if err != nil {
+				rlog.Debugf("[HTTP-request] stream %s Request error: %s (%v,%+v)", ident, err, err, err)
+				continue
+			}
+			success = true
+		} else {
+			err = handleHTTP1ServerStream(b, tcpID, counterPair, emitter)
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			} else if err != nil {
+				rlog.Debugf("[HTTP-response], stream %s Response error: %s (%v,%+v)", ident, err, err, err)
+				continue
+			}
+			success = true
+		}
+	}
+
+	if !success {
+		return err
+	}
+	return nil
+}
+
+func SetHostname(address, newHostname string) string {
+	replacedUrl, err := url.Parse(address)
+	if err != nil {
+		log.Printf("error replacing hostname to %s in address %s, returning original %v", newHostname, address, err)
+		return address
+	}
+	replacedUrl.Host = newHostname
+	return replacedUrl.String()
+}
+
+func (d dissecting) Analyze(item *api.OutputChannelItem, entryId string, resolvedSource string, resolvedDestination string) *api.MizuEntry {
+	var host, scheme, authority, path, service string
+
+	request := item.Pair.Request.Payload.(map[string]interface{})
+	response := item.Pair.Response.Payload.(map[string]interface{})
+	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+
+	for _, header := range reqDetails["headers"].([]interface{}) {
+		h := header.(map[string]interface{})
+		if h["name"] == "Host" {
+			host = h["value"].(string)
+		}
+		if h["name"] == ":authority" {
+			authority = h["value"].(string)
+		}
+		if h["name"] == ":scheme" {
+			scheme = h["value"].(string)
+		}
+		if h["name"] == ":path" {
+			path = h["value"].(string)
+		}
+	}
+
+	if item.Protocol.Version == "2.0" {
+		service = fmt.Sprintf("%s://%s", scheme, authority)
+	} else {
+		service = fmt.Sprintf("http://%s", host)
+		path = reqDetails["url"].(string)
+	}
+
+	request["url"] = path
+	if resolvedDestination != "" {
+		service = SetHostname(service, resolvedDestination)
+	} else if resolvedSource != "" {
+		service = SetHostname(service, resolvedSource)
+	}
+	entryBytes, _ := json.Marshal(item.Pair)
+	return &api.MizuEntry{
+		ProtocolName:        protocol.Name,
+		ProtocolVersion:     item.Protocol.Version,
+		EntryId:             entryId,
+		Entry:               string(entryBytes),
+		Url:                 fmt.Sprintf("%s%s", service, path),
+		Method:              reqDetails["method"].(string),
+		Status:              int(resDetails["status"].(float64)),
+		RequestSenderIp:     item.ConnectionInfo.ClientIP,
+		Service:             service,
+		Timestamp:           item.Timestamp,
+		Path:                path,
+		ResolvedSource:      resolvedSource,
+		ResolvedDestination: resolvedDestination,
+		SourceIp:            item.ConnectionInfo.ClientIP,
+		DestinationIp:       item.ConnectionInfo.ServerIP,
+		SourcePort:          item.ConnectionInfo.ClientPort,
+		DestinationPort:     item.ConnectionInfo.ServerPort,
+		IsOutgoing:          item.ConnectionInfo.IsOutgoing,
+	}
+}
+
+func (d dissecting) Summarize(entry *api.MizuEntry) *api.BaseEntryDetails {
+	var p api.Protocol
+	if entry.ProtocolVersion == "2.0" {
+		p = http2Protocol
+	} else {
+		p = protocol
+	}
+	return &api.BaseEntryDetails{
+		Id:              entry.EntryId,
+		Protocol:        p,
+		Url:             entry.Url,
+		RequestSenderIp: entry.RequestSenderIp,
+		Service:         entry.Service,
+		Summary:         entry.Path,
+		StatusCode:      entry.Status,
+		Method:          entry.Method,
+		Timestamp:       entry.Timestamp,
+		SourceIp:        entry.SourceIp,
+		DestinationIp:   entry.DestinationIp,
+		SourcePort:      entry.SourcePort,
+		DestinationPort: entry.DestinationPort,
+		IsOutgoing:      entry.IsOutgoing,
+		Latency:         0,
+		Rules: api.ApplicableRules{
+			Latency: 0,
+			Status:  false,
+		},
+	}
+}
+
+func representRequest(request map[string]interface{}) []interface{} {
+	repRequest := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Method",
+			"value": request["method"].(string),
+		},
+		{
+			"name":  "URL",
+			"value": request["url"].(string),
+		},
+		{
+			"name":  "Body Size",
+			"value": fmt.Sprintf("%g bytes", request["bodySize"].(float64)),
+		},
+	})
+	repRequest = append(repRequest, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	headers, _ := json.Marshal(request["headers"].([]interface{}))
+	repRequest = append(repRequest, map[string]string{
+		"type":  "table",
+		"title": "Headers",
+		"data":  string(headers),
+	})
+
+	cookies, _ := json.Marshal(request["cookies"].([]interface{}))
+	repRequest = append(repRequest, map[string]string{
+		"type":  "table",
+		"title": "Cookies",
+		"data":  string(cookies),
+	})
+
+	queryString, _ := json.Marshal(request["queryString"].([]interface{}))
+	repRequest = append(repRequest, map[string]string{
+		"type":  "table",
+		"title": "Query String",
+		"data":  string(queryString),
+	})
+
+	postData, _ := request["postData"].(map[string]interface{})
+	mimeType, _ := postData["mimeType"]
+	if mimeType == nil || len(mimeType.(string)) == 0 {
+		mimeType = "text/html"
+	}
+	text, _ := postData["text"]
+	if text != nil {
+		repRequest = append(repRequest, map[string]string{
+			"type":      "body",
+			"title":     "POST Data (text/plain)",
+			"encoding":  "",
+			"mime_type": mimeType.(string),
+			"data":      text.(string),
+		})
+	}
+
+	if postData["params"] != nil {
+		params, _ := json.Marshal(postData["params"].([]interface{}))
+		if len(params) > 0 {
+			if mimeType == "multipart/form-data" {
+				multipart, _ := json.Marshal([]map[string]string{
+					{
+						"name":  "Files",
+						"value": string(params),
+					},
+				})
+				repRequest = append(repRequest, map[string]string{
+					"type":  "table",
+					"title": "POST Data (multipart/form-data)",
+					"data":  string(multipart),
+				})
+			} else {
+				repRequest = append(repRequest, map[string]string{
+					"type":  "table",
+					"title": "POST Data (application/x-www-form-urlencoded)",
+					"data":  string(params),
+				})
+			}
+		}
+	}
+
+	return repRequest
+}
+
+func representResponse(response map[string]interface{}) []interface{} {
+	repResponse := make([]interface{}, 0)
+
+	details, _ := json.Marshal([]map[string]string{
+		{
+			"name":  "Status",
+			"value": fmt.Sprintf("%g", response["status"].(float64)),
+		},
+		{
+			"name":  "Status Text",
+			"value": response["statusText"].(string),
+		},
+		{
+			"name":  "Body Size",
+			"value": fmt.Sprintf("%g bytes", response["bodySize"].(float64)),
+		},
+	})
+	repResponse = append(repResponse, map[string]string{
+		"type":  "table",
+		"title": "Details",
+		"data":  string(details),
+	})
+
+	headers, _ := json.Marshal(response["headers"].([]interface{}))
+	repResponse = append(repResponse, map[string]string{
+		"type":  "table",
+		"title": "Headers",
+		"data":  string(headers),
+	})
+
+	cookies, _ := json.Marshal(response["cookies"].([]interface{}))
+	repResponse = append(repResponse, map[string]string{
+		"type":  "table",
+		"title": "Cookies",
+		"data":  string(cookies),
+	})
+
+	content, _ := response["content"].(map[string]interface{})
+	mimeType, _ := content["mimeType"]
+	if mimeType == nil || len(mimeType.(string)) == 0 {
+		mimeType = "text/html"
+	}
+	encoding, _ := content["encoding"]
+	text, _ := content["text"]
+	if text != nil {
+		repResponse = append(repResponse, map[string]string{
+			"type":      "body",
+			"title":     "Body",
+			"encoding":  encoding.(string),
+			"mime_type": mimeType.(string),
+			"data":      text.(string),
+		})
+	}
+
+	return repResponse
+}
+
+func (d dissecting) Represent(entry *api.MizuEntry) (api.Protocol, []byte, error) {
+	var p api.Protocol
+	if entry.ProtocolVersion == "2.0" {
+		p = http2Protocol
+	} else {
+		p = protocol
+	}
+	var root map[string]interface{}
+	json.Unmarshal([]byte(entry.Entry), &root)
+	representation := make(map[string]interface{}, 0)
+	request := root["request"].(map[string]interface{})["payload"].(map[string]interface{})
+	response := root["response"].(map[string]interface{})["payload"].(map[string]interface{})
+	reqDetails := request["details"].(map[string]interface{})
+	resDetails := response["details"].(map[string]interface{})
+	repRequest := representRequest(reqDetails)
+	repResponse := representResponse(resDetails)
+	representation["request"] = repRequest
+	representation["response"] = repResponse
+	object, err := json.Marshal(representation)
+	return p, object, err
+}
+
+var Dissector dissecting

tap/http_matcher.go → tap/extensions/http/matcher.go

-package tap
+package main
 
 import (
 	"fmt"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/orcaman/concurrent-map"
-)
-
-type requestResponsePair struct {
-	Request  httpMessage `json:"request"`
-	Response httpMessage `json:"response"`
-}
+	"github.com/romana/rlog"
 
-type httpMessage struct {
-	isRequest      bool
-	captureTime    time.Time
-	orig           interface{}
-}
+	"github.com/up9inc/mizu/tap/api"
+)
 
+var reqResMatcher = createResponseRequestMatcher() // global
 
-// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}
+// Key is {client_addr}:{client_port}->{dest_addr}:{dest_port}_{incremental_counter}
 type requestResponseMatcher struct {
-	openMessagesMap cmap.ConcurrentMap
-
+	openMessagesMap *sync.Map
 }
 
 func createResponseRequestMatcher() requestResponseMatcher {
-	newMatcher := &requestResponseMatcher{openMessagesMap: cmap.New()}
+	newMatcher := &requestResponseMatcher{openMessagesMap: &sync.Map{}}
 	return *newMatcher
 }
 
-func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *requestResponsePair {
+func (matcher *requestResponseMatcher) registerRequest(ident string, request *http.Request, captureTime time.Time) *api.OutputChannelItem {
 	split := splitIdent(ident)
 	key := genKey(split)
 
-	requestHTTPMessage := httpMessage{
-		isRequest:      true,
-		captureTime:    captureTime,
-		orig:           request,
+	requestHTTPMessage := api.GenericMessage{
+		IsRequest:   true,
+		CaptureTime: captureTime,
+		Payload: HTTPPayload{
+			Type: TypeHttpRequest,
+			Data: request,
+		},
 	}
 
-	if response, found := matcher.openMessagesMap.Pop(key); found {
-		// Type assertion always succeeds because all of the map's values are of httpMessage type
-		responseHTTPMessage := response.(*httpMessage)
-		if responseHTTPMessage.isRequest {
-			SilentError("Request-Duplicate", "Got duplicate request with same identifier")
+	if response, found := matcher.openMessagesMap.LoadAndDelete(key); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		responseHTTPMessage := response.(*api.GenericMessage)
+		if responseHTTPMessage.IsRequest {
+			rlog.Debugf("[Request-Duplicate] Got duplicate request with same identifier")
 			return nil
 		}
-		Trace("Matched open Response for %s", key)
+		rlog.Tracef(1, "Matched open Response for %s", key)
 		return matcher.preparePair(&requestHTTPMessage, responseHTTPMessage)
 	}
 
-	matcher.openMessagesMap.Set(key, &requestHTTPMessage)
-	Trace("Registered open Request for %s", key)
+	matcher.openMessagesMap.Store(key, &requestHTTPMessage)
+	rlog.Tracef(1, "Registered open Request for %s", key)
 	return nil
 }
 
-func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *requestResponsePair {
+func (matcher *requestResponseMatcher) registerResponse(ident string, response *http.Response, captureTime time.Time) *api.OutputChannelItem {
 	split := splitIdent(ident)
 	key := genKey(split)
 
-	responseHTTPMessage := httpMessage{
-		isRequest:   false,
-		captureTime: captureTime,
-		orig:        response,
+	responseHTTPMessage := api.GenericMessage{
+		IsRequest:   false,
+		CaptureTime: captureTime,
+		Payload: HTTPPayload{
+			Type: TypeHttpResponse,
+			Data: response,
+		},
 	}
 
-	if request, found := matcher.openMessagesMap.Pop(key); found {
-		// Type assertion always succeeds because all of the map's values are of httpMessage type
-		requestHTTPMessage := request.(*httpMessage)
-		if !requestHTTPMessage.isRequest {
-			SilentError("Response-Duplicate", "Got duplicate response with same identifier")
+	if request, found := matcher.openMessagesMap.LoadAndDelete(key); found {
+		// Type assertion always succeeds because all of the map's values are of api.GenericMessage type
+		requestHTTPMessage := request.(*api.GenericMessage)
+		if !requestHTTPMessage.IsRequest {
+			rlog.Debugf("[Response-Duplicate] Got duplicate response with same identifier")
 			return nil
 		}
-		Trace("Matched open Request for %s", key)
+		rlog.Tracef(1, "Matched open Request for %s", key)
 		return matcher.preparePair(requestHTTPMessage, &responseHTTPMessage)
 	}
 
-	matcher.openMessagesMap.Set(key, &responseHTTPMessage)
-	Trace("Registered open Response for %s", key)
+	matcher.openMessagesMap.Store(key, &responseHTTPMessage)
+	rlog.Tracef(1, "Registered open Response for %s", key)
 	return nil
 }
 
-func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *httpMessage, responseHTTPMessage *httpMessage) *requestResponsePair {
-	return &requestResponsePair{
-		Request:  *requestHTTPMessage,
-		Response: *responseHTTPMessage,
+func (matcher *requestResponseMatcher) preparePair(requestHTTPMessage *api.GenericMessage, responseHTTPMessage *api.GenericMessage) *api.OutputChannelItem {
+	return &api.OutputChannelItem{
+		Protocol:       protocol,
+		Timestamp:      time.Now().UnixNano() / int64(time.Millisecond),
+		ConnectionInfo: nil,
+		Pair: &api.RequestResponsePair{
+			Request:  *requestHTTPMessage,
+			Response: *responseHTTPMessage,
+		},
 	}
 }
 
@@ -100,23 +103,3 @@ func genKey(split []string) string {
 	key := fmt.Sprintf("%s:%s->%s:%s,%s", split[0], split[2], split[1], split[3], split[4])
 	return key
 }
-
-func (matcher *requestResponseMatcher) deleteOlderThan(t time.Time) int {
-	keysToPop := make([]string, 0)
-	for item := range matcher.openMessagesMap.IterBuffered() {
-		// Map only contains values of type httpMessage
-		message, _ := item.Val.(*httpMessage)
-
-		if message.captureTime.Before(t) {
-			keysToPop = append(keysToPop, item.Key)
-		}
-	}
-
-	numDeleted := len(keysToPop)
-	
-	for _, key := range keysToPop {
-		_, _ = matcher.openMessagesMap.Pop(key)
-	}
-
-	return numDeleted
-}