"README.md" did not exist on "08a8e2f3bb5568c647626b087415485930ecd1a7"
Unverified Commit 27fa0afb authored by M. Mert Yıldıran's avatar M. Mert Yıldıran Committed by GitHub
Browse files

TRA-4331 Implement full data streaming over WebSocket (#819)

* Implement full data streaming over WebSocket

* Fix the linting error

* Make the empty being the criteria

* Use a label to break the nested loop
parent c98c99e4
develop Fix_images_imports Remove-the-using-of-leftOff-from-websocket TRA-4276_Folder_structure_refactor TRA-4579_Fix_acceptance_test_of_right_panel_checkboxes TRA-4602_Timeline_bars_to_traffic_statistics TRA-4612_Traffic_stats_add_time_range_filter UI/Service-map-GUI-improvements UI/fix/general-gui-fixes UI/servicemap-dynamic-protocol-filters Update_material_ui_to_v5 acceptance_test_by_demand bug/TRA-4387_debugging bug/TRA-4387_fix_tapping_status bug/ui/TRA-4437_line_numbers_checkbox_not_always_working bug/ui/TRA-4513_grpc_heading_overlap_request_size- ci_improvments codegen-placeholders common_migration debug/disable_redaction debug/oas_no_reset debug/profile debug/profile_nimrod debug/profile_nimrod_stable debug/profile_tmp debug/reading-channel debug/triggering_tcp_kprobe_with_go debug/verbose_watch_logs example-limiting feat/acceptance-test-image-caching feat/accumulative-stats feat/accumulative-stats-with-time-frame feat/add-nginx feat/add-option-to-pass-context feat/add-protocols-to-the-endpoint feat/add-timing-endpoint-for-stats feat/afpacket feat/amqp-req-res-matcher feat/basenine-mongodb feat/basenine-mongodb-mess feat/basenine-mongodb-roee feat/basenine-xml-helper feat/bfl-syntax-highlighting feat/capture-source-indicator feat/cypress-10-migration feat/derive-summary-method-fields feat/docker-from-arm64-to-amd64 feat/ebpf-arm64 feat/ebpf-go-abi0 feat/flush-reset feat/golang-tls feat/graphql feat/improve-go-tls-address-availability feat/insertion-filter feat/install_helm_rename feat/log-total-active-tcp-streams-and-goroutines feat/merge-stats-endpoints-and-add-auto-interval feat/namespace-field-ui feat/oas-parameter-patterns feat/query-fetch-first feat/reclustering feat/reply-http-endpoint feat/request-response-size feature/TRA-4320_common_entries_controller_code feature/TRA-4323_depedency_injection_oas_servicemap feature/TRA-4355_cleanup_install feature/TRA-4365_Support_multiple_workspaces feature/TRA-4442_UI_performance feature/TRA-4602_Traffic_statistics feature/TRA-4622_Remove_rules_feature_UI feature/change_redact_to_opt_in feature/close-finished-live-streams feature/close-gopacket-conn-immediately feature/david-poc feature/elasticsearch feature/fix-service-mesh feature/fix-tls-missing-address feature/fix-tls-not-listening feature/improve_tls_info_with_kprobes feature/limit-fd-to-address-map feature/remove_contract feature/remove_duplicate_data feature/remove_redundant_field feature/remove_rules feature/reorginize-some-tls-code-after-refactor feature/service-mesh-badge feature/set-pcap-bpf-filter feature/stop-tapping-self-tapper-traffic feature/support-tcp-keepalive feature/support-tls-big-buffers feature/throttling feature/throttling-by-live-streams feature/tls-ebpf-error-handling feature/ui/TRA-4519_oas_searchable_dropdown fix/TRA-4396_check_pull_image_flag fix/acceptance-tests-fetch-50 fix/acceptance-tests-fetch-wait fix/acceptance-tests-setup-improve fix/add-tests-for-buckets-statistics fix/add-timing-params-to-stats-endpoint fix/amqp-nil-response-payload fix/amqp-tests fix/better-close-other-dissectors fix/better-close-other-dissectors-tmp-1 fix/check-filter-by-method fix/cli-windows-build fix/entry-title-responsive fix/eslint_warning fix/go-ebpf-arm64 fix/go-ebpf-log-level fix/idle-cpu-usage fix/initialize_tapper_before_ws fix/kafka-api-key-name fix/log-when-cannot-start-tappers fix/no_tls_on_enterprise fix/oas-service-names fix/regration-fixes-24.3 fix/remove-grpc-related-mods fix/remove-up9-analyze-feature fix/req-res-matcher fix/some-golang-inspection-solutions fix/spawn-only-two-goroutines fix/support_workspaces_in_oas fix/tls-tapper-client-hello-log-revert fix/toleration-to-api-server fix/ui/Service-map-GUI-issues_TRA-4499 fix/ui/Service-map-filtering-is-not-updated-after-refresh_TRA-4497 fix/ui/ServiceMapModal-filters fix/ui/service-graph-protocols-unSelection-_TRA-4522 fix/update_tap_targets_over_ws githubactions_test hotfix/TRA-4451_TLS_icon_position_develop hotfix/TRA-4451_fix_TLS_icon_position main master move_general_functions_from_timeline_stats_to_helpers oas-feed-rework react_and_node_major_packages_upgrades readme_change refactor_ws revert-941-develop support_stop_oas_service_map task/add_info_to_acceptance_test_slack task/delete-ebpf-object-files test-leon test/permission-acceptance test/service_map_test tmp-acceptance-tests ui-common_package_update_143 ui/-download-request-replay ui/-replay-phase-2 ui/Link ui/Service-map-GUI-improvements ui/Service-map-split-to-ui-common ui/TRA-4255_show-50-recent-entries-and-continue-to-stream ui/TRA-4256_gui_oas_window ui/TRA-4406_add-an-information-icon ui/add-insertion-filter-to-settings-page ui/add-props-to-checkbox ui/fix/selectList-sticky-header-fix ui/fix/show-50-recent-records-on-load ui/mizu-ui-style-lint ui/replay-mizu-requests ui/with-loading ui_common_local_build 37.0 37.0-dev2 37.0-dev1 37.0-dev0 36.0 36.0-dev27 36.0-dev26 36.0-dev25 36.0-dev24 36.0-dev23 36.0-dev22 36.0-dev21 36.0-dev20 36.0-dev19 36.0-dev18 36.0-dev17 36.0-dev16 36.0-dev15 36.0-dev14 36.0-dev13 36.0-dev12 36.0-dev11 36.0-dev10 36.0-dev9 36.0-dev8 36.0-dev7 36.0-dev6 36.0-dev5 36.0-dev4 36.0-dev3 36.0-dev2 36.0-dev1 36.0-dev0 35.1 35.0 35.0-dev22 35.0-dev21 35.0-dev20 35.0-dev19 35.0-dev18 35.0-dev17 35.0-dev16 35.0-dev15 35.0-dev14 35.0-dev13 35.0-dev12 35.0-dev11 35.0-dev10 35.0-dev9 35.0-dev8 35.0-dev7 35.0-dev6 35.0-dev5 35.0-dev4 35.0-dev3 35.0-dev2 35.0-dev1 35.0-dev0 34.0 34.0-dev14 34.0-dev13 34.0-dev12 34.0-dev11 34.0-dev10 34.0-dev9 34.0-dev8 34.0-dev7 34.0-dev6 34.0-dev5 34.0-dev4 34.0-dev3 34.0-dev2 34.0-dev1 34.0-dev0 33.1 33.0 33.0-dev40 33.0-dev39 33.0-dev38 33.0-dev37 33.0-dev36 33.0-dev35 33.0-dev34 33.0-dev33 33.0-dev32 33.0-dev31 33.0-dev30 33.0-dev29 33.0-dev28 33.0-dev27 33.0-dev26 33.0-dev25 33.0-dev24 33.0-dev23 33.0-dev22 33.0-dev21 33.0-dev20 33.0-dev19 33.0-dev18 33.0-dev17 33.0-dev16 33.0-dev15 33.0-dev14 33.0-dev13 33.0-dev12 33.0-dev11 33.0-dev10 33.0-dev9 33.0-dev8 33.0-dev7 33.0-dev6 33.0-dev5 33.0-dev4 33.0-dev3 33.0-dev2 33.0-dev1 33.0-dev0 32.0 32.0-dev23 32.0-dev22 32.0-dev21 32.0-dev20 32.0-dev19 32.0-dev18 32.0-dev17 32.0-dev16 32.0-dev15 32.0-dev14 32.0-dev13 32.0-dev12 32.0-dev11 32.0-dev10 32.0-dev9 32.0-dev8 32.0-dev7 32.0-dev6 32.0-dev5 32.0-dev4 32.0-dev3 32.0-dev2 32.0-dev1 32.0-dev0 31.1 31.0 31.0-dev68 31.0-dev67 31.0-dev66 31.0-dev65 31.0-dev64 31.0-dev63 31.0-dev62 31.0-dev61 31.0-dev60 31.0-dev59 31.0-dev58 31.0-dev57 31.0-dev56 31.0-dev55 31.0-dev54 31.0-dev53 31.0-dev52 31.0-dev51 31.0-dev50 31.0-dev49 31.0-dev48 31.0-dev47 31.0-dev46 31.0-dev45 31.0-dev44 31.0-dev43 31.0-dev42 31.0-dev41 31.0-dev40 31.0-dev39 31.0-dev38 31.0-dev37 31.0-dev36 31.0-dev35 31.0-dev34 31.0-dev33 31.0-dev32 31.0-dev31 31.0-dev30 31.0-dev29 31.0-dev28 31.0-dev27 31.0-dev26 31.0-dev25 31.0-dev24 31.0-dev23 31.0-dev22 31.0-dev21 31.0-dev20 31.0-dev19 31.0-dev18 31.0-dev17 31.0-dev16 31.0-dev15 31.0-dev14 31.0-dev13 31.0-dev12 31.0-dev11 31.0-dev10 31.0-dev9 31.0-dev8 31.0-dev7 31.0-dev6 31.0-dev5 31.0-dev4 31.0-dev3 31.0-dev2 31.0-dev1 31.0-dev0 30.4 30.3 30.2 30.1 30.0 30.0-dev38 30.0-dev37 30.0-dev36 30.0-dev35 30.0-dev34 30.0-dev33 30.0-dev32 30.0-dev31 30.0-dev30 30.0-dev29 30.0-dev28 30.0-dev27 30.0-dev26 30.0-dev25 30.0-dev24 30.0-dev23 30.0-dev22 30.0-dev21 30.0-dev20 30.0-dev19 30.0-dev18 30.0-dev17 30.0-dev16 30.0-dev15 30.0-dev14 30.0-dev13 30.0-dev12 30.0-dev11 30.0-dev10 30.0-dev9 30.0-dev8 30.0-dev7 30.0-dev6 30.0-dev5 30.0-dev4 30.0-dev3 30.0-dev2 30.0-dev1 30.0-dev0 29.0 29.0-dev23 29.0-dev22 29.0-dev21 29.0-dev20 29.0-dev19 29.0-dev18 29.0-dev17 29.0-dev16 29.0-dev15 29.0-dev14 29.0-dev13 29.0-dev12 29.0-dev11 29.0-dev10 29.0-dev9 29.0-dev8 29.0-dev7 29.0-dev6 29.0-dev5 29.0-dev4 29.0-dev3 29.0-dev2 29.0-dev1 29.0-dev0 28.0 28.0-dev32 28.0-dev31 28.0-dev30 28.0-dev29 28.0-dev28 28.0-dev27 28.0-dev26 28.0-dev25 28.0-dev24 28.0-dev23 28.0-dev22 28.0-dev21 28.0-dev20 28.0-dev19 28.0-dev18 28.0-dev17 28.0-dev16 28.0-dev15 28.0-dev14 28.0-dev13 28.0-dev12 28.0-dev11 28.0-dev10 28.0-dev9
No related merge requests found
Showing with 45 additions and 12 deletions
+45 -12
...@@ -110,20 +110,31 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even ...@@ -110,20 +110,31 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
logger.Log.Error(err) logger.Log.Error(err)
} }
out:
for { for {
_, msg, err := ws.ReadMessage() // params[0]: query
if err != nil { // params[1]: enableFullEntries (empty: disable, non-empty: enable)
if _, ok := err.(*websocket.CloseError); ok { params := make([][]byte, 2)
logger.Log.Debugf("Received websocket close message, socket id: %d", socketId) for i := range params {
} else { _, params[i], err = ws.ReadMessage()
logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err) if err != nil {
if _, ok := err.(*websocket.CloseError); ok {
logger.Log.Debugf("Received websocket close message, socket id: %d", socketId)
} else {
logger.Log.Errorf("Error reading message, socket id: %d, error: %v", socketId, err)
}
break out
} }
}
break enableFullEntries := false
if len(params[1]) > 0 {
enableFullEntries = true
} }
if !isTapper && !isQuerySet { if !isTapper && !isQuerySet {
query := string(msg) query := string(params[0])
err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query) err = basenine.Validate(shared.BasenineHost, shared.BaseninePort, query)
if err != nil { if err != nil {
toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{ toastBytes, _ := models.CreateWebsocketToastMessage(&models.ToastMessage{
...@@ -150,10 +161,15 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even ...@@ -150,10 +161,15 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
var entry *tapApi.Entry var entry *tapApi.Entry
err = json.Unmarshal(bytes, &entry) err = json.Unmarshal(bytes, &entry)
base := tapApi.Summarize(entry) var message []byte
if enableFullEntries {
message, _ = models.CreateFullEntryWebSocketMessage(entry)
} else {
base := tapApi.Summarize(entry)
message, _ = models.CreateBaseEntryWebSocketMessage(base)
}
baseEntryBytes, _ := models.CreateBaseEntryWebSocketMessage(base) if err := SendToSocket(socketId, message); err != nil {
if err := SendToSocket(socketId, baseEntryBytes); err != nil {
logger.Log.Error(err) logger.Log.Error(err)
} }
} }
...@@ -185,7 +201,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even ...@@ -185,7 +201,7 @@ func websocketHandler(w http.ResponseWriter, r *http.Request, eventHandlers Even
connection.Query(query, data, meta) connection.Query(query, data, meta)
} else { } else {
eventHandlers.WebSocketMessage(socketId, msg) eventHandlers.WebSocketMessage(socketId, params[0])
} }
} }
} }
......
...@@ -42,6 +42,11 @@ type WebSocketEntryMessage struct { ...@@ -42,6 +42,11 @@ type WebSocketEntryMessage struct {
Data *tapApi.BaseEntry `json:"data,omitempty"` Data *tapApi.BaseEntry `json:"data,omitempty"`
} }
type WebSocketFullEntryMessage struct {
*shared.WebSocketMessageMetadata
Data *tapApi.Entry `json:"data,omitempty"`
}
type WebSocketTappedEntryMessage struct { type WebSocketTappedEntryMessage struct {
*shared.WebSocketMessageMetadata *shared.WebSocketMessageMetadata
Data *tapApi.OutputChannelItem Data *tapApi.OutputChannelItem
...@@ -88,6 +93,16 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) { ...@@ -88,6 +93,16 @@ func CreateBaseEntryWebSocketMessage(base *tapApi.BaseEntry) ([]byte, error) {
return json.Marshal(message) return json.Marshal(message)
} }
func CreateFullEntryWebSocketMessage(entry *tapApi.Entry) ([]byte, error) {
message := &WebSocketFullEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
MessageType: shared.WebSocketMessageTypeFullEntry,
},
Data: entry,
}
return json.Marshal(message)
}
func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) { func CreateWebsocketTappedEntryMessage(base *tapApi.OutputChannelItem) ([]byte, error) {
message := &WebSocketTappedEntryMessage{ message := &WebSocketTappedEntryMessage{
WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{ WebSocketMessageMetadata: &shared.WebSocketMessageMetadata{
......
...@@ -15,6 +15,7 @@ type WebSocketMessageType string ...@@ -15,6 +15,7 @@ type WebSocketMessageType string
const ( const (
WebSocketMessageTypeEntry WebSocketMessageType = "entry" WebSocketMessageTypeEntry WebSocketMessageType = "entry"
WebSocketMessageTypeFullEntry WebSocketMessageType = "fullEntry"
WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry" WebSocketMessageTypeTappedEntry WebSocketMessageType = "tappedEntry"
WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status" WebSocketMessageTypeUpdateStatus WebSocketMessageType = "status"
WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus" WebSocketMessageTypeAnalyzeStatus WebSocketMessageType = "analyzeStatus"
......
...@@ -122,6 +122,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus}) => { ...@@ -122,6 +122,7 @@ export const TrafficPage: React.FC<TrafficPageProps> = ({setAnalyzeStatus}) => {
ws.current.onopen = () => { ws.current.onopen = () => {
setWsConnection(WsConnectionStatus.Connected); setWsConnection(WsConnectionStatus.Connected);
ws.current.send(query); ws.current.send(query);
ws.current.send("");
} }
ws.current.onclose = () => { ws.current.onclose = () => {
setWsConnection(WsConnectionStatus.Closed); setWsConnection(WsConnectionStatus.Closed);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment