Unverified Commit 3ed9bc1e authored by gadotroee's avatar gadotroee Committed by GitHub
Browse files

Upgrade Basenine version to `v0.7.1` (#984)

* Basenine MongoDB mess

* Fix more

* Fix the `mongodb` container arguments

* Add Basenine ARM64 binary

* Make the changes related to `leftOff` becoming a string value

* Make `leftOffTop` state string

* Handle `CloseConnection` in `Fetch`

* Upgrade Basenine to `0.7.0`

* Revert the changes in `package.json` and `package-lock.json`

* Fix the `Dockerfile`

* Remove the binaries

* Increase the Basenine up deadline to 20 seconds

* Revert the changes in `shared/kubernetes/provider.go`

* Fix the OAS generator tests

* Protect from race condition

* Fix mutexes

* Fix unlock

* Fix logging data types

* Try to stabilize the tests

* Remove the `replace` statement

* revert the change the done in https://github.com/up9inc/mizu/pull/984/commits/2899414f2b82ed8fdb2e84a7b6aa00f8e8b8f5e0 to not change the leftOff

* Change `leftOffBottom` empty string default value to `latest`

* Upgrade Basenine to `0.7.1`

* Handle t...
parent 86e5dcea
develop Fix_images_imports Remove-the-using-of-leftOff-from-websocket TRA-4276_Folder_structure_refactor TRA-4579_Fix_acceptance_test_of_right_panel_checkboxes TRA-4602_Timeline_bars_to_traffic_statistics TRA-4612_Traffic_stats_add_time_range_filter UI/Service-map-GUI-improvements UI/fix/general-gui-fixes UI/servicemap-dynamic-protocol-filters Update_material_ui_to_v5 acceptance_test_by_demand bug/ui/TRA-4437_line_numbers_checkbox_not_always_working bug/ui/TRA-4513_grpc_heading_overlap_request_size- codegen-placeholders debug/disable_redaction debug/profile debug/profile_nimrod debug/profile_nimrod_stable debug/profile_tmp debug/triggering_tcp_kprobe_with_go example-limiting feat/acceptance-test-image-caching feat/accumulative-stats feat/accumulative-stats-with-time-frame feat/add-nginx feat/add-protocols-to-the-endpoint feat/add-timing-endpoint-for-stats feat/afpacket feat/amqp-req-res-matcher feat/bfl-syntax-highlighting feat/cypress-10-migration feat/ebpf-arm64 feat/ebpf-go-abi0 feat/golang-tls feat/graphql feat/improve-go-tls-address-availability feat/log-total-active-tcp-streams-and-goroutines feat/merge-stats-endpoints-and-add-auto-interval feat/query-fetch-first feat/reply-http-endpoint feature/TRA-4602_Traffic_statistics feature/TRA-4622_Remove_rules_feature_UI feature/change_redact_to_opt_in feature/close-finished-live-streams feature/close-gopacket-conn-immediately feature/david-poc feature/elasticsearch feature/fix-tls-not-listening feature/improve_tls_info_with_kprobes feature/limit-fd-to-address-map feature/remove_contract feature/remove_duplicate_data feature/remove_redundant_field feature/remove_rules feature/reorginize-some-tls-code-after-refactor feature/stop-tapping-self-tapper-traffic feature/throttling feature/throttling-by-live-streams feature/ui/TRA-4519_oas_searchable_dropdown fix/acceptance-tests-fetch-50 fix/acceptance-tests-fetch-wait fix/acceptance-tests-setup-improve fix/add-tests-for-buckets-statistics fix/add-timing-params-to-stats-endpoint fix/amqp-nil-response-payload fix/amqp-tests fix/better-close-other-dissectors fix/better-close-other-dissectors-tmp-1 fix/check-filter-by-method fix/cli-windows-build fix/eslint_warning fix/go-ebpf-arm64 fix/go-ebpf-log-level fix/kafka-api-key-name fix/log-when-cannot-start-tappers fix/remove-up9-analyze-feature fix/req-res-matcher fix/some-golang-inspection-solutions fix/spawn-only-two-goroutines fix/tls-tapper-client-hello-log-revert fix/toleration-to-api-server fix/ui/Service-map-GUI-issues_TRA-4499 fix/ui/Service-map-filtering-is-not-updated-after-refresh_TRA-4497 fix/ui/service-graph-protocols-unSelection-_TRA-4522 githubactions_test main master move_general_functions_from_timeline_stats_to_helpers react_and_node_major_packages_upgrades task/delete-ebpf-object-files tmp-acceptance-tests ui/-download-request-replay ui/-replay-phase-2 ui/Link ui/Service-map-GUI-improvements ui/TRA-4255_show-50-recent-entries-and-continue-to-stream ui/add-insertion-filter-to-settings-page ui/fix/selectList-sticky-header-fix ui/mizu-ui-style-lint ui/replay-mizu-requests ui/with-loading 37.0 37.0-dev2 37.0-dev1 37.0-dev0 36.0 36.0-dev27 36.0-dev26 36.0-dev25 36.0-dev24 36.0-dev23 36.0-dev22 36.0-dev21 36.0-dev20 36.0-dev19 36.0-dev18 36.0-dev17 36.0-dev16 36.0-dev15 36.0-dev14 36.0-dev13 36.0-dev12 36.0-dev11 36.0-dev10 36.0-dev9 36.0-dev8 36.0-dev7 36.0-dev6 36.0-dev5 36.0-dev4 36.0-dev3 36.0-dev2 36.0-dev1 36.0-dev0 35.1 35.0 35.0-dev22 35.0-dev21 35.0-dev20 35.0-dev19 35.0-dev18 35.0-dev17 35.0-dev16 35.0-dev15 35.0-dev14 35.0-dev13 35.0-dev12 35.0-dev11 35.0-dev10 35.0-dev9 35.0-dev8 35.0-dev7 35.0-dev6 35.0-dev5 35.0-dev4 35.0-dev3 35.0-dev2 35.0-dev1 35.0-dev0 34.0 34.0-dev14 34.0-dev13 34.0-dev12 34.0-dev11 34.0-dev10 34.0-dev9 34.0-dev8 34.0-dev7 34.0-dev6 34.0-dev5 34.0-dev4 34.0-dev3 34.0-dev2 34.0-dev1 34.0-dev0 33.1 33.0 33.0-dev40 33.0-dev39 33.0-dev38 33.0-dev37 33.0-dev36 33.0-dev35 33.0-dev34 33.0-dev33 33.0-dev32 33.0-dev31 33.0-dev30 33.0-dev29 33.0-dev28 33.0-dev27 33.0-dev26 33.0-dev25 33.0-dev24 33.0-dev23 33.0-dev22 33.0-dev21 33.0-dev20 33.0-dev19 33.0-dev18 33.0-dev17 33.0-dev16 33.0-dev15 33.0-dev14 33.0-dev13 33.0-dev12 33.0-dev11 33.0-dev10 33.0-dev9 33.0-dev8 33.0-dev7 33.0-dev6 33.0-dev5 33.0-dev4 33.0-dev3 33.0-dev2 33.0-dev1 33.0-dev0 32.0 32.0-dev23 32.0-dev22 32.0-dev21 32.0-dev20 32.0-dev19 32.0-dev18 32.0-dev17 32.0-dev16 32.0-dev15 32.0-dev14 32.0-dev13 32.0-dev12 32.0-dev11 32.0-dev10 32.0-dev9 32.0-dev8 32.0-dev7 32.0-dev6 32.0-dev5 32.0-dev4 32.0-dev3 32.0-dev2 32.0-dev1 32.0-dev0 31.1 31.0 31.0-dev68 31.0-dev67 31.0-dev66 31.0-dev65 31.0-dev64 31.0-dev63 31.0-dev62 31.0-dev61 31.0-dev60 31.0-dev59 31.0-dev58 31.0-dev57 31.0-dev56 31.0-dev55 31.0-dev54 31.0-dev53 31.0-dev52 31.0-dev51 31.0-dev50 31.0-dev49 31.0-dev48 31.0-dev47 31.0-dev46 31.0-dev45 31.0-dev44 31.0-dev43 31.0-dev42 31.0-dev41 31.0-dev40 31.0-dev39 31.0-dev38
No related merge requests found
Showing with 3393 additions and 107 deletions
+3393 -107
......@@ -87,8 +87,8 @@ RUN go build -ldflags="-extldflags=-static -s -w \
-X 'github.com/up9inc/mizu/agent/pkg/version.Ver=${VER}'" -o mizuagent .
# Download Basenine executable, verify the sha1sum
ADD https://github.com/up9inc/basenine/releases/download/v0.6.6/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.6.6/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
ADD https://github.com/up9inc/basenine/releases/download/v0.7.1/basenine_linux_${GOARCH} ./basenine_linux_${GOARCH}
ADD https://github.com/up9inc/basenine/releases/download/v0.7.1/basenine_linux_${GOARCH}.sha256 ./basenine_linux_${GOARCH}.sha256
RUN shasum -a 256 -c basenine_linux_"${GOARCH}".sha256 && \
chmod +x ./basenine_linux_"${GOARCH}" && \
......
......@@ -21,7 +21,7 @@ require (
github.com/orcaman/concurrent-map v1.0.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/stretchr/testify v1.7.0
github.com/up9inc/basenine/client/go v0.0.0-20220326121918-785f3061c8ce
github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2
github.com/up9inc/mizu/shared v0.0.0
github.com/up9inc/mizu/tap v0.0.0
github.com/up9inc/mizu/tap/api v0.0.0
......
......@@ -683,8 +683,8 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/up9inc/basenine/client/go v0.0.0-20220326121918-785f3061c8ce h1:vMTCpKItc9OyTLJXocNaq2NcBU5EnurJgTVOYb8W8dw=
github.com/up9inc/basenine/client/go v0.0.0-20220326121918-785f3061c8ce/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2 h1:2Ol+X82EOLac/GGP/oB0ACVP/g2vIJrAuzCjZcn+RRI=
github.com/up9inc/basenine/client/go v0.0.0-20220413023528-c741e4aa1cf2/go.mod h1:SvJGPoa/6erhUQV7kvHBwM/0x5LyO6XaG2lUaCaKiUI=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 h1:gga7acRE695APm9hlsSMoOoE65U4/TcqNj90mc69Rlg=
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
github.com/wI2L/jsondiff v0.1.1 h1:r2TkoEet7E4JMO5+s1RCY2R0LrNPNHY6hbDeow2hRHw=
......
......@@ -5,13 +5,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/up9inc/mizu/agent/pkg/models"
"os"
"path"
"sort"
"strings"
"time"
"github.com/up9inc/mizu/agent/pkg/models"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/elastic"
"github.com/up9inc/mizu/agent/pkg/har"
......@@ -104,9 +105,11 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
connection, err := basenine.NewConnection(shared.BasenineHost, shared.BaseninePort)
if err != nil {
panic(err)
logger.Log.Panicf("Can't establish a new connection to Basenine server: %v", err)
}
if err = connection.InsertMode(); err != nil {
logger.Log.Panicf("Insert mode call failed: %v", err)
}
connection.InsertMode()
disableOASValidation := false
ctx := context.Background()
......@@ -148,7 +151,9 @@ func startReadingChannel(outputItems <-chan *tapApi.OutputChannelItem, extension
providers.EntryAdded(len(data))
connection.SendText(string(data))
if err = connection.SendText(string(data)); err != nil {
logger.Log.Panicf("An error occured while inserting a new record to database: %v", err)
}
serviceMapGenerator := dependency.GetInstance(dependency.ServiceMapGeneratorDependency).(servicemap.ServiceMapSink)
serviceMapGenerator.NewTCPEntry(mizuEntry.Source, mizuEntry.Destination, &item.Protocol)
......
......@@ -49,7 +49,7 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
var entry *tapApi.Entry
err = json.Unmarshal(bytes, &entry)
if err != nil {
logger.Log.Debugf("error unmarshalling entry: %v", err.Error())
logger.Log.Debugf("Error unmarshalling entry: %v", err.Error())
continue
}
......@@ -79,7 +79,9 @@ func (e *BasenineEntryStreamer) Get(ctx context.Context, socketId int, params *W
go handleDataChannel(connection, data)
go handleMetaChannel(connection, meta)
connection.Query(query, data, meta)
if err = connection.Query(query, data, meta); err != nil {
logger.Log.Panicf("Query mode call failed: %v", err)
}
go func() {
<-ctx.Done()
......
......@@ -67,7 +67,7 @@ func ConfigureBasenineServer(host string, port string, dbSize int64, logLevel lo
wait.WithProto("tcp"),
wait.WithWait(200*time.Millisecond),
wait.WithBreak(50*time.Millisecond),
wait.WithDeadline(5*time.Second),
wait.WithDeadline(20*time.Second),
wait.WithDebug(logLevel == logging.DEBUG),
).Do([]string{fmt.Sprintf("%s:%s", host, port)}) {
logger.Log.Panicf("Basenine is not available!")
......
......@@ -2,7 +2,6 @@ package controllers
import (
"net/http"
"strconv"
"github.com/up9inc/mizu/agent/pkg/dependency"
"github.com/up9inc/mizu/agent/pkg/entries"
......@@ -69,7 +68,7 @@ func GetEntry(c *gin.Context) {
c.JSON(http.StatusBadRequest, validationError)
}
id, _ := strconv.Atoi(c.Param("id"))
id := c.Param("id")
entriesProvider := dependency.GetInstance(dependency.EntriesProvider).(entries.EntriesProvider)
entry, err := entriesProvider.GetEntry(singleEntryRequest, id)
......
......@@ -15,7 +15,7 @@ import (
type EntriesProvider interface {
GetEntries(entriesRequest *models.EntriesRequest) ([]*tapApi.EntryWrapper, *basenine.Metadata, error)
GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId int) (*tapApi.EntryWrapper, error)
GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error)
}
type BasenineEntriesProvider struct{}
......@@ -56,7 +56,7 @@ func (e *BasenineEntriesProvider) GetEntries(entriesRequest *models.EntriesReque
return dataSlice, metadata, nil
}
func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId int) (*tapApi.EntryWrapper, error) {
func (e *BasenineEntriesProvider) GetEntry(singleEntryRequest *models.SingleEntryRequest, entryId string) (*tapApi.EntryWrapper, error) {
var entry *tapApi.Entry
bytes, err := basenine.Single(shared.BasenineHost, shared.BaseninePort, entryId, singleEntryRequest.Query)
if err != nil {
......
......@@ -13,7 +13,7 @@ import (
)
type EntriesRequest struct {
LeftOff int `form:"leftOff" validate:"required,min=-1"`
LeftOff string `form:"leftOff" validate:"required"`
Direction int `form:"direction" validate:"required,oneof='1' '-1'"`
Query string `form:"query"`
Limit int `form:"limit" validate:"required,min=1"`
......
......@@ -4,6 +4,7 @@ import (
"bufio"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
"net/url"
......@@ -124,13 +125,13 @@ func feedFromHAR(file string, isSync bool, gen *defaultOasGenerator) (uint, erro
cnt := uint(0)
for _, entry := range harDoc.Log.Entries {
cnt += 1
feedEntry(&entry, "", file, gen, cnt)
feedEntry(&entry, "", file, gen, fmt.Sprintf("%024d", cnt))
}
return cnt, nil
}
func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt uint) {
func feedEntry(entry *har.Entry, source string, file string, gen *defaultOasGenerator, cnt string) {
entry.Comment = file
if entry.Response.Status == 302 {
logger.Log.Debugf("Dropped traffic entry due to permanent redirect status: %s", entry.StartedDateTime)
......@@ -192,7 +193,7 @@ func feedFromLDJSON(file string, isSync bool, gen *defaultOasGenerator) (uint, e
logger.Log.Warningf("Failed decoding entry: %s", line)
} else {
cnt += 1
feedEntry(&entry, source, file, gen, cnt)
feedEntry(&entry, source, file, gen, fmt.Sprintf("%024d", cnt))
}
}
}
......
{
"openapi": "3.1.0",
"info": {
"title": "http://carts",
"description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
"version": "1.0"
},
"servers": [
{
"url": "http://carts"
}
],
"paths": {
"/carts/{cartId}/items": {
"get": {
"summary": "/carts/{cartId}/items",
"description": "Mizu observed 3 entries (0 failed), at 2.287 hits/s, average response time is 0.017 seconds",
"operationId": "84c9b926-1f73-4ab4-b381-3c124528959f",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": [
{
"id": "60fe98fb86c0fc000869a90c",
"itemId": "3395a43e-2d88-40de-b95f-e00e1502085b",
"quantity": 1,
"unitPrice": 18
}
],
"x-sample-entry": "000000000000000000000010"
}
},
"x-sample-entry": "000000000000000000000010"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
},
"x-last-seen-ts": 1627298065.2397773,
"x-sample-entry": "000000000000000000000010"
},
"parameters": [
{
"name": "cartId",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "mHK0P7zTktmV1zv57iWAvCTd43FFMHap"
}
},
"x-sample-entry": "000000000000000000000010"
}
]
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298058.3798368,
"lastSeen": 1627298065.2397773,
"sumRT": 0.05,
"sumDuration": 6.859940528869629
}
}
\ No newline at end of file
{
"openapi": "3.1.0",
"info": {
"title": "Preloaded",
"description": "Test file for loading pre-existing OAS",
"version": "0.1"
},
"paths": {
"/catalogue": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue",
"description": "Mizu observed 3 entries (0 failed), at 2.647 hits/s, average response time is 0.008 seconds",
"operationId": "dd6c3dbe-6b6b-4ddd-baed-757e237ddb8a",
"parameters": [
{
"name": "page",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "1"
}
},
"x-sample-entry": "000000000000000000000002"
},
{
"name": "size",
"in": "query",
"required": true,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "6"
},
"example #1": {
"value": "3"
},
"example #2": {
"value": "5"
}
},
"x-sample-entry": "000000000000000000000011"
},
{
"name": "tags",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": ""
},
"example #1": {
"value": "blue"
}
},
"x-sample-entry": "000000000000000000000007"
},
{
"name": "sort",
"in": "query",
"required": false,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "id"
}
},
"x-sample-entry": "000000000000000000000007"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": [
{
"count": 1,
"description": "Socks fit for a Messiah. You too can experience walking in water with these special edition beauties. Each hole is lovingly proggled to leave smooth edges. The only sock approved by a higher power.",
"id": "03fef6ac-1896-4ce8-bd69-b798f85c6e0b",
"imageUrl": [
"/catalogue/images/holy_1.jpeg",
"/catalogue/images/holy_2.jpeg"
],
"name": "Holy",
"price": 99.99,
"tag": [
"action",
"magic"
]
},
{
"count": 438,
"description": "proident occaecat irure et excepteur labore minim nisi amet irure",
"id": "3395a43e-2d88-40de-b95f-e00e1502085b",
"imageUrl": [
"/catalogue/images/colourful_socks.jpg",
"/catalogue/images/colourful_socks.jpg"
],
"name": "Colourful",
"price": 18,
"tag": [
"brown",
"blue"
]
},
{
"count": 820,
"description": "Ready for action. Engineers: be ready to smash that next bug! Be ready, with these super-action-sport-masterpieces. This particular engineer was chased away from the office with a stick.",
"id": "510a0d7e-8e83-4193-b483-e27e09ddc34d",
"imageUrl": [
"/catalogue/images/puma_1.jpeg",
"/catalogue/images/puma_2.jpeg"
],
"name": "SuperSport XL",
"price": 15,
"tag": [
"sport",
"formal",
"black"
]
},
{
"count": 738,
"description": "A mature sock, crossed, with an air of nonchalance.",
"id": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38",
"imageUrl": [
"/catalogue/images/cross_1.jpeg",
"/catalogue/images/cross_2.jpeg"
],
"name": "Crossed",
"price": 17.32,
"tag": [
"blue",
"action",
"red",
"formal"
]
},
{
"count": 808,
"description": "enim officia aliqua excepteur esse deserunt quis aliquip nostrud anim",
"id": "819e1fbf-8b7e-4f6d-811f-693534916a8b",
"imageUrl": [
"/catalogue/images/WAT.jpg",
"/catalogue/images/WAT2.jpg"
],
"name": "Figueroa",
"price": 14,
"tag": [
"green",
"formal",
"blue"
]
},
{
"count": 175,
"description": "consequat amet cupidatat minim laborum tempor elit ex consequat in",
"id": "837ab141-399e-4c1f-9abc-bace40296bac",
"imageUrl": [
"/catalogue/images/catsocks.jpg",
"/catalogue/images/catsocks2.jpg"
],
"name": "Cat socks",
"price": 15,
"tag": [
"brown",
"formal",
"green"
]
}
],
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": "000000000000000000000011"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298057.7849188,
"lastSeen": 1627298065.7258668,
"sumRT": 0.024999999999999998,
"sumDuration": 7.940948009490967
}
},
"x-counters-total": {
"entries": 3,
"failures": 0,
"firstSeen": 1627298057.7849188,
"lastSeen": 1627298065.7258668,
"sumRT": 0.024999999999999998,
"sumDuration": 7.940948009490967
},
"x-last-seen-ts": 1627298065.7258668,
"x-sample-entry": "000000000000000000000011"
}
},
"/catalogue/size": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue/size",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.013 seconds",
"operationId": "2315e69d-9d66-48cf-b3d3-fec9c30bd28b",
"parameters": [
{
"name": "tags",
"in": "query",
"required": true,
"style": "form",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": ""
}
},
"x-sample-entry": "000000000000000000000001"
},
{
"name": "x-some",
"in": "header",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "demo val"
}
},
"x-sample-entry": "000000000000000000000001"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"err": null,
"size": 9
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298057.7841518,
"sumRT": 0.013,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298057.7841518,
"sumRT": 0.013,
"sumDuration": 0
},
"x-last-seen-ts": 1627298057.7841518,
"x-sample-entry": "000000000000000000000001"
}
},
"/catalogue/{id}": {
"get": {
"tags": [
"catalogue"
],
"summary": "/catalogue/{id}",
"description": "Mizu observed 4 entries (0 failed), at 1.899 hits/s, average response time is 0.003 seconds",
"parameters": [
{
"name": "non-required-header",
"in": "header",
"required": false,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
},
{
"name": "x-some",
"in": "header",
"required": false,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "demoval"
}
},
"x-sample-entry": "000000000000000000000004"
}
],
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"count": 438,
"description": "proident occaecat irure et excepteur labore minim nisi amet irure",
"id": "3395a43e-2d88-40de-b95f-e00e1502085b",
"imageUrl": [
"/catalogue/images/colourful_socks.jpg",
"/catalogue/images/colourful_socks.jpg"
],
"name": "Colourful",
"price": 18,
"tag": [
"brown",
"blue"
]
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 4,
"failures": 0,
"firstSeen": 1627298058.1315014,
"lastSeen": 1627298065.7293031,
"sumRT": 0.013999999999999999,
"sumDuration": 7.597801685333252
}
},
"x-counters-total": {
"entries": 4,
"failures": 0,
"firstSeen": 1627298058.1315014,
"lastSeen": 1627298065.7293031,
"sumRT": 0.013999999999999999,
"sumDuration": 7.597801685333252
},
"x-last-seen-ts": 1627298065.7293031,
"x-sample-entry": "000000000000000000000012"
},
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"examples": {
"example #0": {
"value": "3395a43e-2d88-40de-b95f-e00e1502085b"
},
"example #1": {
"value": "808a2de1-1aaa-4c25-a9b9-6612e8f29a38"
}
},
"example": "some-uuid-maybe",
"x-sample-entry": "000000000000000000000012"
}
]
},
"/catalogue/{id}/details": {
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"style": "simple",
"schema": {
"type": "string"
},
"example": "some-uuid-maybe"
}
]
},
"/tags": {
"get": {
"summary": "/tags",
"description": "Mizu observed 1 entries (0 failed), at 0.000 hits/s, average response time is 0.007 seconds",
"operationId": "c4d7d0ed-1a78-4370-a049-efe3abc631a6",
"responses": {
"200": {
"description": "Successful call with status 200",
"content": {
"application/json": {
"example": {
"err": null,
"tags": [
"brown",
"geek",
"formal",
"blue",
"skin",
"red",
"action",
"sport",
"black",
"magic",
"green"
]
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
"some-source": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841816,
"lastSeen": 1627298057.7841816,
"sumRT": 0.007,
"sumDuration": 0
}
},
"x-counters-total": {
"entries": 1,
"failures": 0,
"firstSeen": 1627298057.7841816,
"lastSeen": 1627298057.7841816,
"sumRT": 0.007,
"sumDuration": 0
},
"x-last-seen-ts": 1627298057.7841816,
"x-sample-entry": "000000000000000000000003"
}
}
},
"x-counters-per-source": {
"some-source": {
"entries": 9,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298065.7293031,
"sumRT": 0.05899999999999999,
"sumDuration": 15.538749694824219
}
},
"x-counters-total": {
"entries": 9,
"failures": 0,
"firstSeen": 1627298057.7841518,
"lastSeen": 1627298065.7293031,
"sumRT": 0.05899999999999999,
"sumDuration": 15.538749694824219
}
}
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
......@@ -3,12 +3,13 @@ package oas
import (
"context"
"encoding/json"
"net/url"
"sync"
basenine "github.com/up9inc/basenine/client/go"
"github.com/up9inc/mizu/agent/pkg/har"
"github.com/up9inc/mizu/shared"
"github.com/up9inc/mizu/tap/api"
"net/url"
"sync"
"github.com/up9inc/mizu/shared/logger"
)
......@@ -27,11 +28,12 @@ type OasGenerator interface {
}
type defaultOasGenerator struct {
started bool
ctx context.Context
cancel context.CancelFunc
serviceSpecs *sync.Map
dbConn *basenine.Connection
started bool
ctx context.Context
cancel context.CancelFunc
serviceSpecs *sync.Map
dbConn *basenine.Connection
dbMutex sync.Mutex
entriesQuery string
}
......@@ -77,16 +79,17 @@ func (g *defaultOasGenerator) Stop() {
if !g.started {
return
}
g.started = false
g.cancel()
g.reset()
g.dbMutex.Lock()
defer g.dbMutex.Unlock()
if g.dbConn != nil {
g.dbConn.Close()
g.dbConn = nil
}
g.cancel()
g.reset()
g.started = false
}
func (g *defaultOasGenerator) IsStarted() bool {
......@@ -98,8 +101,12 @@ func (g *defaultOasGenerator) runGenerator() {
dataChan := make(chan []byte)
metaChan := make(chan []byte)
g.dbMutex.Lock()
defer g.dbMutex.Unlock()
logger.Log.Infof("Querying DB for OAS generator with query '%s'", g.entriesQuery)
g.dbConn.Query(g.entriesQuery, dataChan, metaChan)
if err := g.dbConn.Query(g.entriesQuery, dataChan, metaChan); err != nil {
logger.Log.Errorf("Query mode call failed: %v", err)
}
for {
select {
......@@ -174,7 +181,7 @@ func (g *defaultOasGenerator) handleHARWithSource(entryWSource *EntryWithSource)
return
}
logger.Log.Debugf("Handled entry %d as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
logger.Log.Debugf("Handled entry %s as opId: %s", entryWSource.Id, opId) // TODO: set opId back to entry?
}
func (g *defaultOasGenerator) getGen(dest string, urlStr string) *SpecGen {
......
......@@ -4,6 +4,7 @@ import (
"encoding/json"
"github.com/up9inc/mizu/agent/pkg/har"
"testing"
"time"
)
func TestOASGen(t *testing.T) {
......@@ -38,5 +39,8 @@ func TestOASGen(t *testing.T) {
if !gen.IsStarted() {
t.Errorf("Should be started")
}
time.Sleep(100 * time.Millisecond)
gen.Stop()
}
......@@ -3,10 +3,6 @@ package oas
import (
"encoding/json"
"errors"
"github.com/chanced/openapi"
"github.com/google/uuid"
"github.com/nav-inc/datetime"
"github.com/up9inc/mizu/shared/logger"
"io"
"io/ioutil"
"mime"
......@@ -18,6 +14,11 @@ import (
"strings"
"sync"
"github.com/chanced/openapi"
"github.com/google/uuid"
"github.com/nav-inc/datetime"
"github.com/up9inc/mizu/shared/logger"
"github.com/up9inc/mizu/agent/pkg/har"
"time"
......@@ -32,7 +33,7 @@ type EntryWithSource struct {
Source string
Destination string
Entry har.Entry
Id uint
Id string
}
type reqResp struct { // hello, generics in Go
......@@ -67,7 +68,7 @@ func (g *SpecGen) StartFromSpec(oas *openapi.OpenAPI) {
g.tree = new(Node)
for pathStr, pathObj := range oas.Paths.Items {
pathSplit := strings.Split(string(pathStr), "/")
g.tree.getOrSet(pathSplit, pathObj, 0)
g.tree.getOrSet(pathSplit, pathObj, "")
// clean "last entry timestamp" markers from the past
for _, pathAndOp := range g.tree.listOps() {
......@@ -341,7 +342,7 @@ func handleCounters(opObj *openapi.Operation, success bool, entryWithSource *Ent
return nil
}
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId uint) error {
func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
// TODO: we don't handle the situation when header/qstr param can be defined on pathObj level. Also the path param defined on opObj
urlParsed, err := url.Parse(req.URL)
if err != nil {
......@@ -401,7 +402,7 @@ func handleRequest(req *har.Request, opObj *openapi.Operation, isSuccess bool, s
return nil
}
func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId uint) error {
func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool, sampleId string) error {
// TODO: we don't support "default" response
respObj, err := getResponseObj(resp, opObj, isSuccess)
if err != nil {
......@@ -422,7 +423,7 @@ func handleResponse(resp *har.Response, opObj *openapi.Operation, isSuccess bool
return nil
}
func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId uint) {
func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sampleId string) {
visited := map[string]*openapi.HeaderObj{}
for _, pair := range reqHeaders {
if isHeaderIgnored(pair.Name) {
......@@ -466,7 +467,7 @@ func handleRespHeaders(reqHeaders []har.Header, respObj *openapi.ResponseObj, sa
}
}
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId uint) (*openapi.MediaType, error) {
func fillContent(reqResp reqResp, respContent openapi.Content, ctype string, sampleId string) (*openapi.MediaType, error) {
content, found := respContent[ctype]
if !found {
respContent[ctype] = &openapi.MediaType{}
......
......@@ -22,10 +22,10 @@
"content": {
"application/json": {
"example": null,
"x-sample-entry": 4
"x-sample-entry": "000000000000000000000004"
}
},
"x-sample-entry": 4
"x-sample-entry": "000000000000000000000004"
}
},
"x-counters-per-source": {
......@@ -47,7 +47,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750580.04,
"x-sample-entry": 4
"x-sample-entry": "000000000000000000000004"
}
},
"/appears-twice": {
......@@ -61,10 +61,10 @@
"content": {
"application/json": {
"example": null,
"x-sample-entry": 6
"x-sample-entry": "000000000000000000000006"
}
},
"x-sample-entry": 6
"x-sample-entry": "000000000000000000000006"
}
},
"x-counters-per-source": {
......@@ -86,7 +86,7 @@
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.74,
"x-sample-entry": 6
"x-sample-entry": "000000000000000000000006"
}
},
"/body-optional": {
......@@ -99,10 +99,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 12
"x-sample-entry": "000000000000000000000012"
}
},
"x-sample-entry": 12
"x-sample-entry": "000000000000000000000012"
}
},
"x-counters-per-source": {
......@@ -124,16 +124,16 @@
"sumDuration": 0.01
},
"x-last-seen-ts": 1567750581.75,
"x-sample-entry": 12,
"x-sample-entry": "000000000000000000000012",
"requestBody": {
"description": "Generic request body",
"content": {
"application/json": {
"example": "{\"key\", \"val\"}",
"x-sample-entry": 11
"x-sample-entry": "000000000000000000000011"
}
},
"x-sample-entry": 12
"x-sample-entry": "000000000000000000000012"
}
}
},
......@@ -147,10 +147,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 13
"x-sample-entry": "000000000000000000000013"
}
},
"x-sample-entry": 13
"x-sample-entry": "000000000000000000000013"
}
},
"x-counters-per-source": {
......@@ -172,17 +172,17 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750581.75,
"x-sample-entry": 13,
"x-sample-entry": "000000000000000000000013",
"requestBody": {
"description": "Generic request body",
"content": {
"": {
"example": "body exists",
"x-sample-entry": 13
"x-sample-entry": "000000000000000000000013"
}
},
"required": true,
"x-sample-entry": 13
"x-sample-entry": "000000000000000000000013"
}
}
},
......@@ -197,10 +197,10 @@
"content": {
"": {
"example": {},
"x-sample-entry": 9
"x-sample-entry": "000000000000000000000009"
}
},
"x-sample-entry": 9
"x-sample-entry": "000000000000000000000009"
}
},
"x-counters-per-source": {
......@@ -222,7 +222,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.74,
"x-sample-entry": 9,
"x-sample-entry": "000000000000000000000009",
"requestBody": {
"description": "Generic request body",
"content": {
......@@ -250,11 +250,11 @@
}
},
"example": "--BOUNDARY\r\nContent-Disposition: form-data; name=\"file\"; filename=\"metadata.json\"\r\nContent-Type: application/json\r\n\r\n{\"functions\": 123}\r\n--BOUNDARY\r\nContent-Disposition: form-data; name=\"path\"\r\n\r\n/content/components\r\n--BOUNDARY--\r\n",
"x-sample-entry": 9
"x-sample-entry": "000000000000000000000009"
}
},
"required": true,
"x-sample-entry": 9
"x-sample-entry": "000000000000000000000009"
}
}
},
......@@ -268,10 +268,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 8
"x-sample-entry": "000000000000000000000008"
}
},
"x-sample-entry": 8
"x-sample-entry": "000000000000000000000008"
}
},
"x-counters-per-source": {
......@@ -293,7 +293,7 @@
"sumDuration": 1
},
"x-last-seen-ts": 1567750581.74,
"x-sample-entry": 8,
"x-sample-entry": "000000000000000000000008",
"requestBody": {
"description": "Generic request body",
"content": {
......@@ -334,11 +334,11 @@
}
},
"example": "agent-id=ade\u0026callback-url=\u0026token=sometoken",
"x-sample-entry": 8
"x-sample-entry": "000000000000000000000008"
}
},
"required": true,
"x-sample-entry": 8
"x-sample-entry": "000000000000000000000008"
}
}
},
......@@ -355,10 +355,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 14
"x-sample-entry": "000000000000000000000014"
}
},
"x-sample-entry": 14
"x-sample-entry": "000000000000000000000014"
}
},
"x-counters-per-source": {
......@@ -380,7 +380,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750582,
"x-sample-entry": 14
"x-sample-entry": "000000000000000000000014"
},
"parameters": [
{
......@@ -396,7 +396,7 @@
"value": "234324"
}
},
"x-sample-entry": 14
"x-sample-entry": "000000000000000000000014"
}
]
},
......@@ -413,10 +413,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 18
"x-sample-entry": "000000000000000000000018"
}
},
"x-sample-entry": 18
"x-sample-entry": "000000000000000000000018"
}
},
"x-counters-per-source": {
......@@ -438,7 +438,7 @@
"sumDuration": 9.53e-7
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": 18
"x-sample-entry": "000000000000000000000018"
},
"parameters": [
{
......@@ -467,7 +467,7 @@
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
}
]
},
......@@ -484,10 +484,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 15
"x-sample-entry": "000000000000000000000015"
}
},
"x-sample-entry": 15
"x-sample-entry": "000000000000000000000015"
}
},
"x-counters-per-source": {
......@@ -509,7 +509,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": 15
"x-sample-entry": "000000000000000000000015"
},
"parameters": [
{
......@@ -538,7 +538,7 @@
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
}
]
},
......@@ -555,10 +555,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 16
"x-sample-entry": "000000000000000000000016"
}
},
"x-sample-entry": 16
"x-sample-entry": "000000000000000000000016"
}
},
"x-counters-per-source": {
......@@ -580,7 +580,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": 16
"x-sample-entry": "000000000000000000000016"
},
"parameters": [
{
......@@ -609,7 +609,7 @@
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
}
]
},
......@@ -626,10 +626,10 @@
"description": "Successful call with status 200",
"content": {
"": {
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
}
},
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
}
},
"x-counters-per-source": {
......@@ -651,7 +651,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750582.00,
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
},
"parameters": [
{
......@@ -667,7 +667,7 @@
"value": "23421"
}
},
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
},
{
"name": "parampatternId",
......@@ -695,7 +695,7 @@
"value": "prefix-gibberish-afterwards"
}
},
"x-sample-entry": 19
"x-sample-entry": "000000000000000000000019"
}
]
},
......@@ -710,10 +710,10 @@
"content": {
"application/json": {
"example": null,
"x-sample-entry": 3
"x-sample-entry": "000000000000000000000003"
}
},
"x-sample-entry": 3
"x-sample-entry": "000000000000000000000003"
}
},
"x-counters-per-source": {
......@@ -735,7 +735,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750579.74,
"x-sample-entry": 3
"x-sample-entry": "000000000000000000000003"
},
"parameters": [
{
......@@ -754,7 +754,7 @@
"value": "<UUID4>"
}
},
"x-sample-entry": 3
"x-sample-entry": "000000000000000000000003"
}
]
},
......@@ -768,10 +768,10 @@
"description": "Successful call with status 200",
"content": {
"text/html": {
"x-sample-entry": 1
"x-sample-entry": "000000000000000000000001"
}
},
"x-sample-entry": 1
"x-sample-entry": "000000000000000000000001"
}
},
"x-counters-per-source": {
......@@ -793,7 +793,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750483.86,
"x-sample-entry": 1
"x-sample-entry": "000000000000000000000001"
},
"parameters": [
{
......@@ -812,7 +812,7 @@
"value": "<UUID4>"
}
},
"x-sample-entry": 3
"x-sample-entry": "000000000000000000000003"
}
]
},
......@@ -827,10 +827,10 @@
"content": {
"application/json": {
"example": null,
"x-sample-entry": 2
"x-sample-entry": "000000000000000000000002"
}
},
"x-sample-entry": 2
"x-sample-entry": "000000000000000000000002"
}
},
"x-counters-per-source": {
......@@ -852,7 +852,7 @@
"sumDuration": 0
},
"x-last-seen-ts": 1567750578.74,
"x-sample-entry": 2
"x-sample-entry": "000000000000000000000002"
},
"parameters": [
{
......@@ -871,7 +871,7 @@
"value": "<UUID4>"
}
},
"x-sample-entry": 3
"x-sample-entry": "000000000000000000000003"
}
]
}
......
......@@ -2,12 +2,13 @@ package oas
import (
"encoding/json"
"github.com/chanced/openapi"
"github.com/up9inc/mizu/shared/logger"
"net/url"
"regexp"
"strconv"
"strings"
"github.com/chanced/openapi"
"github.com/up9inc/mizu/shared/logger"
)
type NodePath = []string
......@@ -20,7 +21,7 @@ type Node struct {
children []*Node
}
func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId uint) (node *Node) {
func (n *Node) getOrSet(path NodePath, existingPathObj *openapi.PathObj, sampleId string) (node *Node) {
if existingPathObj == nil {
panic("Invalid function call")
}
......
package oas
import (
"fmt"
"strings"
"testing"
......@@ -23,7 +24,7 @@ func TestTree(t *testing.T) {
for i, tc := range testCases {
split := strings.Split(tc.inp, "/")
pathObj := new(openapi.PathObj)
node := tree.getOrSet(split, pathObj, uint(i))
node := tree.getOrSet(split, pathObj, fmt.Sprintf("%024d", i))
fillPathParams(node, pathObj)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment