Commit bba7344b authored by Ankit Anand's avatar Ankit Anand
Browse files

fixes for CH API implementations

parent 51fe6345
develop 'issue141' 1244-edit-alert 1725-expection 24-may-2022-testing 414-uri add-cypress add-github--actions add-testing-framework alertmanager-discovery alerts ankit01-oss-patch-1 anweshknayak-patch-1 bug/synk-security-fixes bump-json-iterator bump-prometheus ch-connection-exit ch-index-changes change-installation change-sample-alert check-antd-css chore-telemetry chore/analytics chore/change-validation-message chore/improve-metrics-perf chore/improve-perf-remote-read chore/install-script-update chore/migration-0.8.0 chore/migration-file-path chore/moving-to-clickhouse-v2 chore/remove-query-service-codeowners chore/update-otelcollector-0.43.0 clickhouse-helm clickhouse-persistence configurable-storage-class crud-dashboards custom_metrics dashboarad-vars dashboard-bug-fix debug-dep debug-dep-migrate docker-swarm effgo enable-alerts error-grpc feat-doc-contribute feat-statsd-receiver feat/addHasErrorColumn feat/amol-ee feat/clickhouse-db-optimizations feat/custom-func-getSubTreeSpans feat/dynamic-tooltip feat/ee feat/exclude-filter-support feat/featureFlagging feat/fields-compression feat/gRPC-code-method feat/getFilteredSpans feat/getSpanFilters feat/gh-bot feat/newTraceFilter feat/searchTraceId feat/support-custom-events feat/support-error-tab-page feat/support-events feat/tagValueSuggestion feat/trace-detail feat/trace-resource-attributes feat/udf-function-getSubTreeSpans feat/usage feat/usage-reporting filter-set fix-ami-linux-docker fix-cors-put fix-double-client fix-eslint fix-filtered-spans fix-gant-chart-timeline fix-husky fix-scroll-gantt-chart fix-service-map-zoom fix-tabs-issue fix-user fix/414-error-trace-filter-APIs fix/aggregate fix/error-exception-page-typo fix/error-exception-sql-issue fix/errorDetailURL fix/exceptionPageOptimization fix/exclude-operation fix/null-values fix/serviceMapDependencies fix/setTTLapis fix/telemetry-bug hostmetrics infra-metrics issue-1228 issue-1252 issue-1293 issue-1294 issue-1442 issue-1485-develop issue-1511 issue-1583 issue-618 issue-pod-687 labels_object main makeavish-patch-1 makeavish-patch-2 metric-suggest-apis-chv2 metrics-builder-all metrics-table new-metrics new-metrics-enums palashgdev-patch-1 palashgdev-patch-2 perf/trace-detail-page playwright pranay01-patch-1 pranshuchittora/feat/dynamic-step-size prashant/add-codeowners prashant/add-deploy-docs prashant/ci-k3s-enchancements prashant/contributing-docs prashant/docker-data-path prashant/e2e-k3s-changes prashant/frontend-docker prashant/hotrod-log-options prashant/hotrod-template prashant/hotrod-yaml prashant/install-script-changes prashant/integrate-behaviorbot prashant/migrate-helm-charts prashant/minor-k3s-changes prashant/nginx-cache-improvement prashant/remove-ports prashant/rename-config prashant/single-hotrod-manifest prashant/two-compose-yaml prashant/version-ping-mini-css-extract-plugin prashant/versioning prod-feedback query_refactor refactor-gant-chart refactor-selected-span refactoring release/v0.10 release/v0.10.0 release/v0.10.1 release/v0.10.2 release/v0.11 release/v0.11.0 release/v0.11.1 release/v0.11.2 release/v0.11.3 release/v0.6 release/v0.6.0 release/v0.6.1 release/v0.6.2 release/v0.7 release/v0.7.0 release/v0.7.1 release/v0.7.2 release/v0.7.3 release/v0.7.4 release/v0.7.5 release/v0.8 release/v0.8.0 release/v0.8.1 release/v0.8.2 release/v0.9 release/v0.9.0 release/v0.9.1 release/v0.9.2 revert-310-snyk-fix-d368e29b07323337ebe6ba0e289d85e7 revert-382-fix-eslint-error revert-386-remove-package revert-456-storybook revert-628-chore/migration-file-path revert-770-trace-styles revert-814-release/v0.7.0 revert-835-pc/feat/shared-styles-for-styled-components snyk-fix-0c242a73fc20c472e37f0b59a7c93a5a snyk-fix-194ab2b3455dd9e2099068fc3ac5f20f snyk-fix-3d7b28e56a36018c4b5cf8438365ff27 snyk-fix-4ef12a6988dec7696867e0b2accb8289 snyk-fix-634b756aa5179844cafb3a54d707b934 snyk-fix-7ee2560cd9d67a9821899135c05c1175 snyk-fix-8f3a2f55a70c214b57d1a3b3bce7d0cf snyk-fix-a7789225303bf723cc0665d987e0bf24 snyk-fix-badf8cd7fc1f284699352ace871e5075 snyk-fix-d1c97e9d4144298a9396308ed03ebd9e snyk-fix-d368e29b07323337ebe6ba0e289d85e7 snyk-fix-d5e919722918d14db890a368b9964203 span-metrics spanAggregatesAPI trace-gantt-chart trace-search troubleshoot ttl-get-int ttl-plus uuid-server wip-release-0.11.1 0.3.5 v0.11.3 v0.11.3-rc.1 v0.11.2 v0.11.2-rc.3 v0.11.2-rc.2 v0.11.2-rc.1 v0.11.1 v0.11.1-rc.1 v0.11.0 v0.11.0-rc.1 v0.11 v0.10.2 v0.10.1 v0.10.1-rc.1 v0.10.0 v0.10.0-rc2 v0.10.0-rc1 v0.10 v0.9.2 v0.9.2-rc1 v0.9.1 v0.9.0 v0.9.0-rc2 v0.9.0-rc1 v0.9 v0.8.2 v0.8.1 v0.8.1-rc5 v0.8.1-rc4 v0.8.1-rc3 v0.8.1-rc2 v0.8.1-rc1 v0.8.0 v0.8.0-rc6 v0.8.0-rc5 v0.8.0-rc4 v0.8.0-rc3 v0.8.0-rc2 v0.8.0-rc1 v0.8 v0.7.5 v0.7.5-rc2 v0.7.5-rc1 v0.7.4 v0.7.4-rc2 v0.7.4-rc1 v0.7.3 v0.7.2 v0.7.2-rc1 v0.7.1 v0.7.0 v0.7 v0.6.2 v0.6.1 v0.6.0 v0.6 v0.5.4 v0.5.3 v0.5.2 v0.5.1 v0.5.0 v0.4.5 v0.4.4 v0.4.3 v0.4.2 v0.4.1 v0.4.0 v0.3.6 v0.3.5 v0.3.4 v0.3.3 v0.3.2 v0.3.1
No related merge requests found
Showing with 89 additions and 40 deletions
+89 -40
File deleted
...@@ -74,42 +74,53 @@ func connect(cfg *namespaceConfig) (*sqlx.DB, error) { ...@@ -74,42 +74,53 @@ func connect(cfg *namespaceConfig) (*sqlx.DB, error) {
return cfg.Connector(cfg) return cfg.Connector(cfg)
} }
func (r *ClickHouseReader) getStrings(ctx context.Context, sql string, args ...interface{}) ([]string, error) { func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.GetServicesParams) (*[]model.ServiceItem, error) {
rows, err := r.db.QueryContext(ctx, sql, args...)
if err != nil { if r.indexTable == "" {
return nil, err return nil, ErrNoIndexTable
} }
defer rows.Close() serviceItems := []model.ServiceItem{}
values := []string{} query := fmt.Sprintf("SELECT serviceName, quantile(0.99)(durationNano) as p99, avg(durationNano) as avgDuration, count(*) as numCalls FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' GROUP BY serviceName", r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10))
for rows.Next() { err := r.db.Select(&serviceItems, query)
var value string
if err := rows.Scan(&value); err != nil { zap.S().Info(query)
return nil, err
}
values = append(values, value)
}
if err := rows.Err(); err != nil { if err != nil {
return nil, err zap.S().Debug("Error in processing sql query: ", err)
return nil, fmt.Errorf("Error in processing sql query")
} }
return values, nil ////////////////// Below block gets 5xx of services
} serviceErrorItems := []model.ServiceItem{}
// GetServices fetches the sorted service list that have not expired query = fmt.Sprintf("SELECT serviceName, count(*) as numErrors FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' AND statusCode>=500 GROUP BY serviceName", r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10))
func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.GetServicesParams) (*[]model.ServiceItem, error) {
if r.indexTable == "" { err = r.db.Select(&serviceErrorItems, query)
return nil, ErrNoIndexTable
zap.S().Info(query)
if err != nil {
zap.S().Debug("Error in processing sql query: ", err)
return nil, fmt.Errorf("Error in processing sql query")
} }
var serviceItems []model.ServiceItem m5xx := make(map[string]int)
query := fmt.Sprintf("SELECT serviceName, quantile(0.99)(durationNano) as p99, avg(durationNano) as avgDuration, count(*) as numCalls FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' GROUP BY serviceName", r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10)) for j, _ := range serviceErrorItems {
err := r.db.Select(&serviceItems, query) m5xx[serviceErrorItems[j].ServiceName] = serviceErrorItems[j].NumErrors
}
///////////////////////////////////////////
////////////////// Below block gets 4xx of services
service4xxItems := []model.ServiceItem{}
query = fmt.Sprintf("SELECT serviceName, count(*) as num4xx FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' AND statusCode>=400 AND statusCode<500 GROUP BY serviceName", r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10))
err = r.db.Select(&service4xxItems, query)
zap.S().Info(query) zap.S().Info(query)
...@@ -118,8 +129,22 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G ...@@ -118,8 +129,22 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G
return nil, fmt.Errorf("Error in processing sql query") return nil, fmt.Errorf("Error in processing sql query")
} }
if serviceItems == nil { m4xx := make(map[string]int)
serviceItems = []model.ServiceItem{}
for j, _ := range service4xxItems {
m5xx[service4xxItems[j].ServiceName] = service4xxItems[j].Num4XX
}
for i, _ := range serviceItems {
if val, ok := m5xx[serviceItems[i].ServiceName]; ok {
serviceItems[i].NumErrors = val
}
if val, ok := m4xx[serviceItems[i].ServiceName]; ok {
serviceItems[i].Num4XX = val
}
serviceItems[i].CallRate = float32(serviceItems[i].NumCalls) / float32(queryParams.Period)
serviceItems[i].FourXXRate = float32(serviceItems[i].Num4XX) / float32(queryParams.Period)
serviceItems[i].ErrorRate = float32(serviceItems[i].NumErrors) / float32(queryParams.Period)
} }
return &serviceItems, nil return &serviceItems, nil
...@@ -127,7 +152,7 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G ...@@ -127,7 +152,7 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G
func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams *model.GetServiceOverviewParams) (*[]model.ServiceOverviewItem, error) { func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams *model.GetServiceOverviewParams) (*[]model.ServiceOverviewItem, error) {
var serviceOverviewItems []model.ServiceOverviewItem serviceOverviewItems := []model.ServiceOverviewItem{}
query := fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %s minute) as time, quantile(0.99)(durationNano) as p99, quantile(0.95)(durationNano) as p95,quantile(0.50)(durationNano) as p50, count(*) as numCalls FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' AND serviceName='%s' GROUP BY time ORDER BY time DESC", strconv.Itoa(int(queryParams.StepSeconds/60)), r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10), queryParams.ServiceName) query := fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %s minute) as time, quantile(0.99)(durationNano) as p99, quantile(0.95)(durationNano) as p95,quantile(0.50)(durationNano) as p50, count(*) as numCalls FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' AND serviceName='%s' GROUP BY time ORDER BY time DESC", strconv.Itoa(int(queryParams.StepSeconds/60)), r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10), queryParams.ServiceName)
...@@ -140,14 +165,36 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams * ...@@ -140,14 +165,36 @@ func (r *ClickHouseReader) GetServiceOverview(ctx context.Context, queryParams *
return nil, fmt.Errorf("Error in processing sql query") return nil, fmt.Errorf("Error in processing sql query")
} }
serviceErrorItems := []model.ServiceErrorItem{}
query = fmt.Sprintf("SELECT toStartOfInterval(timestamp, INTERVAL %s minute) as time, count(*) as numErrors FROM %s WHERE timestamp>='%s' AND timestamp<='%s' AND kind='2' AND serviceName='%s' AND statusCode>=500 GROUP BY time ORDER BY time DESC", strconv.Itoa(int(queryParams.StepSeconds/60)), r.indexTable, strconv.FormatInt(queryParams.Start.UnixNano(), 10), strconv.FormatInt(queryParams.End.UnixNano(), 10), queryParams.ServiceName)
err = r.db.Select(&serviceErrorItems, query)
zap.S().Info(query)
if err != nil {
zap.S().Debug("Error in processing sql query: ", err)
return nil, fmt.Errorf("Error in processing sql query")
}
m := make(map[int64]int)
for j, _ := range serviceErrorItems {
timeObj, _ := time.Parse(time.RFC3339Nano, serviceErrorItems[j].Time)
m[int64(timeObj.UnixNano())] = serviceErrorItems[j].NumErrors
}
for i, _ := range serviceOverviewItems { for i, _ := range serviceOverviewItems {
timeObj, _ := time.Parse(time.RFC3339Nano, serviceOverviewItems[i].Time) timeObj, _ := time.Parse(time.RFC3339Nano, serviceOverviewItems[i].Time)
serviceOverviewItems[i].Timestamp = int64(timeObj.UnixNano()) serviceOverviewItems[i].Timestamp = int64(timeObj.UnixNano())
serviceOverviewItems[i].Time = "" serviceOverviewItems[i].Time = ""
}
if serviceOverviewItems == nil { if val, ok := m[serviceOverviewItems[i].Timestamp]; ok {
serviceOverviewItems = []model.ServiceOverviewItem{} serviceOverviewItems[i].NumErrors = val
}
serviceOverviewItems[i].ErrorRate = float32(serviceOverviewItems[i].NumErrors) * 100 / float32(serviceOverviewItems[i].NumCalls)
serviceOverviewItems[i].CallRate = float32(serviceOverviewItems[i].NumCalls) / float32(queryParams.StepSeconds)
} }
return &serviceOverviewItems, nil return &serviceOverviewItems, nil
...@@ -189,6 +236,11 @@ func (r *ClickHouseReader) SearchSpans(ctx context.Context, queryParams *model.S ...@@ -189,6 +236,11 @@ func (r *ClickHouseReader) SearchSpans(ctx context.Context, queryParams *model.S
for _, item := range queryParams.Tags { for _, item := range queryParams.Tags {
if item.Key == "error" && item.Value == "true" {
query = query + " AND ( has(tags, 'error:true') OR statusCode>=500)"
continue
}
if item.Operator == "equals" { if item.Operator == "equals" {
query = query + " AND has(tags, ?)" query = query + " AND has(tags, ?)"
args = append(args, fmt.Sprintf("%s:%s", item.Key, item.Value)) args = append(args, fmt.Sprintf("%s:%s", item.Key, item.Value))
...@@ -204,13 +256,10 @@ func (r *ClickHouseReader) SearchSpans(ctx context.Context, queryParams *model.S ...@@ -204,13 +256,10 @@ func (r *ClickHouseReader) SearchSpans(ctx context.Context, queryParams *model.S
return nil, fmt.Errorf("Tag Operator %s not supported", item.Operator) return nil, fmt.Errorf("Tag Operator %s not supported", item.Operator)
} }
if item.Key == "error" && item.Value == "true" {
query = query + " OR statusCode>=500"
}
query = query + " ORDER BY timestamp LIMIT 100"
} }
query = query + " ORDER BY timestamp DESC LIMIT 100"
var searchScanReponses []model.SearchSpanReponseItem var searchScanReponses []model.SearchSpanReponseItem
err := r.db.Select(&searchScanReponses, query, args...) err := r.db.Select(&searchScanReponses, query, args...)
......
...@@ -469,7 +469,7 @@ func GetServices(client *SqlClient, query *model.GetServicesParams) (*[]model.Se ...@@ -469,7 +469,7 @@ func GetServices(client *SqlClient, query *model.GetServicesParams) (*[]model.Se
////////////////// Below block gets 4xx of services ////////////////// Below block gets 4xx of services
sqlQuery = fmt.Sprintf(`SELECT COUNT(SpanId) as numErrors, "ServiceName" as "serviceName" FROM %s WHERE "__time" >= '%s' and "__time" <= '%s' and "Kind"='2' and "StatusCode">=400 and "StatusCode" < 500 GROUP BY "ServiceName"`, constants.DruidDatasource, query.StartTime, query.EndTime) sqlQuery = fmt.Sprintf(`SELECT COUNT(SpanId) as num4xx, "ServiceName" as "serviceName" FROM %s WHERE "__time" >= '%s' and "__time" <= '%s' and "Kind"='2' and "StatusCode">=400 and "StatusCode" < 500 GROUP BY "ServiceName"`, constants.DruidDatasource, query.StartTime, query.EndTime)
response4xx, err := client.Query(sqlQuery, "object") response4xx, err := client.Query(sqlQuery, "object")
......
...@@ -15,7 +15,7 @@ type ServiceItem struct { ...@@ -15,7 +15,7 @@ type ServiceItem struct {
CallRate float32 `json:"callRate" db:"callRate"` CallRate float32 `json:"callRate" db:"callRate"`
NumErrors int `json:"numErrors" db:"numErrors"` NumErrors int `json:"numErrors" db:"numErrors"`
ErrorRate float32 `json:"errorRate" db:"errorRate"` ErrorRate float32 `json:"errorRate" db:"errorRate"`
Num4XX int `json:"num4XX" db:"num4XX"` Num4XX int `json:"num4XX" db:"num4xx"`
FourXXRate float32 `json:"fourXXRate" db:"fourXXRate"` FourXXRate float32 `json:"fourXXRate" db:"fourXXRate"`
} }
...@@ -26,9 +26,9 @@ type ServiceListErrorItem struct { ...@@ -26,9 +26,9 @@ type ServiceListErrorItem struct {
} }
type ServiceErrorItem struct { type ServiceErrorItem struct {
Time string `json:"time,omitempty"` Time string `json:"time,omitempty" db:"time,omitempty"`
Timestamp int64 `json:"timestamp"` Timestamp int64 `json:"timestamp" db:"timestamp"`
NumErrors int `json:"numErrors"` NumErrors int `json:"numErrors" db:"numErrors"`
} }
type ServiceOverviewItem struct { type ServiceOverviewItem struct {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment