Skip to content
GitLab
Commit cedab583 authored by makeavish's avatar makeavish
Browse files

test: :white_check_mark: update tests for async TTL api

parent db78bf42
fix/setTTLapis
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 103 additions and 65 deletions
+103 -65
pkg/query-service/app/clickhouseReader/reader.go
+ 21
- 17
View file @ cedab583
......@@ -2267,13 +2267,18 @@ func (r *ClickHouseReader) GetFilteredSpansAggregates(ctx context.Context, query
return &GetFilteredSpansAggregatesResponse, nil
}
// SetTTL sets the TTL for traces or metrics tables.
// This is an async API which creates goroutines to set TTL.
// Status of TTL update is tracked with ttl_status table in sqlite db.
func (r *ClickHouseReader) SetTTL(ctx context.Context,
params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
// Keep only last 100 transactions/requests
_, err := r.localDB.Exec("DELETE FROM ttl_status WHERE transaction_id NOT IN (SELECT distinct transaction_id FROM ttl_status ORDER BY created_at DESC LIMIT 100)")
if err != nil {
zap.S().Debug("Error in processing ttl_status delete sql query: ", err)
}
var req, tableName string
// uuid is used as transaction id
uuidWithHyphen := uuid.New()
uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1)
......@@ -2326,12 +2331,10 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context,
statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
if err := r.db.Exec(context.Background(), req); err != nil {
zap.S().Error(fmt.Errorf("Error in executing set TTL query: %s", err.Error()))
if err != nil {
_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
if dbErr != nil {
zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr)
return
}
_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
if dbErr != nil {
zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr)
return
}
return
}
......@@ -2383,12 +2386,10 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context,
statusItem, _ := r.checkTTLStatusItem(ctx, tableName)
if err := r.db.Exec(ctx, req); err != nil {
zap.S().Error(fmt.Errorf("error while setting ttl. Err=%v", err))
if err != nil {
_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
if dbErr != nil {
zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr)
return
}
_, dbErr := r.localDB.Exec("UPDATE ttl_status SET updated_at = ?, status = ? WHERE id = ?", time.Now(), constants.StatusFailed, statusItem.Id)
if dbErr != nil {
zap.S().Debug("Error in processing ttl_status update sql query: ", dbErr)
return
}
return
}
......@@ -2407,6 +2408,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context,
return &model.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
}
// checkTTLStatusItem checks if ttl_status table has an entry for the given table name
func (r *ClickHouseReader) checkTTLStatusItem(ctx context.Context, tableName string) (model.TTLStatusItem, *model.ApiError) {
statusItem := []model.TTLStatusItem{}
......@@ -2426,6 +2428,7 @@ func (r *ClickHouseReader) checkTTLStatusItem(ctx context.Context, tableName str
return statusItem[0], nil
}
// setTTLQueryStatus fetches ttl_status table status from DB
func (r *ClickHouseReader) setTTLQueryStatus(ctx context.Context, tableNameArray []string) (string, *model.ApiError) {
failFlag := false
status := constants.StatusSuccess
......@@ -2484,6 +2487,7 @@ func (r *ClickHouseReader) GetDisks(ctx context.Context) (*[]model.DiskItem, *mo
return &diskItems, nil
}
// GetTTL returns current ttl, expected ttl and past setTTL status for metrics/traces.
func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError) {
parseTTL := func(queryResp string) (int, int) {
......@@ -2566,9 +2570,9 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa
if err != nil {
return nil, err
}
ttlQuery.TTL = ttlQuery.TTL / 3600
ttlQuery.TTL = ttlQuery.TTL / 3600 // convert to hours
if ttlQuery.ColdStorageTtl != -1 {
ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600
ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600 // convert to hours
}
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
......@@ -2588,15 +2592,15 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, ttlParams *model.GetTTLPa
if err != nil {
return nil, err
}
ttlQuery.TTL = ttlQuery.TTL / 3600
ttlQuery.TTL = ttlQuery.TTL / 3600 // convert to hours
if ttlQuery.ColdStorageTtl != -1 {
ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600
ttlQuery.ColdStorageTtl = ttlQuery.ColdStorageTtl / 3600 // convert to hours
}
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
return &model.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL, ExpectedMetricsTime: ttlQuery.TTL, ExpectedMetricsMoveTime: ttlQuery.ColdStorageTtl, Status: status}, nil
default:
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while getting ttl. ttl type should be <metrics|traces>, got %v",
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while getting ttl. ttl type should be metrics|traces, got %v",
ttlParams.Type)}
}
......
pkg/query-service/tests/auth_test.go
+ 2
- 2
View file @ cedab583
......@@ -99,7 +99,7 @@ func TestAuthInviteAPI(t *testing.T) {
func TestAuthRegisterAPI(t *testing.T) {
email := "alice@signoz.io"
resp, err := register(email, "password", "")
resp, err := register(email, "Password@123", "")
require.NoError(t, err)
require.Contains(t, resp, "user registered successfully")
......@@ -108,7 +108,7 @@ func TestAuthRegisterAPI(t *testing.T) {
func TestAuthLoginAPI(t *testing.T) {
t.Skip()
email := "abc-login@signoz.io"
password := "password123"
password := "Password@123"
inv := invite(t, email)
resp, err := register(email, password, inv.InviteToken)
......
pkg/query-service/tests/cold_storage_test.go
+ 80
- 46
View file @ cedab583
......@@ -20,12 +20,16 @@ var (
client http.Client
)
func setTTL(table, coldStorage, toColdTTL, deleteTTL string) ([]byte, error) {
func setTTL(table, coldStorage, toColdTTL, deleteTTL string, jwtToken string) ([]byte, error) {
params := fmt.Sprintf("type=%s&duration=%s", table, deleteTTL)
if len(toColdTTL) > 0 {
params += fmt.Sprintf("&coldStorage=%s&toColdDuration=%s", coldStorage, toColdTTL)
}
resp, err := client.Post(endpoint+"/api/v1/settings/ttl?"+params, "", nil)
var bearer = "Bearer " + jwtToken
req, err := http.NewRequest("POST", endpoint+"/api/v1/settings/ttl?"+params, nil)
req.Header.Add("Authorization", bearer)
resp, err := client.Do(req)
if err != nil {
return nil, err
}
......@@ -40,7 +44,18 @@ func setTTL(table, coldStorage, toColdTTL, deleteTTL string) ([]byte, error) {
}
func TestListDisks(t *testing.T) {
resp, err := client.Get(endpoint + "/api/v1/disks")
t.Skip()
email := "alice@signoz.io"
password := "Password@123"
loginResp, err := login(email, password, "")
require.NoError(t, err)
var bearer = "Bearer " + loginResp.AccessJwt
req, err := http.NewRequest("POST", endpoint+"/api/v1/disks", nil)
req.Header.Add("Authorization", bearer)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
......@@ -50,6 +65,11 @@ func TestListDisks(t *testing.T) {
}
func TestSetTTL(t *testing.T) {
email := "alice@signoz.io"
password := "Password@123"
loginResp, err := login(email, password, "")
require.NoError(t, err)
testCases := []struct {
caseNo int
......@@ -60,7 +80,7 @@ func TestSetTTL(t *testing.T) {
expected string
}{
{
1, "s3", "traces", "100s", "60s",
1, "s3", "traces", "100h", "60h",
"Delete TTL should be greater than cold storage move TTL.",
},
{
......@@ -72,21 +92,17 @@ func TestSetTTL(t *testing.T) {
"Not a valid TTL duration 100",
},
{
4, "s3", "traces", "", "60s",
4, "s3", "metrics", "1h", "2h",
"move ttl has been successfully set up",
},
{
5, "s3", "traces", "10s", "600s",
5, "s3", "traces", "10s", "6h",
"move ttl has been successfully set up",
},
{
6, "s4", "traces", "10s", "600s",
"No such volume `s4` in storage policy `tiered`",
},
}
for _, tc := range testCases {
r, err := setTTL(tc.table, tc.coldStorage, tc.coldTTL, tc.deleteTTL)
r, err := setTTL(tc.table, tc.coldStorage, tc.coldTTL, tc.deleteTTL, loginResp.AccessJwt)
require.NoErrorf(t, err, "Failed case: %d", tc.caseNo)
require.Containsf(t, string(r), tc.expected, "Failed case: %d", tc.caseNo)
}
......@@ -104,13 +120,17 @@ func TestSetTTL(t *testing.T) {
fmt.Printf("=== Found %d objects in Minio\n", count)
}
func getTTL(t *testing.T, table string) *model.GetTTLResponseItem {
req := endpoint + fmt.Sprintf("/api/v1/settings/ttl?type=%s", table)
func getTTL(t *testing.T, table string, jwtToken string) *model.GetTTLResponseItem {
url := endpoint + fmt.Sprintf("/api/v1/settings/ttl?type=%s", table)
if len(table) == 0 {
req = endpoint + "/api/v1/settings/ttl"
url = endpoint + "/api/v1/settings/ttl"
}
resp, err := client.Get(req)
var bearer = "Bearer " + jwtToken
req, err := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", bearer)
resp, err := client.Do(req)
require.NoError(t, err)
defer resp.Body.Close()
......@@ -123,55 +143,69 @@ func getTTL(t *testing.T, table string) *model.GetTTLResponseItem {
}
func TestGetTTL(t *testing.T) {
r, err := setTTL("traces", "s3", "3600s", "7200s")
email := "alice@signoz.io"
password := "Password@123"
loginResp, err := login(email, password, "")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp := getTTL(t, "traces")
require.Equal(t, 1, resp.TracesMoveTime)
require.Equal(t, 2, resp.TracesTime)
resp := getTTL(t, "traces", loginResp.AccessJwt)
for resp.Status == "pending" {
time.Sleep(time.Second)
}
require.Equal(t, "success", resp.Status)
r, err = setTTL("metrics", "s3", "3600s", "7200s")
r, err := setTTL("traces", "s3", "1h", "2h", loginResp.AccessJwt)
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "metrics")
require.Equal(t, 1, resp.MetricsMoveTime)
require.Equal(t, 2, resp.MetricsTime)
resp = getTTL(t, "traces", loginResp.AccessJwt)
for resp.Status == "pending" {
time.Sleep(time.Second)
resp = getTTL(t, "traces", loginResp.AccessJwt)
require.Equal(t, 1, resp.ExpectedTracesMoveTime)
require.Equal(t, 2, resp.ExpectedTracesTime)
}
resp = getTTL(t, "traces", loginResp.AccessJwt)
require.Equal(t, "success", resp.Status)
require.Equal(t, 1, resp.TracesMoveTime)
require.Equal(t, 2, resp.TracesTime)
resp = getTTL(t, "metrics", loginResp.AccessJwt)
for resp.Status == "pending" {
time.Sleep(time.Second)
}
require.Equal(t, "success", resp.Status)
r, err = setTTL("traces", "s3", "36000s", "72000s")
r, err = setTTL("traces", "s3", "10h", "20h", loginResp.AccessJwt)
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "")
resp = getTTL(t, "traces", loginResp.AccessJwt)
for resp.Status == "pending" {
time.Sleep(time.Second)
resp = getTTL(t, "traces", loginResp.AccessJwt)
}
require.Equal(t, "success", resp.Status)
require.Equal(t, 10, resp.TracesMoveTime)
require.Equal(t, 20, resp.TracesTime)
resp = getTTL(t, "metrics", loginResp.AccessJwt)
for resp.Status != "success" && resp.Status != "failed" {
time.Sleep(time.Second)
resp = getTTL(t, "metrics", loginResp.AccessJwt)
}
require.Equal(t, "success", resp.Status)
require.Equal(t, 1, resp.MetricsMoveTime)
require.Equal(t, 2, resp.MetricsTime)
r, err = setTTL("metrics", "s3", "15h", "50h")
r, err = setTTL("metrics", "s3", "0s", "0s", loginResp.AccessJwt)
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
require.Contains(t, string(r), "Not a valid TTL duration 0s")
resp = getTTL(t, "")
require.Equal(t, 10, resp.TracesMoveTime)
require.Equal(t, 20, resp.TracesTime)
require.Equal(t, 15, resp.MetricsMoveTime)
require.Equal(t, 50, resp.MetricsTime)
r, err = setTTL("metrics", "s3", "0s", "0s")
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
r, err = setTTL("traces", "s3", "0s", "0s")
r, err = setTTL("traces", "s3", "0s", "0s", loginResp.AccessJwt)
require.NoError(t, err)
require.Contains(t, string(r), "successfully set up")
resp = getTTL(t, "")
require.Equal(t, 0, resp.TracesMoveTime)
require.Equal(t, 0, resp.TracesTime)
require.Equal(t, 0, resp.MetricsMoveTime)
require.Equal(t, 0, resp.MetricsTime)
require.Contains(t, string(r), "Not a valid TTL duration 0s")
}
func TestMain(m *testing.M) {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

免费DevSecOps平台，让您的项目体验完整的DevSecOps流程，让项目更安全