Crud APIs for dashboards (#286)

* added signoz.db to gitignore

* model and crud methods for dashboard package

* added signoz.db to dockerignore

* feat: dashboards crud WIP

* chore: moving response format to correct file

* chore: adding dependencies for sqlite3

* feat: CRUD APIs ready for dashboards

* fix: sqlite needs cgo enabled and hence need to add some flags in building go code

* feat: provision dashboards using json

* chore: mounting dashboard folder to container

.gitignore

@@ -39,3 +39,4 @@ frontend/cypress.env.json
 **/__debug_bin
 
 frontend/*.env
+pkg/query-service/signoz.db

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -27,6 +27,7 @@ services:
       - "8080:8080"
     volumes:
       - ./prometheus.yml:/root/config/prometheus.yml
+      - ../dashboards:/root/config/dashboards
 
     environment:
       - ClickHouseUrl=tcp://clickhouse:9000

pkg/query-service/.dockerignore

 .vscode
-README.md
\ No newline at end of file
+README.md
+signoz.db
\ No newline at end of file

pkg/query-service/Dockerfile

@@ -5,7 +5,7 @@ LABEL maintainer="signoz"
 
 ARG TARGETPLATFORM
 
-ENV CGO_ENABLED=0
+ENV CGO_ENABLED=1
 ENV GOPATH=/go
 
 RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
@@ -21,7 +21,7 @@ RUN go mod download -x
 
 # Add the sources and proceed with build
 ADD . .
-RUN go build -o ./bin/query-service ./main.go
+RUN go build -a -ldflags "-linkmode external -extldflags '-static' -s -w" -o ./bin/query-service ./main.go
 RUN chmod +x ./bin/query-service
 
 # use a minimal alpine image

pkg/query-service/app/dashboards/model.go

+package dashboards
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/gosimple/slug"
+	"github.com/jmoiron/sqlx"
+	"go.signoz.io/query-service/model"
+	"go.uber.org/zap"
+)
+
+// const (
+// 	ErrorNone           ErrorType = ""
+// 	ErrorTimeout        ErrorType = "timeout"
+// 	ErrorCanceled       ErrorType = "canceled"
+// 	ErrorExec           ErrorType = "execution"
+// 	ErrorBadData        ErrorType = "bad_data"
+// 	ErrorInternal       ErrorType = "internal"
+// 	ErrorUnavailable    ErrorType = "unavailable"
+// 	ErrorNotFound       ErrorType = "not_found"
+// 	ErrorNotImplemented ErrorType = "not_implemented"
+// )
+
+// This time the global variable is unexported.
+var db *sqlx.DB
+
+// InitDB sets up setting up the connection pool global variable.
+func InitDB(dataSourceName string) error {
+	var err error
+
+	db, err = sqlx.Open("sqlite3", dataSourceName)
+	if err != nil {
+		return err
+	}
+
+	table_schema := `CREATE TABLE IF NOT EXISTS dashboards (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		uuid TEXT NOT NULL UNIQUE,
+		created_at datetime NOT NULL,
+		updated_at datetime NOT NULL,
+		data TEXT NOT NULL
+	);`
+
+	_, err = db.Exec(table_schema)
+	if err != nil {
+		return fmt.Errorf("Error in creating dashboard table: ", err.Error())
+	}
+
+	return nil
+}
+
+type Dashboard struct {
+	Id        int       `json:"id" db:"id"`
+	Uuid      string    `json:"uuid" db:"uuid"`
+	Slug      string    `json:"-" db:"-"`
+	CreatedAt time.Time `json:"created_at" db:"created_at"`
+	UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
+	Title     string    `json:"-" db:"-"`
+	Data      Data      `json:"data" db:"data"`
+}
+
+type Data map[string]interface{}
+
+// func (c *Data) Value() (driver.Value, error) {
+// 	if c != nil {
+// 		b, err := json.Marshal(c)
+// 		if err != nil {
+// 			return nil, err
+// 		}
+// 		return string(b), nil
+// 	}
+// 	return nil, nil
+// }
+
+func (c *Data) Scan(src interface{}) error {
+	var data []byte
+	if b, ok := src.([]byte); ok {
+		data = b
+	} else if s, ok := src.(string); ok {
+		data = []byte(s)
+	}
+	return json.Unmarshal(data, c)
+}
+
+// CreateDashboard creates a new dashboard
+func CreateDashboard(data *map[string]interface{}) (*Dashboard, *model.ApiError) {
+	dash := &Dashboard{
+		Data: *data,
+	}
+	dash.CreatedAt = time.Now()
+	dash.UpdatedAt = time.Now()
+	dash.UpdateSlug()
+	// dash.Uuid = uuid.New().String()
+	dash.Uuid = dash.Data["uuid"].(string)
+
+	map_data, err := json.Marshal(dash.Data)
+	if err != nil {
+		zap.S().Errorf("Error in marshalling data field in dashboard: ", dash, err)
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	// db.Prepare("Insert into dashboards where")
+	result, err := db.Exec("INSERT INTO dashboards (uuid, created_at, updated_at, data) VALUES ($1, $2, $3, $4)", dash.Uuid, dash.CreatedAt, dash.UpdatedAt, map_data)
+
+	if err != nil {
+		zap.S().Errorf("Error in inserting dashboard data: ", dash, err)
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+	lastInsertId, err := result.LastInsertId()
+
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+	dash.Id = int(lastInsertId)
+
+	return dash, nil
+}
+
+func GetDashboards() (*[]Dashboard, *model.ApiError) {
+
+	dashboards := []Dashboard{}
+	query := fmt.Sprintf("SELECT * FROM dashboards;")
+
+	err := db.Select(&dashboards, query)
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	return &dashboards, nil
+}
+
+func DeleteDashboard(uuid string) *model.ApiError {
+
+	query := fmt.Sprintf("DELETE FROM dashboards WHERE uuid='%s';", uuid)
+
+	result, err := db.Exec(query)
+
+	if err != nil {
+		return &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	affectedRows, err := result.RowsAffected()
+	if err != nil {
+		return &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+	if affectedRows == 0 {
+		return &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
+	}
+
+	return nil
+}
+
+func GetDashboard(uuid string) (*Dashboard, *model.ApiError) {
+
+	dashboard := Dashboard{}
+	query := fmt.Sprintf("SELECT * FROM dashboards WHERE uuid='%s';", uuid)
+
+	err := db.Get(&dashboard, query)
+	if err != nil {
+		return nil, &model.ApiError{Typ: model.ErrorNotFound, Err: fmt.Errorf("no dashboard found with uuid: %s", uuid)}
+	}
+
+	return &dashboard, nil
+}
+
+func UpdateDashboard(data *map[string]interface{}) (*Dashboard, *model.ApiError) {
+
+	uuid := (*data)["uuid"].(string)
+
+	map_data, err := json.Marshal(data)
+	if err != nil {
+		zap.S().Errorf("Error in marshalling data field in dashboard: ", data, err)
+		return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
+	}
+
+	dashboard, apiErr := GetDashboard(uuid)
+	if apiErr != nil {
+		return nil, apiErr
+	}
+
+	dashboard.UpdatedAt = time.Now()
+	dashboard.Data = *data
+
+	// db.Prepare("Insert into dashboards where")
+	_, err = db.Exec("UPDATE dashboards SET updated_at=$1, data=$2 WHERE uuid=$3 ", dashboard.UpdatedAt, map_data, dashboard.Uuid)
+
+	if err != nil {
+		zap.S().Errorf("Error in inserting dashboard data: ", data, err)
+		return nil, &model.ApiError{Typ: model.ErrorExec, Err: err}
+	}
+
+	return dashboard, nil
+}
+
+// UpdateSlug updates the slug
+func (d *Dashboard) UpdateSlug() {
+	var title string
+
+	if val, ok := d.Data["title"]; ok {
+		title = val.(string)
+	}
+
+	d.Slug = SlugifyTitle(title)
+}
+
+func IsPostDataSane(data *map[string]interface{}) error {
+
+	val, ok := (*data)["uuid"]
+	if !ok || val == nil {
+		return fmt.Errorf("uuid not found in post data")
+	}
+
+	val, ok = (*data)["title"]
+	if !ok || val == nil {
+		return fmt.Errorf("title not found in post data")
+	}
+
+	return nil
+}
+
+func SlugifyTitle(title string) string {
+	s := slug.Make(strings.ToLower(title))
+	if s == "" {
+		// If the dashboard name is only characters outside of the
+		// sluggable characters, the slug creation will return an
+		// empty string which will mess up URLs. This failsafe picks
+		// that up and creates the slug as a base64 identifier instead.
+		s = base64.RawURLEncoding.EncodeToString([]byte(title))
+		if slug.MaxLength != 0 && len(s) > slug.MaxLength {
+			s = s[:slug.MaxLength]
+		}
+	}
+	return s
+}

pkg/query-service/app/dashboards/provision.go

+package dashboards
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+
+	"go.uber.org/zap"
+)
+
+func readCurrentDir(dir string) error {
+	file, err := os.Open(dir)
+	if err != nil {
+		zap.S().Errorf("failed opening directory: %s", err)
+		return err
+	}
+	defer file.Close()
+
+	list, _ := file.Readdirnames(0) // 0 to read all files and folders
+	for _, filename := range list {
+		// fmt.Println(filename)
+		zap.S().Info("Provisioning dashboard: ", filename)
+		plan, err := ioutil.ReadFile(dir + "/" + filename)
+		if err != nil {
+			zap.S().Errorf("Creating Dashboards: Error in reading json fron file: %s\t%s", filename, err)
+			continue
+		}
+		var data map[string]interface{}
+		err = json.Unmarshal(plan, &data)
+		if err != nil {
+			zap.S().Errorf("Creating Dashboards: Error in unmarshalling json from file: %s\t%s", filename, err)
+			continue
+		}
+		err = IsPostDataSane(&data)
+		if err != nil {
+			zap.S().Infof("Creating Dashboards: Error in file: %s\t%s", filename, err)
+			continue
+		}
+
+		_, apiErr := GetDashboard(data["uuid"].(string))
+		if apiErr == nil {
+			zap.S().Infof("Creating Dashboards: Error in file: %s\t%s", filename, "Dashboard already present in database")
+			continue
+		}
+
+		_, apiErr = CreateDashboard(&data)
+		if apiErr != nil {
+			zap.S().Errorf("Creating Dashboards: Error in file: %s\t%s", filename, apiErr.Err)
+			continue
+		}
+
+	}
+	return nil
+}
+
+func LoadDashboardFiles() error {
+	return readCurrentDir("./config/dashboards")
+}

pkg/query-service/app/http_handler.go

@@ -7,9 +7,12 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/jmoiron/sqlx"
 	jsoniter "github.com/json-iterator/go"
+	_ "github.com/mattn/go-sqlite3"
 	"github.com/posthog/posthog-go"
 	"github.com/prometheus/prometheus/promql"
+	"go.signoz.io/query-service/app/dashboards"
 	"go.signoz.io/query-service/model"
 	"go.uber.org/zap"
 )
@@ -35,11 +38,12 @@ type APIHandler struct {
 	reader     *Reader
 	pc         *posthog.Client
 	distinctId string
+	db         *sqlx.DB
 	ready      func(http.HandlerFunc) http.HandlerFunc
 }
 
 // NewAPIHandler returns an APIHandler
-func NewAPIHandler(reader *Reader, pc *posthog.Client, distinctId string) *APIHandler {
+func NewAPIHandler(reader *Reader, pc *posthog.Client, distinctId string) (*APIHandler, error) {
 
 	aH := &APIHandler{
 		reader:     reader,
@@ -47,7 +51,17 @@ func NewAPIHandler(reader *Reader, pc *posthog.Client, distinctId string) *APIHa
 		distinctId: distinctId,
 	}
 	aH.ready = aH.testReady
-	return aH
+
+	err := dashboards.InitDB("signoz.db")
+	if err != nil {
+		return nil, err
+	}
+
+	errReadingDashboards := dashboards.LoadDashboardFiles()
+	if errReadingDashboards != nil {
+		return nil, errReadingDashboards
+	}
+	return aH, nil
 }
 
 type structuredResponse struct {
@@ -159,6 +173,12 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router) {
 	router.HandleFunc("/api/v1/query_range", aH.queryRangeMetrics).Methods(http.MethodGet)
 	router.HandleFunc("/api/v1/query", aH.queryMetrics).Methods(http.MethodGet)
 
+	router.HandleFunc("/api/v1/dashboards", aH.getDashboards).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/dashboards", aH.createDashboards).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/dashboards/{uuid}", aH.getDashboard).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/dashboards/{uuid}", aH.updateDashboard).Methods(http.MethodPut)
+	router.HandleFunc("/api/v1/dashboards/{uuid}", aH.deleteDashboard).Methods(http.MethodDelete)
+
 	router.HandleFunc("/api/v1/user", aH.user).Methods(http.MethodPost)
 	// router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet)
 	router.HandleFunc("/api/v1/services", aH.getServices).Methods(http.MethodGet)
@@ -178,6 +198,157 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router) {
 	router.HandleFunc("/api/v1/serviceMapDependencies", aH.serviceMapDependencies).Methods(http.MethodGet)
 }
 
+func Intersection(a, b []int) (c []int) {
+	m := make(map[int]bool)
+
+	for _, item := range a {
+		m[item] = true
+	}
+
+	for _, item := range b {
+		if _, ok := m[item]; ok {
+			c = append(c, item)
+		}
+	}
+	return
+}
+
+func (aH *APIHandler) getDashboards(w http.ResponseWriter, r *http.Request) {
+
+	allDashboards, err := dashboards.GetDashboards()
+
+	if err != nil {
+		aH.respondError(w, err, nil)
+		return
+	}
+	tagsFromReq, ok := r.URL.Query()["tags"]
+	if !ok || len(tagsFromReq) == 0 || tagsFromReq[0] == "" {
+		aH.respond(w, &allDashboards)
+		return
+	}
+
+	tags2Dash := make(map[string][]int)
+	for i := 0; i < len(*allDashboards); i++ {
+		tags, ok := (*allDashboards)[i].Data["tags"].([]interface{})
+		if !ok {
+			continue
+		}
+
+		tagsArray := make([]string, len(tags))
+		for i, v := range tags {
+			tagsArray[i] = v.(string)
+		}
+
+		for _, tag := range tagsArray {
+			tags2Dash[tag] = append(tags2Dash[tag], i)
+		}
+
+	}
+
+	inter := make([]int, len(*allDashboards))
+	for i := range inter {
+		inter[i] = i
+	}
+
+	for _, tag := range tagsFromReq {
+		inter = Intersection(inter, tags2Dash[tag])
+	}
+
+	filteredDashboards := []dashboards.Dashboard{}
+	for _, val := range inter {
+		dash := (*allDashboards)[val]
+		filteredDashboards = append(filteredDashboards, dash)
+	}
+
+	aH.respond(w, &filteredDashboards)
+
+}
+func (aH *APIHandler) deleteDashboard(w http.ResponseWriter, r *http.Request) {
+
+	uuid := mux.Vars(r)["uuid"]
+	err := dashboards.DeleteDashboard(uuid)
+
+	if err != nil {
+		aH.respondError(w, err, nil)
+		return
+	}
+
+	aH.respond(w, nil)
+
+}
+
+func (aH *APIHandler) updateDashboard(w http.ResponseWriter, r *http.Request) {
+
+	uuid := mux.Vars(r)["uuid"]
+
+	var postData map[string]interface{}
+	err := json.NewDecoder(r.Body).Decode(&postData)
+	if err != nil {
+		aH.respondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading request body")
+		return
+	}
+	err = dashboards.IsPostDataSane(&postData)
+	if err != nil {
+		aH.respondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, "Error reading request body")
+		return
+	}
+
+	if postData["uuid"] != uuid {
+		aH.respondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("uuid in request param and uuid in request body do not match")}, "Error reading request body")
+		return
+	}
+
+	dashboard, apiError := dashboards.UpdateDashboard(&postData)
+
+	if apiError != nil {
+		aH.respondError(w, apiError, nil)
+		return
+	}
+
+	aH.respond(w, dashboard)
+
+}
+
+func (aH *APIHandler) getDashboard(w http.ResponseWriter, r *http.Request) {
+
+	uuid := mux.Vars(r)["uuid"]
+
+	dashboard, apiError := dashboards.GetDashboard(uuid)
+
+	if apiError != nil {
+		aH.respondError(w, apiError, nil)
+		return
+	}
+
+	aH.respond(w, dashboard)
+
+}
+
+func (aH *APIHandler) createDashboards(w http.ResponseWriter, r *http.Request) {
+
+	var postData map[string]interface{}
+	err := json.NewDecoder(r.Body).Decode(&postData)
+	if err != nil {
+		aH.respondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, "Error reading request body")
+		return
+	}
+	err = dashboards.IsPostDataSane(&postData)
+	if err != nil {
+		aH.respondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, "Error reading request body")
+		return
+	}
+
+	dash, apiErr := dashboards.CreateDashboard(&postData)
+
+	if apiErr != nil {
+		aH.respondError(w, apiErr, nil)
+		return
+	}
+
+	aH.respond(w, dash)
+
+}
+
 func (aH *APIHandler) queryRangeMetrics(w http.ResponseWriter, r *http.Request) {
 
 	query, apiErrorObj := parseQueryRangeRequest(r)

pkg/query-service/app/server.go

@@ -106,7 +106,11 @@ func createHTTPServer() (*http.Server, error) {
 		return nil, fmt.Errorf("Storage type: %s is not supported in query service", storage)
 	}
 
-	apiHandler := NewAPIHandler(&reader, &posthogClient, distinctId)
+	apiHandler, err := NewAPIHandler(&reader, &posthogClient, distinctId)
+	if err != nil {
+		return nil, err
+	}
+
 	r := NewRouter()
 
 	r.Use(analyticsMiddleware)

pkg/query-service/go.mod

@@ -14,9 +14,9 @@ require (
 	github.com/go-kit/log v0.1.0
 	github.com/gogo/protobuf v1.2.1 // indirect
 	github.com/google/uuid v1.1.2
-	github.com/googleapis/gnostic v0.2.3-0.20180520015035-48a0ecefe2e4 // indirect
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/mux v1.8.0
+	github.com/gosimple/slug v1.10.0
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
@@ -28,8 +28,8 @@ require (
 	github.com/jmoiron/sqlx v1.3.4
 	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.8
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/miekg/dns v1.0.4 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect

pkg/query-service/model/queryParams.go

@@ -5,24 +5,6 @@ import (
 	"time"
 )
 
-type ApiError struct {
-	Typ ErrorType
-	Err error
-}
-type ErrorType string
-
-const (
-	ErrorNone           ErrorType = ""
-	ErrorTimeout        ErrorType = "timeout"
-	ErrorCanceled       ErrorType = "canceled"
-	ErrorExec           ErrorType = "execution"
-	ErrorBadData        ErrorType = "bad_data"
-	ErrorInternal       ErrorType = "internal"
-	ErrorUnavailable    ErrorType = "unavailable"
-	ErrorNotFound       ErrorType = "not_found"
-	ErrorNotImplemented ErrorType = "not_implemented"
-)
-
 type InstantQueryMetricsParams struct {
 	Time  time.Time
 	Query string

pkg/query-service/model/response.go

@@ -10,6 +10,24 @@ import (
 	"github.com/prometheus/prometheus/util/stats"
 )
 
+type ApiError struct {
+	Typ ErrorType
+	Err error
+}
+type ErrorType string
+
+const (
+	ErrorNone           ErrorType = ""
+	ErrorTimeout        ErrorType = "timeout"
+	ErrorCanceled       ErrorType = "canceled"
+	ErrorExec           ErrorType = "execution"
+	ErrorBadData        ErrorType = "bad_data"
+	ErrorInternal       ErrorType = "internal"
+	ErrorUnavailable    ErrorType = "unavailable"
+	ErrorNotFound       ErrorType = "not_found"
+	ErrorNotImplemented ErrorType = "not_implemented"
+)
+
 type QueryData struct {
 	ResultType promql.ValueType  `json:"resultType"`
 	Result     promql.Value      `json:"result"`