Unverified Commit 5eed384f authored by Ankit Nayan's avatar Ankit Nayan Committed by GitHub
Browse files

Merge pull request #1637 from SigNoz/release/v0.11.2

Release/v0.11.2
parents adda2e8a 6a3c1c10
Showing with 1498 additions and 3 deletions
+1498 -3
package license
import (
"context"
"fmt"
"sync/atomic"
"time"
"github.com/jmoiron/sqlx"
"sync"
validate "go.signoz.io/signoz/ee/query-service/integrations/signozio"
"go.signoz.io/signoz/ee/query-service/model"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/telemetry"
"go.uber.org/zap"
)
var LM *Manager
// validate and update license every 24 hours
var validationFrequency = 24 * 60 * time.Minute
type Manager struct {
repo *Repo
mutex sync.Mutex
validatorRunning bool
// end the license validation, this is important to gracefully
// stopping validation and protect in-consistent updates
done chan struct{}
// terminated waits for the validate go routine to end
terminated chan struct{}
// last time the license was validated
lastValidated int64
// keep track of validation failure attempts
failedAttempts uint64
// keep track of active license and features
activeLicense *model.License
activeFeatures basemodel.FeatureSet
}
func StartManager(dbType string, db *sqlx.DB) (*Manager, error) {
if LM != nil {
return LM, nil
}
repo := NewLicenseRepo(db)
err := repo.InitDB(dbType)
if err != nil {
return nil, fmt.Errorf("failed to initiate license repo: %v", err)
}
m := &Manager{
repo: &repo,
}
if err := m.start(); err != nil {
return m, err
}
LM = m
return m, nil
}
// start loads active license in memory and initiates validator
func (lm *Manager) start() error {
err := lm.LoadActiveLicense()
return err
}
func (lm *Manager) Stop() {
close(lm.done)
<-lm.terminated
}
func (lm *Manager) SetActive(l *model.License) {
lm.mutex.Lock()
defer lm.mutex.Unlock()
if l == nil {
return
}
lm.activeLicense = l
lm.activeFeatures = l.FeatureSet
if !lm.validatorRunning {
// we want to make sure only one validator runs,
// we already have lock() so good to go
lm.validatorRunning = true
go lm.Validator(context.Background())
}
}
// LoadActiveLicense loads the most recent active licenseex
func (lm *Manager) LoadActiveLicense() error {
var err error
active, err := lm.repo.GetActiveLicense(context.Background())
if err != nil {
return err
}
if active != nil {
lm.SetActive(active)
} else {
zap.S().Info("No active license found.")
}
return nil
}
func (lm *Manager) GetLicenses(ctx context.Context) (response []model.License, apiError *model.ApiError) {
licenses, err := lm.repo.GetLicenses(ctx)
if err != nil {
return nil, model.InternalError(err)
}
for _, l := range licenses {
l.ParsePlan()
if l.Key == lm.activeLicense.Key {
l.IsCurrent = true
}
if l.ValidUntil == -1 {
// for subscriptions, there is no end-date as such
// but for showing user some validity we default one year timespan
l.ValidUntil = l.ValidFrom + 31556926
}
response = append(response, l)
}
return
}
// Validator validates license after an epoch of time
func (lm *Manager) Validator(ctx context.Context) {
defer close(lm.terminated)
tick := time.NewTicker(validationFrequency)
defer tick.Stop()
lm.Validate(ctx)
for {
select {
case <-lm.done:
return
default:
select {
case <-lm.done:
return
case <-tick.C:
lm.Validate(ctx)
}
}
}
}
// Validate validates the current active license
func (lm *Manager) Validate(ctx context.Context) (reterr error) {
zap.S().Info("License validation started")
if lm.activeLicense == nil {
return nil
}
defer func() {
lm.mutex.Lock()
lm.lastValidated = time.Now().Unix()
if reterr != nil {
zap.S().Errorf("License validation completed with error", reterr)
atomic.AddUint64(&lm.failedAttempts, 1)
telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_LICENSE_CHECK_FAILED,
map[string]interface{}{"err": reterr.Error()})
} else {
zap.S().Info("License validation completed with no errors")
}
lm.mutex.Unlock()
}()
response, apiError := validate.ValidateLicense(lm.activeLicense.ActivationId)
if apiError != nil {
zap.S().Errorf("failed to validate license", apiError)
return apiError.Err
}
if response.PlanDetails == lm.activeLicense.PlanDetails {
// license plan hasnt changed, nothing to do
return nil
}
if response.PlanDetails != "" {
// copy and replace the active license record
l := model.License{
Key: lm.activeLicense.Key,
CreatedAt: lm.activeLicense.CreatedAt,
PlanDetails: response.PlanDetails,
ValidationMessage: lm.activeLicense.ValidationMessage,
ActivationId: lm.activeLicense.ActivationId,
}
if err := l.ParsePlan(); err != nil {
zap.S().Errorf("failed to parse updated license", zap.Error(err))
return err
}
// updated plan is parsable, check if plan has changed
if lm.activeLicense.PlanDetails != response.PlanDetails {
err := lm.repo.UpdatePlanDetails(ctx, lm.activeLicense.Key, response.PlanDetails)
if err != nil {
// unexpected db write issue but we can let the user continue
// and wait for update to work in next cycle.
zap.S().Errorf("failed to validate license", zap.Error(err))
}
}
// activate the update license plan
lm.SetActive(&l)
}
return nil
}
// Activate activates a license key with signoz server
func (lm *Manager) Activate(ctx context.Context, key string) (licenseResponse *model.License, errResponse *model.ApiError) {
defer func() {
if errResponse != nil {
telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_LICENSE_ACT_FAILED,
map[string]interface{}{"err": errResponse.Err.Error()})
}
}()
response, apiError := validate.ActivateLicense(key, "")
if apiError != nil {
zap.S().Errorf("failed to activate license", zap.Error(apiError.Err))
return nil, apiError
}
l := &model.License{
Key: key,
ActivationId: response.ActivationId,
PlanDetails: response.PlanDetails,
}
// parse validity and features from the plan details
err := l.ParsePlan()
if err != nil {
zap.S().Errorf("failed to activate license", zap.Error(err))
return nil, model.InternalError(err)
}
// store the license before activating it
err = lm.repo.InsertLicense(ctx, l)
if err != nil {
zap.S().Errorf("failed to activate license", zap.Error(err))
return nil, model.InternalError(err)
}
// license is valid, activate it
lm.SetActive(l)
return l, nil
}
// CheckFeature will be internally used by backend routines
// for feature gating
func (lm *Manager) CheckFeature(featureKey string) error {
if _, ok := lm.activeFeatures[featureKey]; ok {
return nil
}
return basemodel.ErrFeatureUnavailable{Key: featureKey}
}
// GetFeatureFlags returns current active features
func (lm *Manager) GetFeatureFlags() basemodel.FeatureSet {
return lm.activeFeatures
}
// GetRepo return the license repo
func (lm *Manager) GetRepo() *Repo {
return lm.repo
}
package sqlite
import (
"fmt"
"github.com/jmoiron/sqlx"
)
func InitDB(db *sqlx.DB) error {
var err error
if db == nil {
return fmt.Errorf("invalid db connection")
}
table_schema := `CREATE TABLE IF NOT EXISTS licenses(
key TEXT PRIMARY KEY,
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
planDetails TEXT,
activationId TEXT,
validationMessage TEXT,
lastValidated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS sites(
uuid TEXT PRIMARY KEY,
alias VARCHAR(180) DEFAULT 'PROD',
url VARCHAR(300),
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
`
_, err = db.Exec(table_schema)
if err != nil {
return fmt.Errorf("Error in creating licenses table: %s", err.Error())
}
return nil
}
package main
import (
"context"
"flag"
"os"
"os/signal"
"syscall"
"go.signoz.io/signoz/ee/query-service/app"
"go.signoz.io/signoz/pkg/query-service/auth"
baseconst "go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/version"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
func initZapLog() *zap.Logger {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, _ := config.Build()
return logger
}
func main() {
var promConfigPath string
// disables rule execution but allows change to the rule definition
var disableRules bool
// the url used to build link in the alert messages in slack and other systems
var ruleRepoURL string
flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)")
flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)")
flag.StringVar(&ruleRepoURL, "rules.repo-url", baseconst.AlertHelpPage, "(host address used to build rule link in alert messages)")
flag.Parse()
loggerMgr := initZapLog()
zap.ReplaceGlobals(loggerMgr)
defer loggerMgr.Sync() // flushes buffer, if any
logger := loggerMgr.Sugar()
version.PrintVersion()
serverOptions := &app.ServerOptions{
HTTPHostPort: baseconst.HTTPHostPort,
PromConfigPath: promConfigPath,
PrivateHostPort: baseconst.PrivateHostPort,
DisableRules: disableRules,
RuleRepoURL: ruleRepoURL,
}
// Read the jwt secret key
auth.JwtSecret = os.Getenv("SIGNOZ_JWT_SECRET")
if len(auth.JwtSecret) == 0 {
zap.S().Warn("No JWT secret key is specified.")
} else {
zap.S().Info("No JWT secret key set successfully.")
}
server, err := app.NewServer(serverOptions)
if err != nil {
logger.Fatal("Failed to create server", zap.Error(err))
}
if err := server.Start(); err != nil {
logger.Fatal("Could not start servers", zap.Error(err))
}
if err := auth.InitAuthCache(context.Background()); err != nil {
logger.Fatal("Failed to initialize auth cache", zap.Error(err))
}
signalsChannel := make(chan os.Signal, 1)
signal.Notify(signalsChannel, os.Interrupt, syscall.SIGTERM)
for {
select {
case status := <-server.HealthCheckStatus():
logger.Info("Received HealthCheck status: ", zap.Int("status", int(status)))
case <-signalsChannel:
logger.Fatal("Received OS Interrupt Signal ... ")
}
}
}
package model
import (
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
// PrecheckResponse contains login precheck response
type PrecheckResponse struct {
SSO bool `json:"sso"`
SsoUrl string `json:"ssoUrl"`
CanSelfRegister bool `json:"canSelfRegister"`
IsUser bool `json:"isUser"`
SsoError string `json:"ssoError"`
}
// GettableInvitation overrides base object and adds precheck into
// response
type GettableInvitation struct {
*basemodel.InvitationResponseObject
Precheck *PrecheckResponse `json:"precheck"`
}
package model
import (
"encoding/json"
"fmt"
"net/url"
"strings"
"github.com/google/uuid"
"github.com/pkg/errors"
saml2 "github.com/russellhaering/gosaml2"
"go.signoz.io/signoz/ee/query-service/saml"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type SSOType string
const (
SAML SSOType = "SAML"
GoogleAuth SSOType = "GOOGLE_AUTH"
)
type SamlConfig struct {
SamlEntity string `json:"samlEntity"`
SamlIdp string `json:"samlIdp"`
SamlCert string `json:"samlCert"`
}
// OrgDomain identify org owned web domains for auth and other purposes
type OrgDomain struct {
Id uuid.UUID `json:"id"`
Name string `json:"name"`
OrgId string `json:"orgId"`
SsoEnabled bool `json:"ssoEnabled"`
SsoType SSOType `json:"ssoType"`
SamlConfig *SamlConfig `json:"samlConfig"`
Org *basemodel.Organization
}
// Valid is used a pipeline function to check if org domain
// loaded from db is valid
func (od *OrgDomain) Valid(err error) error {
if err != nil {
return err
}
if od.Id == uuid.Nil || od.OrgId == "" {
return fmt.Errorf("both id and orgId are required")
}
return nil
}
// ValidNew cheks if the org domain is valid for insertion in db
func (od *OrgDomain) ValidNew() error {
if od.OrgId == "" {
return fmt.Errorf("orgId is required")
}
if od.Name == "" {
return fmt.Errorf("name is required")
}
return nil
}
// LoadConfig loads config params from json text
func (od *OrgDomain) LoadConfig(jsondata string) error {
d := *od
err := json.Unmarshal([]byte(jsondata), &d)
if err != nil {
return errors.Wrap(err, "failed to marshal json to OrgDomain{}")
}
*od = d
return nil
}
func (od *OrgDomain) GetSAMLEntityID() string {
if od.SamlConfig != nil {
return od.SamlConfig.SamlEntity
}
return ""
}
func (od *OrgDomain) GetSAMLIdpURL() string {
if od.SamlConfig != nil {
return od.SamlConfig.SamlIdp
}
return ""
}
func (od *OrgDomain) GetSAMLCert() string {
if od.SamlConfig != nil {
return od.SamlConfig.SamlCert
}
return ""
}
// PrepareSamlRequest creates a request accordingly gosaml2
func (od *OrgDomain) PrepareSamlRequest(siteUrl *url.URL) (*saml2.SAMLServiceProvider, error) {
// this is the url Idp will call after login completion
acs := fmt.Sprintf("%s://%s/%s",
siteUrl.Scheme,
siteUrl.Host,
"api/v1/complete/saml")
// this is the address of the calling url, useful to redirect user
sourceUrl := fmt.Sprintf("%s://%s%s",
siteUrl.Scheme,
siteUrl.Host,
siteUrl.Path)
// ideally this should be some unique ID for each installation
// but since we dont have UI to support it, we default it to
// host. this issuer is an identifier of service provider (signoz)
// on id provider (e.g. azure, okta). Azure requires this id to be configured
// in their system, while others seem to not care about it.
// currently we default it to host from window.location (received from browser)
issuer := siteUrl.Host
return saml.PrepareRequest(issuer, acs, sourceUrl, od.GetSAMLEntityID(), od.GetSAMLIdpURL(), od.GetSAMLCert())
}
func (od *OrgDomain) BuildSsoUrl(siteUrl *url.URL) (ssoUrl string, err error) {
sp, err := od.PrepareSamlRequest(siteUrl)
if err != nil {
return "", err
}
fmtDomainId := strings.Replace(od.Id.String(), "-", ":", -1)
relayState := fmt.Sprintf("%s://%s%s?domainId=%s",
siteUrl.Scheme,
siteUrl.Host,
siteUrl.Path,
fmtDomainId)
return sp.BuildAuthURL(relayState)
}
package model
import (
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type ApiError struct {
Typ basemodel.ErrorType
Err error
}
func (a *ApiError) Type() basemodel.ErrorType {
return a.Typ
}
func (a *ApiError) ToError() error {
if a != nil {
return a.Err
}
return a.Err
}
func (a *ApiError) Error() string {
return a.Err.Error()
}
func (a *ApiError) IsNil() bool {
return a == nil || a.Err == nil
}
// NewApiError returns a ApiError object of given type
func NewApiError(typ basemodel.ErrorType, err error) *ApiError {
return &ApiError{
Typ: typ,
Err: err,
}
}
// BadRequest returns a ApiError object of bad request
func BadRequest(err error) *ApiError {
return &ApiError{
Typ: basemodel.ErrorBadData,
Err: err,
}
}
// InternalError returns a ApiError object of internal type
func InternalError(err error) *ApiError {
return &ApiError{
Typ: basemodel.ErrorInternal,
Err: err,
}
}
var (
ErrorNone basemodel.ErrorType = ""
ErrorTimeout basemodel.ErrorType = "timeout"
ErrorCanceled basemodel.ErrorType = "canceled"
ErrorExec basemodel.ErrorType = "execution"
ErrorBadData basemodel.ErrorType = "bad_data"
ErrorInternal basemodel.ErrorType = "internal"
ErrorUnavailable basemodel.ErrorType = "unavailable"
ErrorNotFound basemodel.ErrorType = "not_found"
ErrorNotImplemented basemodel.ErrorType = "not_implemented"
ErrorUnauthorized basemodel.ErrorType = "unauthorized"
ErrorForbidden basemodel.ErrorType = "forbidden"
ErrorConflict basemodel.ErrorType = "conflict"
ErrorStreamingNotSupported basemodel.ErrorType = "streaming is not supported"
)
func init() {
ErrorNone = basemodel.ErrorNone
ErrorTimeout = basemodel.ErrorTimeout
ErrorCanceled = basemodel.ErrorCanceled
ErrorExec = basemodel.ErrorExec
ErrorBadData = basemodel.ErrorBadData
ErrorInternal = basemodel.ErrorInternal
ErrorUnavailable = basemodel.ErrorUnavailable
ErrorNotFound = basemodel.ErrorNotFound
ErrorNotImplemented = basemodel.ErrorNotImplemented
ErrorUnauthorized = basemodel.ErrorUnauthorized
ErrorForbidden = basemodel.ErrorForbidden
ErrorConflict = basemodel.ErrorConflict
ErrorStreamingNotSupported = basemodel.ErrorStreamingNotSupported
}
type ErrUnsupportedAuth struct{}
func (errUnsupportedAuth ErrUnsupportedAuth) Error() string {
return "this authentication method not supported"
}
package model
import (
"encoding/base64"
"encoding/json"
"time"
"github.com/pkg/errors"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type License struct {
Key string `json:"key" db:"key"`
ActivationId string `json:"activationId" db:"activationId"`
CreatedAt time.Time `db:"created_at"`
// PlanDetails contains the encrypted plan info
PlanDetails string `json:"planDetails" db:"planDetails"`
// stores parsed license details
LicensePlan
FeatureSet basemodel.FeatureSet
// populated in case license has any errors
ValidationMessage string `db:"validationMessage"`
// used only for sending details to front-end
IsCurrent bool `json:"isCurrent"`
}
func (l *License) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
Key string `json:"key" db:"key"`
ActivationId string `json:"activationId" db:"activationId"`
ValidationMessage string `db:"validationMessage"`
IsCurrent bool `json:"isCurrent"`
PlanKey string `json:"planKey"`
ValidFrom time.Time `json:"ValidFrom"`
ValidUntil time.Time `json:"ValidUntil"`
Status string `json:"status"`
}{
Key: l.Key,
ActivationId: l.ActivationId,
IsCurrent: l.IsCurrent,
PlanKey: l.PlanKey,
ValidFrom: time.Unix(l.ValidFrom, 0),
ValidUntil: time.Unix(l.ValidUntil, 0),
Status: l.Status,
ValidationMessage: l.ValidationMessage,
})
}
type LicensePlan struct {
PlanKey string `json:"planKey"`
ValidFrom int64 `json:"validFrom"`
ValidUntil int64 `json:"validUntil"`
Status string `json:"status"`
}
func (l *License) ParsePlan() error {
l.LicensePlan = LicensePlan{}
planData, err := base64.StdEncoding.DecodeString(l.PlanDetails)
if err != nil {
return err
}
plan := LicensePlan{}
err = json.Unmarshal([]byte(planData), &plan)
if err != nil {
l.ValidationMessage = "failed to parse plan from license"
return errors.Wrap(err, "failed to parse plan from license")
}
l.LicensePlan = plan
l.ParseFeatures()
return nil
}
func (l *License) ParseFeatures() {
switch l.PlanKey {
case Pro:
l.FeatureSet = ProPlan
case Enterprise:
l.FeatureSet = EnterprisePlan
default:
l.FeatureSet = BasicPlan
}
}
package model
import (
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
const SSO = "SSO"
const Basic = "BASIC_PLAN"
const Pro = "PRO_PLAN"
const Enterprise = "ENTERPRISE_PLAN"
const DisableUpsell = "DISABLE_UPSELL"
var BasicPlan = basemodel.FeatureSet{
Basic: true,
SSO: false,
DisableUpsell: false,
}
var ProPlan = basemodel.FeatureSet{
Pro: true,
SSO: true,
}
var EnterprisePlan = basemodel.FeatureSet{
Enterprise: true,
SSO: true,
}
package model
import (
"time"
"github.com/google/uuid"
)
type UsageSnapshot struct {
CurrentLogSizeBytes uint64 `json:"currentLogSizeBytes"`
CurrentLogSizeBytesColdStorage uint64 `json:"currentLogSizeBytesColdStorage"`
CurrentSpansCount uint64 `json:"currentSpansCount"`
CurrentSpansCountColdStorage uint64 `json:"currentSpansCountColdStorage"`
CurrentSamplesCount uint64 `json:"currentSamplesCount"`
CurrentSamplesCountColdStorage uint64 `json:"currentSamplesCountColdStorage"`
}
type UsageBase struct {
Id uuid.UUID `json:"id" db:"id"`
InstallationId uuid.UUID `json:"installationId" db:"installation_id"`
ActivationId uuid.UUID `json:"activationId" db:"activation_id"`
CreatedAt time.Time `json:"createdAt" db:"created_at"`
FailedSyncRequest int `json:"failedSyncRequest" db:"failed_sync_request_count"`
}
type UsagePayload struct {
UsageBase
Metrics UsageSnapshot `json:"metrics"`
SnapshotDate time.Time `json:"snapshotDate"`
}
type Usage struct {
UsageBase
Snapshot string `db:"snapshot"`
}
package saml
import (
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"strings"
saml2 "github.com/russellhaering/gosaml2"
dsig "github.com/russellhaering/goxmldsig"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.uber.org/zap"
)
func LoadCertificateStore(certString string) (dsig.X509CertificateStore, error) {
certStore := &dsig.MemoryX509CertificateStore{
Roots: []*x509.Certificate{},
}
certData, err := base64.StdEncoding.DecodeString(certString)
if err != nil {
return certStore, fmt.Errorf(fmt.Sprintf("failed to read certificate: %v", err))
}
idpCert, err := x509.ParseCertificate(certData)
if err != nil {
return certStore, fmt.Errorf(fmt.Sprintf("failed to prepare saml request, invalid cert: %s", err.Error()))
}
certStore.Roots = append(certStore.Roots, idpCert)
return certStore, nil
}
func LoadCertFromPem(certString string) (dsig.X509CertificateStore, error) {
certStore := &dsig.MemoryX509CertificateStore{
Roots: []*x509.Certificate{},
}
block, _ := pem.Decode([]byte(certString))
if block == nil {
return certStore, fmt.Errorf("no valid pem cert found")
}
idpCert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return certStore, fmt.Errorf(fmt.Sprintf("failed to parse pem cert: %s", err.Error()))
}
certStore.Roots = append(certStore.Roots, idpCert)
return certStore, nil
}
// PrepareRequest prepares authorization URL (Idp Provider URL)
func PrepareRequest(issuer, acsUrl, audience, entity, idp, certString string) (*saml2.SAMLServiceProvider, error) {
var certStore dsig.X509CertificateStore
if certString == "" {
return nil, fmt.Errorf("invalid certificate data")
}
var err error
if strings.Contains(certString, "-----BEGIN CERTIFICATE-----") {
certStore, err = LoadCertFromPem(certString)
} else {
certStore, err = LoadCertificateStore(certString)
}
// certificate store can not be created, throw error
if err != nil {
return nil, err
}
randomKeyStore := dsig.RandomKeyStoreForTest()
// SIGNOZ_SAML_RETURN_URL env var would support overriding window.location
// as return destination after saml request is complete from IdP side.
// this var is also useful for development, as it is easy to override with backend endpoint
// e.g. http://localhost:8080/api/v1/complete/saml
acsUrl = constants.GetOrDefaultEnv("SIGNOZ_SAML_RETURN_URL", acsUrl)
sp := &saml2.SAMLServiceProvider{
IdentityProviderSSOURL: idp,
IdentityProviderIssuer: entity,
ServiceProviderIssuer: issuer,
AssertionConsumerServiceURL: acsUrl,
SignAuthnRequests: true,
AllowMissingAttributes: true,
// about cert stores -sender(signoz app) and receiver (idp)
// The random key (random key store) is sender cert. The public cert store(IDPCertificateStore) that you see on org domain is receiver cert (idp provided).
// At the moment, the library we use doesn't bother about sender cert and IdP too. It just adds additional layer of security, which we can explore in future versions
// The receiver (Idp) cert will be different for each org domain. Imagine cloud setup where each company setups their domain that integrates with their Idp.
// @signoz.io
// @next.io
// Each of above will have their own Idp setup and hence separate public cert to decrypt the response.
// The way SAML request travels is -
// SigNoz Backend -> IdP Login Screen -> SigNoz Backend -> SigNoz Frontend
// ---------------- | -------------------| -------------------------------------
// The dotted lines indicate request boundries. So if you notice, the response from Idp starts a new request. hence we need relay state to pass the context around.
IDPCertificateStore: certStore,
SPKeyStore: randomKeyStore,
}
zap.S().Debugf("SAML request:", sp)
return sp, nil
}
package usage
import (
"context"
"encoding/json"
"fmt"
"sync/atomic"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
licenseserver "go.signoz.io/signoz/ee/query-service/integrations/signozio"
"go.signoz.io/signoz/ee/query-service/license"
"go.signoz.io/signoz/ee/query-service/model"
"go.signoz.io/signoz/ee/query-service/usage/repository"
"go.signoz.io/signoz/pkg/query-service/utils/encryption"
)
const (
MaxRetries = 3
RetryInterval = 5 * time.Second
stateUnlocked uint32 = 0
stateLocked uint32 = 1
)
var (
// collect usage every hour
collectionFrequency = 1 * time.Hour
// send usage every 24 hour
uploadFrequency = 24 * time.Hour
locker = stateUnlocked
)
type Manager struct {
repository *repository.Repository
clickhouseConn clickhouse.Conn
licenseRepo *license.Repo
// end the usage routine, this is important to gracefully
// stopping usage reporting and protect in-consistent updates
done chan struct{}
// terminated waits for the UsageExporter go routine to end
terminated chan struct{}
}
func New(dbType string, db *sqlx.DB, licenseRepo *license.Repo, clickhouseConn clickhouse.Conn) (*Manager, error) {
repo := repository.New(db)
err := repo.Init(dbType)
if err != nil {
return nil, fmt.Errorf("failed to initiate usage repo: %v", err)
}
m := &Manager{
repository: repo,
clickhouseConn: clickhouseConn,
licenseRepo: licenseRepo,
}
return m, nil
}
// start loads collects and exports any exported snapshot and starts the exporter
func (lm *Manager) Start() error {
// compares the locker and stateUnlocked if both are same lock is applied else returns error
if !atomic.CompareAndSwapUint32(&locker, stateUnlocked, stateLocked) {
return fmt.Errorf("usage exporter is locked")
}
// check if license is present or not
license, err := lm.licenseRepo.GetActiveLicense(context.Background())
if err != nil {
return fmt.Errorf("failed to get active license")
}
if license == nil {
// we will not start the usage reporting if license is not present.
zap.S().Info("no license present, skipping usage reporting")
return nil
}
// upload previous snapshots if any
err = lm.UploadUsage(context.Background())
if err != nil {
return err
}
// collect snapshot if incase it wasn't collect in (t - collectionFrequency)
err = lm.CollectCurrentUsage(context.Background())
if err != nil {
return err
}
go lm.UsageExporter(context.Background())
return nil
}
// CollectCurrentUsage checks if needs to collect usage data
func (lm *Manager) CollectCurrentUsage(ctx context.Context) error {
// check the DB if anything exist where timestamp > t - collectionFrequency
ts := time.Now().Add(-collectionFrequency)
alreadyCreated, err := lm.repository.CheckSnapshotGtCreatedAt(ctx, ts)
if err != nil {
return err
}
if !alreadyCreated {
zap.S().Info("Collecting current usage")
exportError := lm.CollectAndStoreUsage(ctx)
if exportError != nil {
return exportError
}
} else {
zap.S().Info("Nothing to collect")
}
return nil
}
func (lm *Manager) UsageExporter(ctx context.Context) {
defer close(lm.terminated)
collectionTicker := time.NewTicker(collectionFrequency)
defer collectionTicker.Stop()
uploadTicker := time.NewTicker(uploadFrequency)
defer uploadTicker.Stop()
for {
select {
case <-lm.done:
return
case <-collectionTicker.C:
lm.CollectAndStoreUsage(ctx)
case <-uploadTicker.C:
lm.UploadUsage(ctx)
// remove the old snapshots
lm.repository.DropOldSnapshots(ctx)
}
}
}
type TableSize struct {
Table string `ch:"table"`
DiskName string `ch:"disk_name"`
Rows uint64 `ch:"rows"`
UncompressedBytes uint64 `ch:"uncompressed_bytes"`
}
func (lm *Manager) CollectAndStoreUsage(ctx context.Context) error {
snap, err := lm.GetUsageFromClickHouse(ctx)
if err != nil {
return err
}
license, err := lm.licenseRepo.GetActiveLicense(ctx)
if err != nil {
return err
}
activationId, _ := uuid.Parse(license.ActivationId)
// TODO (nitya) : Add installation ID in the payload
payload := model.UsagePayload{
UsageBase: model.UsageBase{
ActivationId: activationId,
FailedSyncRequest: 0,
},
Metrics: *snap,
SnapshotDate: time.Now(),
}
err = lm.repository.InsertSnapshot(ctx, &payload)
if err != nil {
return err
}
return nil
}
func (lm *Manager) GetUsageFromClickHouse(ctx context.Context) (*model.UsageSnapshot, error) {
tableSizes := []TableSize{}
snap := model.UsageSnapshot{}
// get usage from clickhouse
query := `
SELECT
table,
disk_name,
sum(rows) as rows,
sum(data_uncompressed_bytes) AS uncompressed_bytes
FROM system.parts
WHERE active AND (database in ('signoz_logs', 'signoz_metrics', 'signoz_traces')) AND (table in ('logs','samples_v2', 'signoz_index_v2'))
GROUP BY
table,
disk_name
ORDER BY table
`
err := lm.clickhouseConn.Select(ctx, &tableSizes, query)
if err != nil {
return nil, err
}
for _, val := range tableSizes {
switch val.Table {
case "logs":
if val.DiskName == "default" {
snap.CurrentLogSizeBytes = val.UncompressedBytes
} else {
snap.CurrentLogSizeBytesColdStorage = val.UncompressedBytes
}
case "samples_v2":
if val.DiskName == "default" {
snap.CurrentSamplesCount = val.Rows
} else {
snap.CurrentSamplesCountColdStorage = val.Rows
}
case "signoz_index_v2":
if val.DiskName == "default" {
snap.CurrentSpansCount = val.Rows
} else {
snap.CurrentSpansCountColdStorage = val.Rows
}
}
}
return &snap, nil
}
func (lm *Manager) UploadUsage(ctx context.Context) error {
snapshots, err := lm.repository.GetSnapshotsNotSynced(ctx)
if err != nil {
return err
}
if len(snapshots) <= 0 {
zap.S().Info("no snapshots to upload, skipping.")
return nil
}
zap.S().Info("uploading snapshots")
for _, snap := range snapshots {
metricsBytes, err := encryption.Decrypt([]byte(snap.ActivationId.String()[:32]), []byte(snap.Snapshot))
if err != nil {
return err
}
metrics := model.UsageSnapshot{}
err = json.Unmarshal(metricsBytes, &metrics)
if err != nil {
return err
}
err = lm.UploadUsageWithExponentalBackOff(ctx, model.UsagePayload{
UsageBase: model.UsageBase{
Id: snap.Id,
InstallationId: snap.InstallationId,
ActivationId: snap.ActivationId,
FailedSyncRequest: snap.FailedSyncRequest,
},
SnapshotDate: snap.CreatedAt,
Metrics: metrics,
})
if err != nil {
return err
}
}
return nil
}
func (lm *Manager) UploadUsageWithExponentalBackOff(ctx context.Context, payload model.UsagePayload) error {
for i := 1; i <= MaxRetries; i++ {
apiErr := licenseserver.SendUsage(ctx, &payload)
if apiErr != nil && i == MaxRetries {
err := lm.repository.IncrementFailedRequestCount(ctx, payload.Id)
if err != nil {
zap.S().Errorf("failed to updated the failure count for snapshot in DB : ", zap.Error(err))
return err
}
zap.S().Errorf("retries stopped : %v", zap.Error(err))
// not returning error here since it is captured in the failed count
return nil
} else if apiErr != nil {
// sleeping for exponential backoff
sleepDuration := RetryInterval * time.Duration(i)
zap.S().Errorf("failed to upload snapshot retrying after %v secs : %v", sleepDuration.Seconds(), zap.Error(apiErr.Err))
time.Sleep(sleepDuration)
// update the failed request count
err := lm.repository.IncrementFailedRequestCount(ctx, payload.Id)
if err != nil {
zap.S().Errorf("failed to updated the failure count for snapshot in DB : %v", zap.Error(err))
return err
}
} else {
break
}
}
// update the database that it is synced
err := lm.repository.MoveToSynced(ctx, payload.Id)
if err != nil {
return err
}
return nil
}
func (lm *Manager) Stop() {
close(lm.done)
atomic.StoreUint32(&locker, stateUnlocked)
<-lm.terminated
}
package repository
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
"go.signoz.io/signoz/ee/query-service/model"
"go.signoz.io/signoz/ee/query-service/usage/sqlite"
"go.signoz.io/signoz/pkg/query-service/utils/encryption"
)
const (
MaxFailedSyncCount = 9 // a snapshot will be ignored if the max failed count is greater than or equal to 9
SnapShotLife = 3 * 24 * time.Hour
)
// Repository is usage Repository which stores usage snapshot in a secured DB
type Repository struct {
db *sqlx.DB
}
// New initiates a new usage Repository
func New(db *sqlx.DB) *Repository {
return &Repository{
db: db,
}
}
func (r *Repository) Init(engine string) error {
switch engine {
case "sqlite3", "sqlite":
return sqlite.InitDB(r.db)
default:
return fmt.Errorf("unsupported db")
}
}
func (r *Repository) InsertSnapshot(ctx context.Context, usage *model.UsagePayload) error {
snapshotBytes, err := json.Marshal(usage.Metrics)
if err != nil {
return err
}
usage.Id = uuid.New()
encryptedSnapshot, err := encryption.Encrypt([]byte(usage.ActivationId.String()[:32]), snapshotBytes)
if err != nil {
return err
}
query := `INSERT INTO usage(id, activation_id, snapshot)
VALUES ($1, $2, $3)`
_, err = r.db.ExecContext(ctx,
query,
usage.Id,
usage.ActivationId,
string(encryptedSnapshot),
)
if err != nil {
zap.S().Errorf("error inserting usage data: %v", zap.Error(err))
return fmt.Errorf("failed to insert usage in db: %v", err)
}
return nil
}
func (r *Repository) MoveToSynced(ctx context.Context, id uuid.UUID) error {
query := `UPDATE usage
SET synced = 'true',
synced_at = $1
WHERE id = $2`
_, err := r.db.ExecContext(ctx, query, time.Now(), id)
if err != nil {
zap.S().Errorf("error in updating usage: %v", zap.Error(err))
return fmt.Errorf("failed to update usage in db: %v", err)
}
return nil
}
func (r *Repository) IncrementFailedRequestCount(ctx context.Context, id uuid.UUID) error {
query := `UPDATE usage SET failed_sync_request_count = failed_sync_request_count + 1 WHERE id = $1`
_, err := r.db.ExecContext(ctx, query, id)
if err != nil {
zap.S().Errorf("error in updating usage: %v", zap.Error(err))
return fmt.Errorf("failed to update usage in db: %v", err)
}
return nil
}
func (r *Repository) GetSnapshotsNotSynced(ctx context.Context) ([]*model.Usage, error) {
snapshots := []*model.Usage{}
query := `SELECT id,created_at, activation_id, snapshot, failed_sync_request_count from usage where synced!='true' and failed_sync_request_count < $1 order by created_at asc `
err := r.db.SelectContext(ctx, &snapshots, query, MaxFailedSyncCount)
if err != nil {
return nil, err
}
return snapshots, nil
}
func (r *Repository) DropOldSnapshots(ctx context.Context) error {
query := `delete from usage where created_at <= $1`
_, err := r.db.ExecContext(ctx, query, time.Now().Add(-(SnapShotLife)))
if err != nil {
zap.S().Errorf("failed to remove old snapshots from db: %v", zap.Error(err))
return err
}
return nil
}
// CheckSnapshotGtCreatedAt checks if there is any snapshot greater than the provided timestamp
func (r *Repository) CheckSnapshotGtCreatedAt(ctx context.Context, ts time.Time) (bool, error) {
var snapshots uint64
query := `SELECT count() from usage where created_at > '$1'`
err := r.db.QueryRowContext(ctx, query, ts).Scan(&snapshots)
if err != nil {
return false, err
}
return snapshots > 0, err
}
package sqlite
import (
"fmt"
"github.com/jmoiron/sqlx"
)
func InitDB(db *sqlx.DB) error {
var err error
if db == nil {
return fmt.Errorf("invalid db connection")
}
table_schema := `CREATE TABLE IF NOT EXISTS usage(
id UUID PRIMARY KEY,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
activation_id UUID,
snapshot TEXT,
synced BOOLEAN DEFAULT 'false',
synced_at TIMESTAMP,
failed_sync_request_count INTEGER DEFAULT 0
);
`
_, err = db.Exec(table_schema)
if err != nil {
return fmt.Errorf("error in creating usage table: %v", err.Error())
}
return nil
}
......@@ -13,7 +13,7 @@ server {
# to handle uri issue 414 from nginx
client_max_body_size 24M;
large_client_header_buffers 8 16k;
large_client_header_buffers 8 128k;
location / {
root /usr/share/nginx/html;
......
{
"column_license_key": "License Key",
"column_valid_from": "Valid From",
"column_valid_until": "Valid Until",
"column_license_status": "Status",
"button_apply": "Apply",
"placeholder_license_key": "Enter a License Key",
"tab_current_license": "Current License",
"tab_license_history": "History",
"loading_licenses": "Loading licenses...",
"enter_license_key": "Please enter a license key",
"license_applied": "License applied successfully, please refresh the page to see changes."
}
\ No newline at end of file
{
"label_email": "Email",
"placeholder_email": "name@yourcompany.com",
"label_password": "Password",
"button_initiate_login": "Next",
"button_login": "Login",
"login_page_title": "Login with SigNoz",
"login_with_sso": "Login with SSO",
"login_with_pwd": "Login with password",
"forgot_password": "Forgot password?",
"create_an_account": "Create an account",
"prompt_if_admin": "If you are admin,",
"prompt_create_account": "If you are setting up SigNoz for the first time,",
"prompt_no_account": "Don't have an account? Contact your admin to send you an invite link.",
"prompt_forgot_password": "Ask your admin to reset your password and send you a new invite link",
"prompt_on_sso_error": "Are you trying to resolve SSO configuration issue?",
"unexpected_error": "Sorry, something went wrong",
"failed_to_login": "sorry, failed to login",
"invalid_email": "Please enter a valid email address",
"invalid_account": "This account does not exist. To create a new account, contact your admin to get an invite link",
"invalid_config": "Invalid configuration detected, please contact your administrator"
}
\ No newline at end of file
......@@ -9,5 +9,10 @@
"add_another_team_member": "Add another team member",
"invite_team_members": "Invite team members",
"invite_members": "Invite Members",
"pending_invites": "Pending Invites"
"pending_invites": "Pending Invites",
"authenticated_domains": "Authenticated Domains",
"delete_domain_message": "Are you sure you want to delete this domain?",
"delete_domain": "Delete Domain",
"add_domain": "Add Domains",
"saml_settings":"Your SAML settings have been saved, please login from incognito window to confirm that it has been set up correctly"
}
{
"label_email": "Email",
"placeholder_email": "name@yourcompany.com",
"label_password": "Password",
"label_confirm_password": "Confirm Password",
"label_firstname": "First Name",
"placeholder_firstname": "Your Name",
"label_orgname": "Organization Name",
"placeholder_orgname": "Your Company",
"prompt_keepme_posted": "Keep me updated on new SigNoz features",
"prompt_anonymise": "Anonymise my usage date. We collect data to measure product usage",
"failed_confirm_password": "Passwords don’t match. Please try again",
"unexpected_error": "Something went wrong",
"failed_to_initiate_login": "Signup completed but failed to initiate login",
"token_required": "Invite token is required for signup, please request one from your admin",
"button_get_started": "Get Started",
"prompt_admin_warning": "This will create an admin account. If you are not an admin, please ask your admin for an invite link"
}
\ No newline at end of file
......@@ -6,7 +6,7 @@
"release_notes": "Release Notes",
"read_how_to_upgrade": "Read instructions on how to upgrade",
"latest_version_signoz": "You are running the latest version of SigNoz.",
"stale_version": "You are on an older version and may be loosing on the latest features we have shipped. We recommend to upgrade to the latest version",
"stale_version": "You are on an older version and may be losing out on the latest features we have shipped. We recommend to upgrade to the latest version",
"oops_something_went_wrong_version": "Oops.. facing issues with fetching updated version information",
"n_a": "N/A",
"routes": {
......
{
"column_license_key": "License Key",
"column_valid_from": "Valid From",
"column_valid_until": "Valid Until",
"column_license_status": "Status",
"button_apply": "Apply",
"placeholder_license_key": "Enter a License Key",
"tab_current_license": "Current License",
"tab_license_history": "History",
"loading_licenses": "Loading licenses...",
"enter_license_key": "Please enter a license key",
"license_applied": "License applied successfully, please refresh the page to see changes."
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment