diff --git a/cmd/enterprise/server.go b/cmd/enterprise/server.go index 2da0d564be..3dd09a970a 100644 --- a/cmd/enterprise/server.go +++ b/cmd/enterprise/server.go @@ -177,9 +177,11 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e return nil, err } azureCloudProviderModule := implcloudprovider.NewAzureCloudProvider(defStore) + gcpCloudProviderModule := implcloudprovider.NewGCPCloudProvider(defStore) cloudProvidersMap := map[cloudintegrationtypes.CloudProviderType]cloudintegration.CloudProviderModule{ cloudintegrationtypes.CloudProviderTypeAWS: awsCloudProviderModule, cloudintegrationtypes.CloudProviderTypeAzure: azureCloudProviderModule, + cloudintegrationtypes.CloudProviderTypeGCP: gcpCloudProviderModule, } return implcloudintegration.NewModule(pkgcloudintegration.NewStore(sqlStore), dashboardModule, global, zeus, gateway, licensing, serviceAccount, cloudProvidersMap, config) diff --git a/conf/example.yaml b/conf/example.yaml index e0bce3567c..e70696a3b9 100644 --- a/conf/example.yaml +++ b/conf/example.yaml @@ -65,15 +65,31 @@ web: posthog: # Whether to enable PostHog in web. enabled: false + # The PostHog project API key. + key: "" + # The PostHog API host. Defaults to https://us.i.posthog.com when empty. + api_host: "" + # The PostHog UI host. Used when api_host points at a reverse proxy. + ui_host: "" appcues: # Whether to enable Appcues in web. enabled: false + # The Appcues account/app ID. + app_id: "" sentry: # Whether to enable Sentry in web. enabled: false + # The Sentry DSN. + dsn: "" + # The Sentry tunnel URL. + tunnel: "" pylon: # Whether to enable Pylon in web. enabled: false + # The Pylon app ID. + app_id: "" + # The Pylon identity verification secret. + identity_secret: "" ##################### Cache ##################### cache: diff --git a/docs/api/openapi.yml b/docs/api/openapi.yml index 9ef8b4c887..7edc4a4b90 100644 --- a/docs/api/openapi.yml +++ b/docs/api/openapi.yml @@ -1024,6 +1024,8 @@ components: $ref: '#/components/schemas/CloudintegrationtypesAWSAccountConfig' azure: $ref: '#/components/schemas/CloudintegrationtypesAzureAccountConfig' + gcp: + $ref: '#/components/schemas/CloudintegrationtypesGCPAccountConfig' type: object CloudintegrationtypesAgentReport: nullable: true @@ -1169,6 +1171,8 @@ components: $ref: '#/components/schemas/CloudintegrationtypesAWSConnectionArtifact' azure: $ref: '#/components/schemas/CloudintegrationtypesAzureConnectionArtifact' + gcp: + $ref: '#/components/schemas/CloudintegrationtypesGCPConnectionArtifact' type: object CloudintegrationtypesCredentials: properties: @@ -1199,6 +1203,46 @@ components: nullable: true type: array type: object + CloudintegrationtypesGCPAccountConfig: + properties: + deploymentProjectId: + type: string + deploymentRegion: + type: string + projectIds: + items: + type: string + type: array + required: + - deploymentProjectId + - deploymentRegion + - projectIds + type: object + CloudintegrationtypesGCPConnectionArtifact: + type: object + CloudintegrationtypesGCPIntegrationConfig: + type: object + CloudintegrationtypesGCPServiceConfig: + properties: + logs: + $ref: '#/components/schemas/CloudintegrationtypesGCPServiceLogsConfig' + metrics: + $ref: '#/components/schemas/CloudintegrationtypesGCPServiceMetricsConfig' + type: object + CloudintegrationtypesGCPServiceLogsConfig: + properties: + enabled: + type: boolean + required: + - enabled + type: object + CloudintegrationtypesGCPServiceMetricsConfig: + properties: + enabled: + type: boolean + required: + - enabled + type: object CloudintegrationtypesGettableAccountWithConnectionArtifact: properties: connectionArtifact: @@ -1331,6 +1375,8 @@ components: $ref: '#/components/schemas/CloudintegrationtypesAWSPostableAccountConfig' azure: $ref: '#/components/schemas/CloudintegrationtypesAzureAccountConfig' + gcp: + $ref: '#/components/schemas/CloudintegrationtypesGCPAccountConfig' type: object CloudintegrationtypesPostableAgentCheckIn: properties: @@ -1355,6 +1401,8 @@ components: $ref: '#/components/schemas/CloudintegrationtypesAWSIntegrationConfig' azure: $ref: '#/components/schemas/CloudintegrationtypesAzureIntegrationConfig' + gcp: + $ref: '#/components/schemas/CloudintegrationtypesGCPIntegrationConfig' type: object CloudintegrationtypesService: properties: @@ -1399,6 +1447,8 @@ components: $ref: '#/components/schemas/CloudintegrationtypesAWSServiceConfig' azure: $ref: '#/components/schemas/CloudintegrationtypesAzureServiceConfig' + gcp: + $ref: '#/components/schemas/CloudintegrationtypesGCPServiceConfig' type: object CloudintegrationtypesServiceDashboard: properties: @@ -1441,6 +1491,7 @@ components: - cosmosdb - cassandradb - redis + - cloudsql type: string CloudintegrationtypesServiceMetadata: properties: @@ -1502,6 +1553,8 @@ components: $ref: '#/components/schemas/CloudintegrationtypesAWSAccountConfig' azure: $ref: '#/components/schemas/CloudintegrationtypesUpdatableAzureAccountConfig' + gcp: + $ref: '#/components/schemas/CloudintegrationtypesUpdatableGCPAccountConfig' type: object CloudintegrationtypesUpdatableAzureAccountConfig: properties: @@ -1512,6 +1565,22 @@ components: required: - resourceGroups type: object + CloudintegrationtypesUpdatableGCPAccountConfig: + properties: + deploymentProjectId: + type: string + deploymentRegion: + type: string + projectIds: + items: + type: string + nullable: true + type: array + required: + - deploymentProjectId + - deploymentRegion + - projectIds + type: object CloudintegrationtypesUpdatableService: properties: config: @@ -2668,6 +2737,14 @@ components: type: string dashboardName: type: string + filterBy: + items: + type: string + type: array + groupBy: + items: + type: string + type: array panelId: type: string panelName: @@ -5454,6 +5531,9 @@ components: type: string id: type: string + ingestedSamples: + minimum: 0 + type: integer ingestedSeries: minimum: 0 type: integer @@ -5466,9 +5546,9 @@ components: $ref: '#/components/schemas/MetricreductionruletypesMatchType' metricName: type: string - reductionPercent: - format: double - type: number + retainedSamples: + minimum: 0 + type: integer retainedSeries: minimum: 0 type: integer @@ -5486,7 +5566,8 @@ components: - active - ingestedSeries - retainedSeries - - reductionPercent + - ingestedSamples + - retainedSamples type: object MetricreductionruletypesGettableReductionRulePreview: properties: @@ -5529,15 +5610,23 @@ components: estimatedMonthlySavingsUsd: format: double type: number + ingestedSamples: + minimum: 0 + type: integer ingestedSeries: minimum: 0 type: integer + retainedSamples: + minimum: 0 + type: integer retainedSeries: minimum: 0 type: integer required: - ingestedSeries - retainedSeries + - ingestedSamples + - retainedSamples - estimatedMonthlySavingsUsd type: object MetricreductionruletypesGettableReductionRules: @@ -5603,7 +5692,6 @@ components: - metric - ingested_volume - reduced_volume - - reduction - last_updated type: string MetricreductionruletypesUpdatableReductionRule: @@ -6323,6 +6411,25 @@ components: - asc - desc type: string + Querybuildertypesv5PreviewStatement: + properties: + db.statement.args: + items: {} + type: array + db.statement.query: + type: string + estimate: + items: + $ref: '#/components/schemas/TelemetrystoretypesEstimateEntry' + type: array + granules: + $ref: '#/components/schemas/TelemetrystoretypesGranules' + required: + - db.statement.query + - db.statement.args + - estimate + - granules + type: object Querybuildertypesv5PromQuery: properties: disabled: @@ -6643,6 +6750,40 @@ components: required: - type type: object + Querybuildertypesv5QueryPreview: + properties: + error: {} + statements: + items: + $ref: '#/components/schemas/Querybuildertypesv5PreviewStatement' + type: array + valid: + type: boolean + warnings: + items: + type: string + type: array + required: + - valid + - error + - warnings + - statements + type: object + Querybuildertypesv5QueryRangePreviewResponse: + description: Response from the v5 query range preview (dry-run) endpoint. For + each query in the composite query, returns the underlying ClickHouse statement(s) + it renders to without executing them (one per PromQL metric selector; exactly + one for builder/ClickHouse/trace-operator queries), with the optional EXPLAIN + ESTIMATE and granule analysis attached per statement when requested. + properties: + compositeQuery: + additionalProperties: + $ref: '#/components/schemas/Querybuildertypesv5QueryPreview' + nullable: true + type: object + required: + - compositeQuery + type: object Querybuildertypesv5QueryRangeRequest: description: Request body for the v5 query range endpoint. Supports builder queries (traces, logs, metrics), formulas, joins, trace operators, PromQL, @@ -7993,6 +8134,96 @@ components: - key - value type: object + TelemetrystoretypesEstimateEntry: + properties: + database: + type: string + marks: + format: int64 + type: integer + parts: + format: int64 + type: integer + rows: + format: int64 + type: integer + table: + type: string + required: + - database + - table + - parts + - rows + - marks + type: object + TelemetrystoretypesGranules: + nullable: true + properties: + initial: + format: int64 + type: integer + reads: + items: + $ref: '#/components/schemas/TelemetrystoretypesMergeTreeRead' + type: array + selected: + format: int64 + type: integer + skipped: + format: int64 + type: integer + required: + - initial + - selected + - skipped + - reads + type: object + TelemetrystoretypesIndexStep: + properties: + condition: + type: string + initialGranules: + format: int64 + type: integer + initialParts: + format: int64 + type: integer + keys: + items: + type: string + type: array + name: + type: string + selectedGranules: + format: int64 + type: integer + selectedParts: + format: int64 + type: integer + type: + type: string + required: + - type + - name + - keys + - condition + - initialParts + - selectedParts + - initialGranules + - selectedGranules + type: object + TelemetrystoretypesMergeTreeRead: + properties: + steps: + items: + $ref: '#/components/schemas/TelemetrystoretypesIndexStep' + type: array + table: + type: string + required: + - table + - steps + type: object TelemetrytypesFieldContext: enum: - metric @@ -23524,6 +23755,75 @@ paths: summary: Query range tags: - querier + /api/v5/query_range/preview: + post: + deprecated: false + description: 'Validate a composite query without executing it. Accepts the same + payload as the query range endpoint. By default (verbose=true) returns, for + each query, the rendered underlying ClickHouse statement(s) with each statement''s + EXPLAIN ESTIMATE (per-table parts/rows/marks) and granule index analysis (candidate/surviving + granules and the per-index pruning funnel). Pass ?verbose=false for the lightweight + per-query verdict (valid/error/warnings) with no rendered SQL and no ClickHouse + round trips. Intended for agentic/dry-run consumption: per-query errors are + reported in the response rather than failing the whole request.' + operationId: QueryRangePreviewV5 + parameters: + - in: query + name: verbose + schema: + type: string + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Querybuildertypesv5QueryRangeRequest' + responses: + "200": + content: + application/json: + schema: + properties: + data: + $ref: '#/components/schemas/Querybuildertypesv5QueryRangePreviewResponse' + status: + type: string + required: + - status + - data + type: object + description: OK + "400": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Bad Request + "401": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Unauthorized + "403": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Forbidden + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/RenderErrorResponse' + description: Internal Server Error + security: + - api_key: + - VIEWER + - tokenizer: + - VIEWER + summary: Query range preview + tags: + - querier /api/v5/substitute_vars: post: deprecated: false diff --git a/ee/modules/cloudintegration/implcloudintegration/implcloudprovider/gcpcloudprovider.go b/ee/modules/cloudintegration/implcloudintegration/implcloudprovider/gcpcloudprovider.go new file mode 100644 index 0000000000..41232099cc --- /dev/null +++ b/ee/modules/cloudintegration/implcloudintegration/implcloudprovider/gcpcloudprovider.go @@ -0,0 +1,36 @@ +package implcloudprovider + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/modules/cloudintegration" + "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes" +) + +type gcpcloudprovider struct { + serviceDefinitions cloudintegrationtypes.ServiceDefinitionStore +} + +func NewGCPCloudProvider(defStore cloudintegrationtypes.ServiceDefinitionStore) cloudintegration.CloudProviderModule { + return &gcpcloudprovider{ + serviceDefinitions: defStore, + } +} + +func (g *gcpcloudprovider) BuildIntegrationConfig(ctx context.Context, account *cloudintegrationtypes.Account, services []*cloudintegrationtypes.StorableCloudIntegrationService) (*cloudintegrationtypes.ProviderIntegrationConfig, error) { + // for manual flow we don't have any integration config to return, so returning empty config for now. + return &cloudintegrationtypes.ProviderIntegrationConfig{}, nil +} + +func (g *gcpcloudprovider) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.GetConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) { + // for manual flow we don't have any connection artifact to return, so returning empty artifact for now. + return &cloudintegrationtypes.ConnectionArtifact{}, nil +} + +func (g *gcpcloudprovider) GetServiceDefinition(ctx context.Context, serviceID cloudintegrationtypes.ServiceID) (*cloudintegrationtypes.ServiceDefinition, error) { + return g.serviceDefinitions.Get(ctx, cloudintegrationtypes.CloudProviderTypeGCP, serviceID) +} + +func (g *gcpcloudprovider) ListServiceDefinitions(ctx context.Context) ([]*cloudintegrationtypes.ServiceDefinition, error) { + return g.serviceDefinitions.List(ctx, cloudintegrationtypes.CloudProviderTypeGCP) +} diff --git a/ee/modules/dashboard/impldashboard/module.go b/ee/modules/dashboard/impldashboard/module.go index 8b2c8e8dea..024196884f 100644 --- a/ee/modules/dashboard/impldashboard/module.go +++ b/ee/modules/dashboard/impldashboard/module.go @@ -286,7 +286,7 @@ func (module *module) Get(ctx context.Context, orgID valuer.UUID, id valuer.UUID return module.pkgDashboardModule.Get(ctx, orgID, id) } -func (module *module) GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]map[string]string, error) { +func (module *module) GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]dashboardtypes.DashboardPanelRef, error) { return module.pkgDashboardModule.GetByMetricNames(ctx, orgID, metricNames) } diff --git a/ee/modules/metricreductionrule/implmetricreductionrule/clickhouse.go b/ee/modules/metricreductionrule/implmetricreductionrule/clickhouse.go index 1c47b1a5de..2529905085 100644 --- a/ee/modules/metricreductionrule/implmetricreductionrule/clickhouse.go +++ b/ee/modules/metricreductionrule/implmetricreductionrule/clickhouse.go @@ -22,6 +22,8 @@ var ( const timeSeriesBucketMilli = int64(time.Hour / time.Millisecond) +const sampleBucketExpr = "toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalMinute(10)))) * 1000 AS bucket" + type volumeRow struct { MetricName string Ingested uint64 @@ -289,12 +291,9 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff } ctx = c.withThreads(ctx) - orderExpr := "ingested" - switch orderBy { - case metricreductionruletypes.OrderByReducedVolume: - orderExpr = "reduced" - case metricreductionruletypes.OrderByReduction: - orderExpr = "if(ingested = 0, 0, (toFloat64(ingested) - toFloat64(reduced)) / toFloat64(ingested))" + orderExpr := "ifNull(i.samples, 0)" + if orderBy == metricreductionruletypes.OrderByReducedVolume { + orderExpr = "if(ifNull(d.samples, 0) = 0 OR ifNull(d.samples, 0) > ifNull(i.samples, 0), ifNull(i.samples, 0), ifNull(d.samples, 0))" } direction := "ASC" if order == metricreductionruletypes.OrderDesc { @@ -310,17 +309,17 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff sb.From("(SELECT arrayJoin(" + sb.Var(metricNames) + ") AS metric_name) AS base") sb.JoinWithOption( sqlbuilder.LeftJoin, - "(SELECT metric_name, uniq(fingerprint) AS cnt FROM "+ingestedTable+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name) AS i", + "(SELECT metric_name, uniq(fingerprint) AS cnt, count() AS samples FROM "+ingestedTable+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name) AS i", "base.metric_name = i.metric_name", ) // Reduced series are spread across two type-specific tables; union the per-table distinct // reduced_fingerprints and sum per metric (a metric only lands in the table matching its type). sb.JoinWithOption( sqlbuilder.LeftJoin, - "(SELECT metric_name, sum(cnt) AS cnt FROM ("+ - "SELECT metric_name, uniq(reduced_fingerprint) AS cnt FROM "+reducedLast+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+ + "(SELECT metric_name, sum(cnt) AS cnt, sum(samples) AS samples FROM ("+ + "SELECT metric_name, uniq(reduced_fingerprint) AS cnt, uniq(reduced_fingerprint, unix_milli) AS samples FROM "+reducedLast+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+ " UNION ALL "+ - "SELECT metric_name, uniq(reduced_fingerprint) AS cnt FROM "+reducedSum+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+ + "SELECT metric_name, uniq(reduced_fingerprint) AS cnt, uniq(reduced_fingerprint, unix_milli) AS samples FROM "+reducedSum+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+ ") GROUP BY metric_name) AS d", "base.metric_name = d.metric_name", ) @@ -347,122 +346,186 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff return out, rows.Err() } -func (c *clickhouse) SampleVolume(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (uint64, uint64, error) { +func (c *clickhouse) SampleVolumeByMetric(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]volumeRow, error) { if len(metricNames) == 0 { - return 0, 0, nil + return map[string]volumeRow{}, nil } ctx = c.withThreads(ctx) - ingested, err := c.countRawSamples(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, metricNames, effectiveFrom, startMs, endMs) - if err != nil { - return 0, 0, err - } - - last, err := c.countReducedSamples(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, metricNames, effectiveFrom, startMs, endMs) - if err != nil { - return 0, 0, err - } - sum, err := c.countReducedSamples(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, metricNames, effectiveFrom, startMs, endMs) - if err != nil { - return 0, 0, err - } - - return ingested, min(last+sum, ingested), nil -} - -func (c *clickhouse) countRawSamples(ctx context.Context, table string, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (uint64, error) { - names := make([]any, len(metricNames)) - for i, name := range metricNames { - names[i] = name - } - - sb := sqlbuilder.NewSelectBuilder() - sb.Select("count()") - sb.From(table) - conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)} - if len(effectiveFrom) > 0 { - conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) - } - sb.Where(conds...) - - query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) - var count uint64 - if err := c.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...).Scan(&count); err != nil { - return 0, errors.WrapInternalf(err, errors.CodeInternal, "failed to count ingested samples") - } - return count, nil -} - -func (c *clickhouse) countReducedSamples(ctx context.Context, table string, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (uint64, error) { - names := make([]any, len(metricNames)) - for i, name := range metricNames { - names[i] = name - } - - sb := sqlbuilder.NewSelectBuilder() - // Reduced tables key the series on reduced_fingerprint (not fingerprint); dedupe ReplacingMergeTree recomputes. - sb.Select("uniq(reduced_fingerprint, unix_milli)") - sb.From(table) - conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)} - if len(effectiveFrom) > 0 { - conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) - } - sb.Where(conds...) - - query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) - var count uint64 - if err := c.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...).Scan(&count); err != nil { - return 0, errors.WrapInternalf(err, errors.CodeInternal, "failed to count reduced samples") - } - return count, nil -} - -// SeriesTimeseries returns ingested vs reduced series per 60s bucket from the samples tables, gated -// to each metric's strict effective_from (see strictEffectiveFrom). -func (c *clickhouse) SeriesTimeseries(ctx context.Context, allMetrics, reducedMetrics []string, effectiveFrom map[string]int64, startMs, endMs int64) ([]volumePoint, error) { - if len(allMetrics) == 0 { - return []volumePoint{}, nil - } - ctx = c.withThreads(ctx) - - ingested, err := c.ingestedSeriesByBucket(ctx, allMetrics, effectiveFrom, startMs, endMs) + ingested, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs) if err != nil { return nil, err } - retained := make(map[int64]uint64) - if len(reducedMetrics) > 0 { - reduced, err := c.reducedSeriesByBucket(ctx, reducedMetrics, effectiveFrom, startMs, endMs) + last, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs) + if err != nil { + return nil, err + } + sum, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs) + if err != nil { + return nil, err + } + + out := make(map[string]volumeRow, len(metricNames)) + for _, name := range metricNames { + out[name] = volumeRow{MetricName: name, Ingested: ingested[name], Reduced: last[name] + sum[name]} + } + return out, nil +} + +func (c *clickhouse) countSamplesByMetric(ctx context.Context, table, countExpr string, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]uint64, error) { + names := make([]any, len(metricNames)) + for i, name := range metricNames { + names[i] = name + } + + sb := sqlbuilder.NewSelectBuilder() + sb.Select("metric_name", countExpr) + sb.From(table) + conds := []string{ + sb.In("metric_name", names...), + sb.GE("unix_milli", startMs), + sb.LT("unix_milli", endMs), + } + if len(effectiveFrom) > 0 { + conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) + } + sb.Where(conds...) + sb.GroupBy("metric_name") + + query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...) + if err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to count samples") + } + defer rows.Close() + + out := make(map[string]uint64, len(metricNames)) + for rows.Next() { + var ( + metricName string + count uint64 + ) + if err := rows.Scan(&metricName, &count); err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan series count") + } + out[metricName] = count + } + return out, rows.Err() +} + +func (c *clickhouse) TotalVolume(ctx context.Context, startMs, endMs int64) (uint64, uint64, error) { + ctx = c.withThreads(ctx) + + sb := sqlbuilder.NewSelectBuilder() + sb.Select("uniq(fingerprint)", "count()") + sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName) + sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)) + + query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) + var series, samples uint64 + if err := c.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...).Scan(&series, &samples); err != nil { + return 0, 0, errors.WrapInternalf(err, errors.CodeInternal, "failed to count total ingested volume") + } + return series, samples, nil +} + +func (c *clickhouse) SampleTimeseries(ctx context.Context, ruledMetrics []string, effectiveFrom map[string]int64, startMs, endMs int64) ([]volumePoint, error) { + ctx = c.withThreads(ctx) + + ingested, err := c.totalSamplesByBucket(ctx, startMs, endMs) + if err != nil { + return nil, err + } + + ruledIngested := make(map[int64]uint64) + ruledRetained := make(map[int64]uint64) + if len(ruledMetrics) > 0 { + ruledIngested, err = c.ruledIngestedSamplesByBucket(ctx, ruledMetrics, effectiveFrom, startMs, endMs) if err != nil { return nil, err } - for ts, count := range reduced { - retained[ts] += count - } - } - reducedSet := make(map[string]struct{}, len(reducedMetrics)) - for _, name := range reducedMetrics { - reducedSet[name] = struct{}{} - } - nonReduced := make([]string, 0, len(allMetrics)) - for _, name := range allMetrics { - if _, ok := reducedSet[name]; !ok { - nonReduced = append(nonReduced, name) - } - } - if len(nonReduced) > 0 { - nonReducedIngested, err := c.ingestedSeriesByBucket(ctx, nonReduced, effectiveFrom, startMs, endMs) + ruledRetained, err = c.ruledRetainedSamplesByBucket(ctx, ruledMetrics, effectiveFrom, startMs, endMs) if err != nil { return nil, err } - for ts, count := range nonReducedIngested { - retained[ts] += count + } + + retained := make(map[int64]uint64, len(ingested)) + for ts, total := range ingested { + shed := uint64(0) + if ri := ruledIngested[ts]; ri > ruledRetained[ts] { + shed = ri - ruledRetained[ts] + } + if total > shed { + retained[ts] = total - shed + } else { + retained[ts] = 0 } } return mergeVolumePoints(ingested, retained), nil } +func (c *clickhouse) totalSamplesByBucket(ctx context.Context, startMs, endMs int64) (map[int64]uint64, error) { + sb := sqlbuilder.NewSelectBuilder() + sb.Select(sampleBucketExpr, "count()") + sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName) + sb.Where(sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)) + sb.GroupBy("bucket") + + return c.scanBuckets(ctx, sb) +} + +func (c *clickhouse) ruledIngestedSamplesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) { + names := make([]any, len(metricNames)) + for i, name := range metricNames { + names[i] = name + } + + sb := sqlbuilder.NewSelectBuilder() + sb.Select(sampleBucketExpr, "count()") + sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName) + conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)} + if len(effectiveFrom) > 0 { + conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) + } + sb.Where(conds...) + sb.GroupBy("bucket") + + return c.scanBuckets(ctx, sb) +} + +// reduced 60s rows are versioned by computed_at, so count distinct buckets. +func (c *clickhouse) ruledRetainedSamplesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) { + out := make(map[int64]uint64) + for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} { + names := make([]any, len(metricNames)) + for i, name := range metricNames { + names[i] = name + } + + sb := sqlbuilder.NewSelectBuilder() + sb.Select(sampleBucketExpr, "uniq(reduced_fingerprint, unix_milli)") + sb.From(telemetrymetrics.DBName + "." + table) + conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)} + if len(effectiveFrom) > 0 { + conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) + } + sb.Where(conds...) + sb.GroupBy("bucket") + + counts, err := c.scanBuckets(ctx, sb) + if err != nil { + return nil, err + } + for ts, count := range counts { + out[ts] += count + } + } + return out, nil +} + func mergeVolumePoints(ingested, reduced map[int64]uint64) []volumePoint { buckets := make(map[int64]struct{}, len(ingested)) for ts := range ingested { @@ -488,60 +551,6 @@ func mergeVolumePoints(ingested, reduced map[int64]uint64) []volumePoint { return points } -// ingestedSeriesByBucket counts distinct raw fingerprints per hourly bucket from the samples buffer. -func (c *clickhouse) ingestedSeriesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) { - names := make([]any, len(metricNames)) - for i, name := range metricNames { - names[i] = name - } - - sb := sqlbuilder.NewSelectBuilder() - bucketExpr := "toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalHour(1)))) * 1000 AS bucket" - sb.Select(bucketExpr, "uniq(fingerprint)") - sb.From(telemetrymetrics.DBName + "." + telemetrymetrics.SamplesV4BufferTableName) - conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)} - if len(effectiveFrom) > 0 { - conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) - } - sb.Where(conds...) - sb.GroupBy("bucket") - - return c.scanBuckets(ctx, sb) -} - -// reducedSeriesByBucket counts distinct reduced_fingerprints per hourly bucket, summed across the two -// reduced sample tables (a metric only lands in the table matching its type, so per-bucket sums are -// exact). -func (c *clickhouse) reducedSeriesByBucket(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[int64]uint64, error) { - out := make(map[int64]uint64) - for _, table := range []string{telemetrymetrics.SamplesV4ReducedLastTableName, telemetrymetrics.SamplesV4ReducedSumTableName} { - names := make([]any, len(metricNames)) - for i, name := range metricNames { - names[i] = name - } - - sb := sqlbuilder.NewSelectBuilder() - bucketExpr := "toInt64(toUnixTimestamp(toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalHour(1)))) * 1000 AS bucket" - sb.Select(bucketExpr, "uniq(reduced_fingerprint)") - sb.From(telemetrymetrics.DBName + "." + table) - conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)} - if len(effectiveFrom) > 0 { - conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom)) - } - sb.Where(conds...) - sb.GroupBy("bucket") - - counts, err := c.scanBuckets(ctx, sb) - if err != nil { - return nil, err - } - for ts, count := range counts { - out[ts] += count - } - } - return out, nil -} - func (c *clickhouse) scanBuckets(ctx context.Context, sb *sqlbuilder.SelectBuilder) (map[int64]uint64, error) { query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...) diff --git a/ee/modules/metricreductionrule/implmetricreductionrule/module.go b/ee/modules/metricreductionrule/implmetricreductionrule/module.go index 94c953d683..e9656ce41a 100644 --- a/ee/modules/metricreductionrule/implmetricreductionrule/module.go +++ b/ee/modules/metricreductionrule/implmetricreductionrule/module.go @@ -4,7 +4,6 @@ import ( "context" "log/slog" "sort" - "strings" "time" "github.com/SigNoz/signoz/pkg/errors" @@ -28,9 +27,16 @@ import ( const ( // effectiveFromMargin delays effective_from so the collector picks up the synced rule before it - // goes live; it must be >= the collector's rule-refresh interval (see signoz-otel-collector#839). - effectiveFromMargin = 5 * time.Minute - defaultPreviewLookback = 24 * time.Hour + // goes live; it must be >= the collector's rule-refresh interval (~2m worst case, + // see signoz-otel-collector#839). + effectiveFromMargin = 2 * time.Minute + // uiActivationDelay keeps a rule shown as "pending" in the UI for a while after it goes live to + // the collector, so the user doesn't see "active" before reduced data is actually flowing. The + // user-facing pending window is effectiveFromMargin + uiActivationDelay (~5m). + uiActivationDelay = 3 * time.Minute + defaultPreviewLookback = 1 * time.Hour + statsLookback = 1 * time.Hour + timeseriesLookback = 6 * time.Hour pricePerMillionSamplesUSD = 0.1 monthDuration = 30 * 24 * time.Hour @@ -80,7 +86,7 @@ func (m *module) List(ctx context.Context, orgID valuer.UUID, params *metricredu } now := time.Now() - startMs := now.Add(-defaultPreviewLookback).UnixMilli() + startMs := now.Add(-statsLookback).UnixMilli() endMs := now.UnixMilli() switch params.OrderBy { @@ -107,10 +113,14 @@ func (m *module) listSortedByColumn(ctx context.Context, orgID valuer.UUID, para if err != nil { return nil, err } + sampleVolumes, err := m.ch.SampleVolumeByMetric(ctx, metricNames, effectiveFrom, startMs, endMs) + if err != nil { + return nil, err + } rules := make([]metricreductionruletypes.GettableReductionRule, 0, len(domainRules)) for _, rule := range domainRules { - rules = append(rules, withVolume(toGettableReductionRule(rule), volumes[rule.MetricName])) + rules = append(rules, withVolume(toGettableReductionRule(rule), volumes[rule.MetricName], sampleVolumes[rule.MetricName])) } return &metricreductionruletypes.GettableReductionRules{Rules: rules, Total: total}, nil @@ -139,13 +149,24 @@ func (m *module) listSortedByVolume(ctx context.Context, orgID valuer.UUID, para return nil, err } + pageMetricNames := make([]string, 0, len(ranked)) + for _, row := range ranked { + pageMetricNames = append(pageMetricNames, row.MetricName) + } + + // TODO(srikanthccv): do we need to run this query? can we just get the same from RankByVolume? + sampleVolumes, err := m.ch.SampleVolumeByMetric(ctx, pageMetricNames, effectiveFrom, startMs, endMs) + if err != nil { + return nil, err + } + rules := make([]metricreductionruletypes.GettableReductionRule, 0, len(ranked)) for _, row := range ranked { rule, ok := ruleByMetric[row.MetricName] if !ok { continue } - rules = append(rules, withVolume(toGettableReductionRule(rule), row)) + rules = append(rules, withVolume(toGettableReductionRule(rule), row, sampleVolumes[row.MetricName])) } return &metricreductionruletypes.GettableReductionRules{Rules: rules, Total: total}, nil @@ -288,20 +309,17 @@ func (m *module) Stats(ctx context.Context, orgID valuer.UUID) (*metricreduction } now := time.Now() - startMs := now.Add(-defaultPreviewLookback).UnixMilli() + startMs := now.Add(-statsLookback).UnixMilli() endMs := now.UnixMilli() - allRules, total, err := m.store.List(ctx, orgID, &metricreductionruletypes.ListReductionRulesParams{}) + rules, _, err := m.store.List(ctx, orgID, &metricreductionruletypes.ListReductionRulesParams{}) if err != nil { return nil, err } - if total == 0 { - return &metricreductionruletypes.GettableReductionRuleStats{}, nil - } - metricNames := make([]string, len(allRules)) - effectiveFrom := make(map[string]int64, len(allRules)) - for i, rule := range allRules { + metricNames := make([]string, len(rules)) + effectiveFrom := make(map[string]int64, len(rules)) + for i, rule := range rules { metricNames[i] = rule.MetricName effectiveFrom[rule.MetricName] = rule.EffectiveFrom.UnixMilli() } @@ -310,31 +328,43 @@ func (m *module) Stats(ctx context.Context, orgID valuer.UUID) (*metricreduction if err != nil { return nil, err } - var ingestedSeries, retainedSeries uint64 - reducedMetricNames := make([]string, 0, len(volumes)) - reducedEffectiveFrom := make(map[string]int64, len(volumes)) - for name, volume := range volumes { - ingestedSeries += volume.Ingested - retained := effectiveRetained(volume.Ingested, volume.Reduced) - retainedSeries += retained - if retained < volume.Ingested { - reducedMetricNames = append(reducedMetricNames, name) - reducedEffectiveFrom[name] = effectiveFrom[name] - } + var ruledIngestedSeries, ruledRetainedSeries uint64 + for _, volume := range volumes { + ruledIngestedSeries += volume.Ingested + ruledRetainedSeries += effectiveRetained(volume.Ingested, volume.Reduced) } - ingestedSamples, reducedSamples, err := m.ch.SampleVolume(ctx, reducedMetricNames, reducedEffectiveFrom, startMs, endMs) + sampleVolumes, err := m.ch.SampleVolumeByMetric(ctx, metricNames, effectiveFrom, startMs, endMs) + if err != nil { + return nil, err + } + var ruledIngestedSamples, ruledRetainedSamples uint64 + for _, sv := range sampleVolumes { + ruledIngestedSamples += sv.Ingested + ruledRetainedSamples += effectiveRetained(sv.Ingested, sv.Reduced) + } + + totalSeries, totalSamples, err := m.ch.TotalVolume(ctx, startMs, endMs) if err != nil { return nil, err } return &metricreductionruletypes.GettableReductionRuleStats{ - IngestedSeries: ingestedSeries, - RetainedSeries: retainedSeries, - EstimatedMonthlySavingsUsd: monthlySavingsUSD(ingestedSamples, reducedSamples, startMs, endMs), + IngestedSeries: totalSeries, + RetainedSeries: clampSub(totalSeries, ruledIngestedSeries-ruledRetainedSeries), + IngestedSamples: totalSamples, + RetainedSamples: clampSub(totalSamples, ruledIngestedSamples-ruledRetainedSamples), + EstimatedMonthlySavingsUsd: monthlySavingsUSD(ruledIngestedSamples, ruledRetainedSamples, startMs, endMs), }, nil } +func clampSub(a, b uint64) uint64 { + if a < b { + return 0 + } + return a - b +} + // monthlySavingsUSD extrapolates the windowed sample reduction to a monthly figure at the per-sample // list price. Ingested is gated to effective_from upstream, so pre-activation hours don't inflate it. func monthlySavingsUSD(ingestedSamples, reducedSamples uint64, startMs, endMs int64) float64 { @@ -352,7 +382,7 @@ func (m *module) Timeseries(ctx context.Context, orgID valuer.UUID) (*querybuild } now := time.Now() - startMs := now.Add(-defaultPreviewLookback).UnixMilli() + startMs := now.Add(-timeseriesLookback).UnixMilli() endMs := now.UnixMilli() allRules, _, err := m.store.List(ctx, orgID, &metricreductionruletypes.ListReductionRulesParams{}) @@ -366,18 +396,7 @@ func (m *module) Timeseries(ctx context.Context, orgID valuer.UUID) (*querybuild effectiveFrom[rule.MetricName] = rule.EffectiveFrom.UnixMilli() } - volumes, err := m.ch.VolumeByMetric(ctx, metricNames, effectiveFrom, startMs, endMs) - if err != nil { - return nil, err - } - reducedNames := make([]string, 0, len(volumes)) - for name, volume := range volumes { - if effectiveRetained(volume.Ingested, volume.Reduced) < volume.Ingested { - reducedNames = append(reducedNames, name) - } - } - - points, err := m.ch.SeriesTimeseries(ctx, metricNames, reducedNames, effectiveFrom, startMs, endMs) + points, err := m.ch.SampleTimeseries(ctx, metricNames, effectiveFrom, startMs, endMs) if err != nil { return nil, err } @@ -414,7 +433,7 @@ func buildVolumeTimeseries(points []volumePoint) *querybuildertypesv5.QueryRange } func (m *module) validateMetricForReduction(ctx context.Context, orgID valuer.UUID, metricName string) error { - lastSeen, err := m.metadataStore.FetchLastSeenInfoMulti(ctx, metricName) + lastSeen, err := m.metadataStore.FetchLastSeenInfoMulti(ctx, orgID, metricName) if err != nil { return err } @@ -447,12 +466,12 @@ func (m *module) relatedAssetImpact(ctx context.Context, orgID valuer.UUID, metr m.logger.WarnContext(ctx, "failed to fetch related dashboards for reduction preview", slog.String("metric_name", metricName), errors.Attr(err)) } else { for _, item := range dashboards[metricName] { - usedLabels := append(splitCSV(item["group_by"]), splitCSV(item["filter_by"])...) + usedLabels := append(append([]string{}, item.GroupBy...), item.FilterBy...) affected = append(affected, metricreductionruletypes.AffectedAsset{ Type: metricreductionruletypes.AssetTypeDashboard, - ID: item["dashboard_id"], - Name: item["dashboard_name"], - Widget: &metricreductionruletypes.AffectedWidget{ID: item["widget_id"], Name: item["widget_name"]}, + ID: item.DashboardID, + Name: item.DashboardName, + Widget: &metricreductionruletypes.AffectedWidget{ID: item.PanelID, Name: item.PanelName}, ImpactedLabels: intersectLabels(usedLabels, droppedSet), }) } @@ -482,7 +501,7 @@ func toGettableReductionRule(rule *metricreductionruletypes.ReductionRule) metri MatchType: rule.MatchType, Labels: rule.Labels, EffectiveFrom: rule.EffectiveFrom, - Active: !rule.EffectiveFrom.After(time.Now()), + Active: !rule.EffectiveFrom.Add(uiActivationDelay).After(time.Now()), } } @@ -493,12 +512,11 @@ func effectiveRetained(ingested, reduced uint64) uint64 { return reduced } -func withVolume(rule metricreductionruletypes.GettableReductionRule, volume volumeRow) metricreductionruletypes.GettableReductionRule { - rule.IngestedSeries = volume.Ingested - rule.RetainedSeries = effectiveRetained(volume.Ingested, volume.Reduced) - if volume.Ingested > 0 { - rule.ReductionPercent = (1 - float64(rule.RetainedSeries)/float64(volume.Ingested)) * 100 - } +func withVolume(rule metricreductionruletypes.GettableReductionRule, series volumeRow, samples volumeRow) metricreductionruletypes.GettableReductionRule { + rule.IngestedSeries = series.Ingested + rule.RetainedSeries = effectiveRetained(series.Ingested, series.Reduced) + rule.IngestedSamples = samples.Ingested + rule.RetainedSamples = effectiveRetained(samples.Ingested, samples.Reduced) return rule } @@ -518,13 +536,6 @@ func intersectLabels(keys []string, droppedSet map[string]struct{}) []string { return out } -func splitCSV(s string) []string { - if s == "" { - return nil - } - return strings.Split(s, ",") -} - func resolveDroppedKept(matchType metricreductionruletypes.MatchType, ruleLabels, keys []string) (dropped, kept []string) { ruleSet := make(map[string]struct{}, len(ruleLabels)) for _, l := range ruleLabels { diff --git a/ee/modules/metricreductionrule/implmetricreductionrule/store.go b/ee/modules/metricreductionrule/implmetricreductionrule/store.go index 6090518b2f..28d64a3fb2 100644 --- a/ee/modules/metricreductionrule/implmetricreductionrule/store.go +++ b/ee/modules/metricreductionrule/implmetricreductionrule/store.go @@ -35,7 +35,7 @@ func (s *store) List(ctx context.Context, orgID valuer.UUID, params *metricreduc Where("org_id = ?", orgID). Order(column + " " + direction) if params.Search != "" { - query = query.Where("metric_name LIKE ?", "%"+params.Search+"%") + query = query.Where("metric_name LIKE ? ESCAPE '\\'", "%"+s.sqlstore.Formatter().EscapeLikePattern(params.Search)+"%") } if params.MetricName != "" { query = query.Where("metric_name = ?", params.MetricName) diff --git a/ee/querier/handler.go b/ee/querier/handler.go index 02bafd7790..e28abd1100 100644 --- a/ee/querier/handler.go +++ b/ee/querier/handler.go @@ -101,6 +101,10 @@ func (h *handler) QueryRange(rw http.ResponseWriter, req *http.Request) { h.community.QueryRange(rw, req) } +func (h *handler) QueryRangePreview(rw http.ResponseWriter, req *http.Request) { + h.community.QueryRangePreview(rw, req) +} + func (h *handler) QueryRawStream(rw http.ResponseWriter, req *http.Request) { h.community.QueryRawStream(rw, req) } diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 028f309576..1f5898acac 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -64,6 +64,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) { signoz.Prometheus, signoz.TelemetryStore.Cluster(), signoz.Cache, + signoz.Flagger, nil, ) diff --git a/frontend/public/locales/en-GB/titles.json b/frontend/public/locales/en-GB/titles.json index ee8581ef13..7dd6000c60 100644 --- a/frontend/public/locales/en-GB/titles.json +++ b/frontend/public/locales/en-GB/titles.json @@ -61,5 +61,7 @@ "ROLE_DETAILS": "SigNoz | Role Details", "TRACES_FUNNELS_DETAIL": "SigNoz | Funnel", "INTEGRATIONS_DETAIL": "SigNoz | Integration", - "PUBLIC_DASHBOARD": "SigNoz | Dashboard" + "PUBLIC_DASHBOARD": "SigNoz | Dashboard", + "LLM_OBSERVABILITY_BASE": "SigNoz | LLM Observability", + "LLM_OBSERVABILITY_MODEL_PRICING": "SigNoz | Model Pricing" } diff --git a/frontend/public/locales/en/titles.json b/frontend/public/locales/en/titles.json index 5dea25508b..cb0f91a67e 100644 --- a/frontend/public/locales/en/titles.json +++ b/frontend/public/locales/en/titles.json @@ -86,5 +86,7 @@ "ROLE_EDIT": "SigNoz | Edit Role", "TRACES_FUNNELS_DETAIL": "SigNoz | Funnel", "INTEGRATIONS_DETAIL": "SigNoz | Integration", - "PUBLIC_DASHBOARD": "SigNoz | Dashboard" + "PUBLIC_DASHBOARD": "SigNoz | Dashboard", + "LLM_OBSERVABILITY_BASE": "SigNoz | LLM Observability", + "LLM_OBSERVABILITY_MODEL_PRICING": "SigNoz | Model Pricing" } diff --git a/frontend/src/api/generated/services/querier/index.ts b/frontend/src/api/generated/services/querier/index.ts index 3bfa6f0dce..406fa2a6bc 100644 --- a/frontend/src/api/generated/services/querier/index.ts +++ b/frontend/src/api/generated/services/querier/index.ts @@ -12,6 +12,8 @@ import type { } from 'react-query'; import type { + QueryRangePreviewV5200, + QueryRangePreviewV5Params, QueryRangeV5200, Querybuildertypesv5QueryRangeRequestDTO, RenderErrorResponseDTO, @@ -104,6 +106,107 @@ export const useQueryRangeV5 = < > => { return useMutation(getQueryRangeV5MutationOptions(options)); }; +/** + * Validate a composite query without executing it. Accepts the same payload as the query range endpoint. By default (verbose=true) returns, for each query, the rendered underlying ClickHouse statement(s) with each statement's EXPLAIN ESTIMATE (per-table parts/rows/marks) and granule index analysis (candidate/surviving granules and the per-index pruning funnel). Pass ?verbose=false for the lightweight per-query verdict (valid/error/warnings) with no rendered SQL and no ClickHouse round trips. Intended for agentic/dry-run consumption: per-query errors are reported in the response rather than failing the whole request. + * @summary Query range preview + */ +export const queryRangePreviewV5 = ( + querybuildertypesv5QueryRangeRequestDTO?: BodyType, + params?: QueryRangePreviewV5Params, + signal?: AbortSignal, +) => { + return GeneratedAPIInstance({ + url: `/api/v5/query_range/preview`, + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + data: querybuildertypesv5QueryRangeRequestDTO, + params, + signal, + }); +}; + +export const getQueryRangePreviewV5MutationOptions = < + TError = ErrorType, + TContext = unknown, +>(options?: { + mutation?: UseMutationOptions< + Awaited>, + TError, + { + data?: BodyType; + params?: QueryRangePreviewV5Params; + }, + TContext + >; +}): UseMutationOptions< + Awaited>, + TError, + { + data?: BodyType; + params?: QueryRangePreviewV5Params; + }, + TContext +> => { + const mutationKey = ['queryRangePreviewV5']; + const { mutation: mutationOptions } = options + ? options.mutation && + 'mutationKey' in options.mutation && + options.mutation.mutationKey + ? options + : { ...options, mutation: { ...options.mutation, mutationKey } } + : { mutation: { mutationKey } }; + + const mutationFn: MutationFunction< + Awaited>, + { + data?: BodyType; + params?: QueryRangePreviewV5Params; + } + > = (props) => { + const { data, params } = props ?? {}; + + return queryRangePreviewV5(data, params); + }; + + return { mutationFn, ...mutationOptions }; +}; + +export type QueryRangePreviewV5MutationResult = NonNullable< + Awaited> +>; +export type QueryRangePreviewV5MutationBody = + | BodyType + | undefined; +export type QueryRangePreviewV5MutationError = + ErrorType; + +/** + * @summary Query range preview + */ +export const useQueryRangePreviewV5 = < + TError = ErrorType, + TContext = unknown, +>(options?: { + mutation?: UseMutationOptions< + Awaited>, + TError, + { + data?: BodyType; + params?: QueryRangePreviewV5Params; + }, + TContext + >; +}): UseMutationResult< + Awaited>, + TError, + { + data?: BodyType; + params?: QueryRangePreviewV5Params; + }, + TContext +> => { + return useMutation(getQueryRangePreviewV5MutationOptions(options)); +}; /** * Replace variables in a query * @summary Replace variables diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.ts index 907e2192a9..9d72a59b1d 100644 --- a/frontend/src/api/generated/services/sigNoz.schemas.ts +++ b/frontend/src/api/generated/services/sigNoz.schemas.ts @@ -2630,9 +2630,25 @@ export interface CloudintegrationtypesAzureAccountConfigDTO { resourceGroups: string[]; } +export interface CloudintegrationtypesGCPAccountConfigDTO { + /** + * @type string + */ + deploymentProjectId: string; + /** + * @type string + */ + deploymentRegion: string; + /** + * @type array + */ + projectIds: string[]; +} + export interface CloudintegrationtypesAccountConfigDTO { aws?: CloudintegrationtypesAWSAccountConfigDTO; azure?: CloudintegrationtypesAzureAccountConfigDTO; + gcp?: CloudintegrationtypesGCPAccountConfigDTO; } export interface CloudintegrationtypesAccountDTO { @@ -2740,9 +2756,29 @@ export interface CloudintegrationtypesAzureServiceConfigDTO { metrics: CloudintegrationtypesAzureServiceMetricsConfigDTO; } +export interface CloudintegrationtypesGCPServiceLogsConfigDTO { + /** + * @type boolean + */ + enabled: boolean; +} + +export interface CloudintegrationtypesGCPServiceMetricsConfigDTO { + /** + * @type boolean + */ + enabled: boolean; +} + +export interface CloudintegrationtypesGCPServiceConfigDTO { + logs?: CloudintegrationtypesGCPServiceLogsConfigDTO; + metrics?: CloudintegrationtypesGCPServiceMetricsConfigDTO; +} + export interface CloudintegrationtypesServiceConfigDTO { aws?: CloudintegrationtypesAWSServiceConfigDTO; azure?: CloudintegrationtypesAzureServiceConfigDTO; + gcp?: CloudintegrationtypesGCPServiceConfigDTO; } export enum CloudintegrationtypesServiceIDDTO { @@ -2773,6 +2809,7 @@ export enum CloudintegrationtypesServiceIDDTO { cosmosdb = 'cosmosdb', cassandradb = 'cassandradb', redis = 'redis', + cloudsql = 'cloudsql', } export type CloudintegrationtypesCloudIntegrationServiceDTOAnyOf = { /** @@ -2837,9 +2874,14 @@ export interface CloudintegrationtypesCollectedMetricDTO { unit?: string; } +export interface CloudintegrationtypesGCPConnectionArtifactDTO { + [key: string]: unknown; +} + export interface CloudintegrationtypesConnectionArtifactDTO { aws?: CloudintegrationtypesAWSConnectionArtifactDTO; azure?: CloudintegrationtypesAzureConnectionArtifactDTO; + gcp?: CloudintegrationtypesGCPConnectionArtifactDTO; } export interface CloudintegrationtypesCredentialsDTO { @@ -2872,6 +2914,10 @@ export interface CloudintegrationtypesDataCollectedDTO { metrics?: CloudintegrationtypesCollectedMetricDTO[] | null; } +export interface CloudintegrationtypesGCPIntegrationConfigDTO { + [key: string]: unknown; +} + export interface CloudintegrationtypesGettableAccountWithConnectionArtifactDTO { connectionArtifact: CloudintegrationtypesConnectionArtifactDTO; /** @@ -2963,6 +3009,7 @@ export type CloudintegrationtypesIntegrationConfigDTO = export interface CloudintegrationtypesProviderIntegrationConfigDTO { aws?: CloudintegrationtypesAWSIntegrationConfigDTO; azure?: CloudintegrationtypesAzureIntegrationConfigDTO; + gcp?: CloudintegrationtypesGCPIntegrationConfigDTO; } export interface CloudintegrationtypesGettableAgentCheckInDTO { @@ -3025,6 +3072,7 @@ export interface CloudintegrationtypesGettableServicesMetadataDTO { export interface CloudintegrationtypesPostableAccountConfigDTO { aws?: CloudintegrationtypesAWSPostableAccountConfigDTO; azure?: CloudintegrationtypesAzureAccountConfigDTO; + gcp?: CloudintegrationtypesGCPAccountConfigDTO; } export interface CloudintegrationtypesPostableAccountDTO { @@ -3154,9 +3202,25 @@ export interface CloudintegrationtypesUpdatableAzureAccountConfigDTO { resourceGroups: string[]; } +export interface CloudintegrationtypesUpdatableGCPAccountConfigDTO { + /** + * @type string + */ + deploymentProjectId: string; + /** + * @type string + */ + deploymentRegion: string; + /** + * @type array,null + */ + projectIds: string[] | null; +} + export interface CloudintegrationtypesUpdatableAccountConfigDTO { aws?: CloudintegrationtypesAWSAccountConfigDTO; azure?: CloudintegrationtypesUpdatableAzureAccountConfigDTO; + gcp?: CloudintegrationtypesUpdatableGCPAccountConfigDTO; } export interface CloudintegrationtypesUpdatableAccountDTO { @@ -3931,6 +3995,14 @@ export interface DashboardtypesDashboardPanelRefDTO { * @type string */ dashboardName: string; + /** + * @type array + */ + filterBy?: string[]; + /** + * @type array + */ + groupBy?: string[]; /** * @type string */ @@ -6965,6 +7037,11 @@ export interface MetricreductionruletypesGettableReductionRuleDTO { * @type string */ id: string; + /** + * @type integer + * @minimum 0 + */ + ingestedSamples: number; /** * @type integer * @minimum 0 @@ -6980,10 +7057,10 @@ export interface MetricreductionruletypesGettableReductionRuleDTO { */ metricName: string; /** - * @type number - * @format double + * @type integer + * @minimum 0 */ - reductionPercent: number; + retainedSamples: number; /** * @type integer * @minimum 0 @@ -7042,11 +7119,21 @@ export interface MetricreductionruletypesGettableReductionRuleStatsDTO { * @format double */ estimatedMonthlySavingsUsd: number; + /** + * @type integer + * @minimum 0 + */ + ingestedSamples: number; /** * @type integer * @minimum 0 */ ingestedSeries: number; + /** + * @type integer + * @minimum 0 + */ + retainedSamples: number; /** * @type integer * @minimum 0 @@ -7102,7 +7189,6 @@ export enum MetricreductionruletypesReductionRuleOrderByDTO { metric = 'metric', ingested_volume = 'ingested_volume', reduced_volume = 'reduced_volume', - reduction = 'reduction', last_updated = 'last_updated', } export interface MetricreductionruletypesUpdatableReductionRuleDTO { @@ -7665,6 +7751,126 @@ export interface Querybuildertypesv5FormatOptionsDTO { formatTableResultForUI?: boolean; } +export interface TelemetrystoretypesEstimateEntryDTO { + /** + * @type string + */ + database: string; + /** + * @type integer + * @format int64 + */ + marks: number; + /** + * @type integer + * @format int64 + */ + parts: number; + /** + * @type integer + * @format int64 + */ + rows: number; + /** + * @type string + */ + table: string; +} + +export interface TelemetrystoretypesIndexStepDTO { + /** + * @type string + */ + condition: string; + /** + * @type integer + * @format int64 + */ + initialGranules: number; + /** + * @type integer + * @format int64 + */ + initialParts: number; + /** + * @type array + */ + keys: string[]; + /** + * @type string + */ + name: string; + /** + * @type integer + * @format int64 + */ + selectedGranules: number; + /** + * @type integer + * @format int64 + */ + selectedParts: number; + /** + * @type string + */ + type: string; +} + +export interface TelemetrystoretypesMergeTreeReadDTO { + /** + * @type array + */ + steps: TelemetrystoretypesIndexStepDTO[]; + /** + * @type string + */ + table: string; +} + +export type TelemetrystoretypesGranulesDTOAnyOf = { + /** + * @type integer + * @format int64 + */ + initial: number; + /** + * @type array + */ + reads: TelemetrystoretypesMergeTreeReadDTO[]; + /** + * @type integer + * @format int64 + */ + selected: number; + /** + * @type integer + * @format int64 + */ + skipped: number; +}; + +/** + * @nullable + */ +export type TelemetrystoretypesGranulesDTO = + TelemetrystoretypesGranulesDTOAnyOf | null; + +export interface Querybuildertypesv5PreviewStatementDTO { + /** + * @type array + */ + 'db.statement.args': unknown[]; + /** + * @type string + */ + 'db.statement.query': string; + /** + * @type array + */ + estimate: TelemetrystoretypesEstimateEntryDTO[]; + granules: TelemetrystoretypesGranulesDTO | null; +} + export interface Querybuildertypesv5TimeSeriesDataDTO { /** * @type array,null @@ -7746,6 +7952,41 @@ export type Querybuildertypesv5QueryDataDTO = results?: unknown[] | null; }); +export interface Querybuildertypesv5QueryPreviewDTO { + error: unknown; + /** + * @type array + */ + statements: Querybuildertypesv5PreviewStatementDTO[]; + /** + * @type boolean + */ + valid: boolean; + /** + * @type array + */ + warnings: string[]; +} + +export type Querybuildertypesv5QueryRangePreviewResponseDTOCompositeQueryAnyOf = + { [key: string]: Querybuildertypesv5QueryPreviewDTO }; + +/** + * @nullable + */ +export type Querybuildertypesv5QueryRangePreviewResponseDTOCompositeQuery = + Querybuildertypesv5QueryRangePreviewResponseDTOCompositeQueryAnyOf | null; + +/** + * Response from the v5 query range preview (dry-run) endpoint. For each query in the composite query, returns the underlying ClickHouse statement(s) it renders to without executing them (one per PromQL metric selector; exactly one for builder/ClickHouse/trace-operator queries), with the optional EXPLAIN ESTIMATE and granule analysis attached per statement when requested. + */ +export interface Querybuildertypesv5QueryRangePreviewResponseDTO { + /** + * @type object,null + */ + compositeQuery: Querybuildertypesv5QueryRangePreviewResponseDTOCompositeQuery; +} + export enum Querybuildertypesv5VariableTypeDTO { query = 'query', dynamic = 'dynamic', @@ -11620,6 +11861,22 @@ export type QueryRangeV5200 = { status: string; }; +export type QueryRangePreviewV5Params = { + /** + * @type string + * @description undefined + */ + verbose?: string; +}; + +export type QueryRangePreviewV5200 = { + data: Querybuildertypesv5QueryRangePreviewResponseDTO; + /** + * @type string + */ + status: string; +}; + export type ReplaceVariables200 = { data: Querybuildertypesv5QueryRangeRequestDTO; /** diff --git a/frontend/src/components/InviteMembers/__tests__/InviteMembers.submission.test.tsx b/frontend/src/components/InviteMembers/__tests__/InviteMembers.submission.test.tsx index 4869dd5f38..b86c0eafca 100644 --- a/frontend/src/components/InviteMembers/__tests__/InviteMembers.submission.test.tsx +++ b/frontend/src/components/InviteMembers/__tests__/InviteMembers.submission.test.tsx @@ -167,6 +167,7 @@ describe('InviteMembers - Submission', () => { success: false, }), ]), + expect.any(Array), ); }); }); @@ -243,6 +244,7 @@ describe('InviteMembers - Submission', () => { error: 'User already exists', }), ]), + expect.any(Array), ); await expect( diff --git a/frontend/src/components/InviteMembers/types.ts b/frontend/src/components/InviteMembers/types.ts index e3028175b4..787b1617fd 100644 --- a/frontend/src/components/InviteMembers/types.ts +++ b/frontend/src/components/InviteMembers/types.ts @@ -22,9 +22,9 @@ export interface FooterRenderProps { export interface UseInviteMembersOptions { initialRowCount?: number; - onSuccess?: () => void; - onPartialSuccess?: (results: InviteResult[]) => void; - onAllFailed?: (results: InviteResult[]) => void; + onSuccess?: (results: InviteResult[], rows: InviteMemberRow[]) => void; + onPartialSuccess?: (results: InviteResult[], rows: InviteMemberRow[]) => void; + onAllFailed?: (results: InviteResult[], rows: InviteMemberRow[]) => void; } export interface UseInviteMembersReturn { @@ -56,9 +56,9 @@ export interface InviteMembersProps { showHeader?: boolean; showAddButton?: boolean; - onSuccess?: () => void; - onPartialSuccess?: (results: InviteResult[]) => void; - onAllFailed?: (results: InviteResult[]) => void; + onSuccess?: (results: InviteResult[], rows: InviteMemberRow[]) => void; + onPartialSuccess?: (results: InviteResult[], rows: InviteMemberRow[]) => void; + onAllFailed?: (results: InviteResult[], rows: InviteMemberRow[]) => void; renderFooter?: (props: FooterRenderProps) => ReactNode; } diff --git a/frontend/src/components/InviteMembers/useInviteMembers.ts b/frontend/src/components/InviteMembers/useInviteMembers.ts index 2faf0d8243..00276263fb 100644 --- a/frontend/src/components/InviteMembers/useInviteMembers.ts +++ b/frontend/src/components/InviteMembers/useInviteMembers.ts @@ -207,11 +207,11 @@ export function useInviteMembers( const successes = results.filter((r) => r.success); if (failures.length === 0) { - onSuccess?.(); + onSuccess?.(results, touched); } else if (successes.length > 0) { - onPartialSuccess?.(results); + onPartialSuccess?.(results, touched); } else { - onAllFailed?.(results); + onAllFailed?.(results, touched); } return results; diff --git a/frontend/src/components/InviteMembersModal/InviteMembersModal.styles.scss b/frontend/src/components/InviteMembersModal/InviteMembersModal.styles.scss deleted file mode 100644 index 3b2c6fade5..0000000000 --- a/frontend/src/components/InviteMembersModal/InviteMembersModal.styles.scss +++ /dev/null @@ -1,254 +0,0 @@ -.invite-members-modal { - max-width: 700px; - background: var(--popover); - border: 1px solid var(--secondary); - border-radius: 4px; - box-shadow: 0 4px 9px 0 rgba(0, 0, 0, 0.04); - - [data-slot='dialog-header'] { - padding: var(--padding-4); - border-bottom: 1px solid var(--secondary); - flex-shrink: 0; - background: transparent; - margin: 0; - } - - [data-slot='dialog-title'] { - font-family: Inter, sans-serif; - font-size: var(--label-base-400-font-size); - font-weight: var(--label-base-400-font-weight); - line-height: var(--label-base-400-line-height); - letter-spacing: -0.065px; - color: var(--l1-foreground); - margin: 0; - } - - [data-slot='dialog-description'] { - padding: 0; - - .invite-members-modal__content { - display: flex; - flex-direction: column; - gap: var(--spacing-8); - padding: var(--padding-4); - } - } -} - -.invite-members-modal__table { - display: flex; - flex-direction: column; - gap: var(--spacing-4); - width: 100%; -} - -.invite-members-modal__table-header { - display: flex; - align-items: center; - gap: var(--spacing-8); - width: 100%; - - .email-header { - flex: 0 0 240px; - } - - .role-header { - flex: 1 0 0; - min-width: 0; - } - - .action-header { - flex: 0 0 32px; - } - - .table-header-cell { - font-family: Inter, sans-serif; - font-size: var(--paragraph-base-400-font-size); - font-weight: var(--paragraph-base-400-font-weight); - line-height: var(--paragraph-base-400-line-height); - letter-spacing: -0.07px; - color: var(--foreground); - } -} - -.invite-members-modal__container { - display: flex; - flex-direction: column; - gap: var(--spacing-8); - width: 100%; -} - -.team-member-row { - display: flex; - align-items: flex-start; - gap: var(--spacing-8); - width: 100%; - - > .email-cell { - flex: 0 0 240px; - } - - > .role-cell { - flex: 1 0 0; - min-width: 0; - } - - > .action-cell { - flex: 0 0 32px; - } -} - -.team-member-cell { - display: flex; - flex-direction: column; - gap: var(--spacing-2); - - &.action-cell { - display: flex; - align-items: center; - justify-content: center; - height: 32px; - } -} - -.team-member-email-input { - width: 100%; - height: 32px; - color: var(--l1-foreground); - background-color: var(--l2-background); - border-color: var(--l1-border); - font-size: var(--paragraph-base-400-font-size); - - &::placeholder { - color: var(--l3-foreground); - } - - &:focus { - border-color: var(--primary); - box-shadow: none; - } -} -.team-member-role-select { - width: 100%; - - .ant-select-selector { - height: 32px; - border-radius: 2px; - background-color: var(--l2-background) !important; - border: 1px solid var(--border) !important; - padding: 0 var(--padding-2) !important; - - .ant-select-selection-placeholder { - color: var(--l3-foreground); - opacity: 0.4; - font-size: var(--paragraph-base-400-font-size); - letter-spacing: -0.07px; - line-height: 32px; - } - - .ant-select-selection-item { - font-size: var(--paragraph-base-400-font-size); - letter-spacing: -0.07px; - color: var(--l1-foreground); - line-height: 32px; - } - } - - .ant-select-arrow { - color: var(--foreground); - } - - &.ant-select-focused .ant-select-selector, - &:not(.ant-select-disabled):hover .ant-select-selector { - border-color: var(--primary); - } -} - -.remove-team-member-button { - display: flex; - align-items: center; - justify-content: center; - width: 32px; - height: 32px; - min-width: 32px; - border: none; - border-radius: 2px; - background: transparent; - color: var(--destructive); - opacity: 0.6; - padding: 0; - transition: - background-color 0.2s, - opacity 0.2s; - box-shadow: none; - - &:hover { - background: color-mix(in srgb, var(--danger-background) 10%, transparent); - opacity: 0.9; - } -} - -.email-error-message { - display: block; - font-family: Inter, sans-serif; - font-size: var(--font-size-xs); - font-weight: var(--font-weight-normal); - line-height: var(--line-height-18); - color: var(--destructive); -} - -.invite-team-members-error-callout { - background: color-mix(in srgb, var(--danger-background) 10%, transparent); - border: 1px solid color-mix(in srgb, var(--danger-background) 20%, transparent); - border-radius: 4px; - animation: horizontal-shaking 300ms ease-out; -} - -.invite-members-modal__error-callout { - display: flex; -} - -@keyframes horizontal-shaking { - 0% { - transform: translateX(0); - } - 25% { - transform: translateX(5px); - } - 50% { - transform: translateX(-5px); - } - 75% { - transform: translateX(5px); - } - 100% { - transform: translateX(0); - } -} - -.invite-members-modal__footer { - display: flex; - flex-direction: row; - align-items: center; - justify-content: space-between; - padding: 0 var(--padding-4); - height: 56px; - min-height: 56px; - border-top: 1px solid var(--secondary); - gap: 0; - flex-shrink: 0; -} - -.invite-members-modal__footer-right { - display: flex; - align-items: center; - gap: var(--spacing-6); -} - -.add-another-member-button { - &:hover { - border-color: var(--primary); - border-style: dashed; - color: var(--l1-foreground); - } -} diff --git a/frontend/src/components/InviteMembersModal/InviteMembersModal.tsx b/frontend/src/components/InviteMembersModal/InviteMembersModal.tsx deleted file mode 100644 index 08101a2060..0000000000 --- a/frontend/src/components/InviteMembersModal/InviteMembersModal.tsx +++ /dev/null @@ -1,337 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from 'react'; -import { Style } from '@signozhq/design-tokens'; -import { ChevronDown, Plus, Trash2, X } from '@signozhq/icons'; -import { Button } from '@signozhq/ui/button'; -import { Callout } from '@signozhq/ui/callout'; -import { DialogFooter, DialogWrapper } from '@signozhq/ui/dialog'; -import { Input } from '@signozhq/ui/input'; -import { toast } from '@signozhq/ui/sonner'; -import { Select } from 'antd'; -import inviteUsers from 'api/v1/invite/bulk/create'; -import sendInvite from 'api/v1/invite/create'; -import { cloneDeep, debounce } from 'lodash-es'; -import { useErrorModal } from 'providers/ErrorModalProvider'; -import APIError from 'types/api/error'; -import { ROLES } from 'types/roles'; -import { EMAIL_REGEX } from 'utils/app'; -import { getBaseUrl } from 'utils/basePath'; -import { popupContainer } from 'utils/selectPopupContainer'; -import { v4 as uuid } from 'uuid'; - -import './InviteMembersModal.styles.scss'; - -interface InviteRow { - id: string; - email: string; - role: ROLES | ''; -} - -export interface InviteMembersModalProps { - open: boolean; - onClose: () => void; - onComplete?: () => void; -} - -const EMPTY_ROW = (): InviteRow => ({ id: uuid(), email: '', role: '' }); - -const isRowTouched = (row: InviteRow): boolean => - row.email.trim() !== '' || Boolean(row.role && row.role.trim() !== ''); - -function InviteMembersModal({ - open, - onClose, - onComplete, -}: InviteMembersModalProps): JSX.Element { - const { showErrorModal, isErrorModalVisible } = useErrorModal(); - - const [rows, setRows] = useState(() => [ - EMPTY_ROW(), - EMPTY_ROW(), - EMPTY_ROW(), - ]); - const [isSubmitting, setIsSubmitting] = useState(false); - const [emailValidity, setEmailValidity] = useState>( - {}, - ); - const [hasInvalidEmails, setHasInvalidEmails] = useState(false); - const [hasInvalidRoles, setHasInvalidRoles] = useState(false); - - const resetAndClose = useCallback((): void => { - setRows([EMPTY_ROW(), EMPTY_ROW(), EMPTY_ROW()]); - setEmailValidity({}); - setHasInvalidEmails(false); - setHasInvalidRoles(false); - onClose(); - }, [onClose]); - - useEffect(() => { - if (open) { - setRows([EMPTY_ROW(), EMPTY_ROW(), EMPTY_ROW()]); - } - }, [open]); - - const getValidationErrorMessage = (): string => { - if (hasInvalidEmails && hasInvalidRoles) { - return 'Please enter valid emails and select roles for team members'; - } - if (hasInvalidEmails) { - return 'Please enter valid emails for team members'; - } - return 'Please select roles for team members'; - }; - - const validateAllUsers = useCallback((): boolean => { - let isValid = true; - let hasEmailErrors = false; - let hasRoleErrors = false; - - const updatedEmailValidity: Record = {}; - - const touchedRows = rows.filter(isRowTouched); - - touchedRows.forEach((row) => { - const emailValid = EMAIL_REGEX.test(row.email); - const roleValid = Boolean(row.role && row.role.trim() !== ''); - - if (!emailValid || !row.email) { - isValid = false; - hasEmailErrors = true; - } - if (!roleValid) { - isValid = false; - hasRoleErrors = true; - } - - if (row.id) { - updatedEmailValidity[row.id] = emailValid; - } - }); - - setEmailValidity(updatedEmailValidity); - setHasInvalidEmails(hasEmailErrors); - setHasInvalidRoles(hasRoleErrors); - - return isValid; - }, [rows]); - - const debouncedValidateEmail = useMemo( - () => - debounce((email: string, rowId: string) => { - const isValid = EMAIL_REGEX.test(email); - setEmailValidity((prev) => ({ ...prev, [rowId]: isValid })); - }, 500), - [], - ); - - useEffect(() => { - if (!open) { - debouncedValidateEmail.cancel(); - } - return (): void => { - debouncedValidateEmail.cancel(); - }; - }, [open, debouncedValidateEmail]); - - const updateEmail = (id: string, email: string): void => { - const updatedRows = cloneDeep(rows); - const rowToUpdate = updatedRows.find((r) => r.id === id); - if (rowToUpdate) { - rowToUpdate.email = email; - setRows(updatedRows); - - if (hasInvalidEmails) { - setHasInvalidEmails(false); - } - if (emailValidity[id] === false) { - setEmailValidity((prev) => ({ ...prev, [id]: true })); - } - - debouncedValidateEmail(email, id); - } - }; - - const updateRole = (id: string, role: ROLES): void => { - const updatedRows = cloneDeep(rows); - const rowToUpdate = updatedRows.find((r) => r.id === id); - if (rowToUpdate) { - rowToUpdate.role = role; - setRows(updatedRows); - - if (hasInvalidRoles) { - setHasInvalidRoles(false); - } - } - }; - - const addRow = (): void => { - setRows((prev) => [...prev, EMPTY_ROW()]); - }; - - const removeRow = (id: string): void => { - setRows((prev) => prev.filter((r) => r.id !== id)); - }; - - const handleSubmit = useCallback(async (): Promise => { - if (!validateAllUsers()) { - return; - } - - const touchedRows = rows.filter(isRowTouched); - if (touchedRows.length === 0) { - return; - } - - setIsSubmitting(true); - try { - if (touchedRows.length === 1) { - const row = touchedRows[0]; - await sendInvite({ - email: row.email.trim(), - name: '', - role: row.role as ROLES, - frontendBaseUrl: getBaseUrl(), - }); - } else { - await inviteUsers({ - invites: touchedRows.map((row) => ({ - email: row.email.trim(), - name: '', - role: row.role, - frontendBaseUrl: getBaseUrl(), - })), - }); - } - toast.success('Invites sent successfully', { position: 'top-right' }); - resetAndClose(); - onComplete?.(); - } catch (err) { - showErrorModal(err as APIError); - } finally { - setIsSubmitting(false); - } - }, [validateAllUsers, rows, resetAndClose, onComplete, showErrorModal]); - - const touchedRows = rows.filter(isRowTouched); - const isSubmitDisabled = isSubmitting || touchedRows.length === 0; - - return ( - { - if (!isOpen) { - resetAndClose(); - } - }} - showCloseButton - width="wide" - className="invite-members-modal" - disableOutsideClick={isErrorModalVisible} - > -
-
-
-
Email address
-
Roles
-
-
-
- {rows.map( - (row): JSX.Element => ( -
-
- updateEmail(row.id, e.target.value)} - className="team-member-email-input" - name={`invite-email-${row.id}`} - autoComplete="email" - /> - {emailValidity[row.id] === false && row.email.trim() !== '' && ( - Invalid email address - )} -
-
- -
-
- {rows.length > 1 && ( - - )} -
-
- ), - )} -
-
- - {(hasInvalidEmails || hasInvalidRoles) && ( -
- -
- )} -
- - - - -
- - - -
-
- - ); -} - -export default InviteMembersModal; diff --git a/frontend/src/components/InviteMembersModal/__tests__/InviteMembersModal.test.tsx b/frontend/src/components/InviteMembersModal/__tests__/InviteMembersModal.test.tsx deleted file mode 100644 index 31fb59ddad..0000000000 --- a/frontend/src/components/InviteMembersModal/__tests__/InviteMembersModal.test.tsx +++ /dev/null @@ -1,276 +0,0 @@ -import inviteUsers from 'api/v1/invite/bulk/create'; -import sendInvite from 'api/v1/invite/create'; -import { StatusCodes } from 'http-status-codes'; -import { render, screen, userEvent, waitFor } from 'tests/test-utils'; -import APIError from 'types/api/error'; - -import InviteMembersModal from '../InviteMembersModal'; - -const makeApiError = (message: string, code = StatusCodes.CONFLICT): APIError => - new APIError({ - httpStatusCode: code, - error: { code: 'already_exists', message, url: '', errors: [] }, - }); - -jest.mock('api/v1/invite/create'); -jest.mock('api/v1/invite/bulk/create'); -jest.mock('@signozhq/ui/sonner', () => ({ - ...jest.requireActual('@signozhq/ui/sonner'), - toast: { - success: jest.fn(), - error: jest.fn(), - }, -})); - -const showErrorModal = jest.fn(); -jest.mock('providers/ErrorModalProvider', () => ({ - __esModule: true, - ...jest.requireActual('providers/ErrorModalProvider'), - useErrorModal: jest.fn(() => ({ - showErrorModal, - isErrorModalVisible: false, - })), -})); - -const mockSendInvite = jest.mocked(sendInvite); -const mockInviteUsers = jest.mocked(inviteUsers); - -const defaultProps = { - open: true, - onClose: jest.fn(), - onComplete: jest.fn(), -}; - -describe('InviteMembersModal', () => { - beforeEach(() => { - jest.clearAllMocks(); - showErrorModal.mockClear(); - mockSendInvite.mockResolvedValue({ - httpStatusCode: 200, - data: { data: 'test', status: 'success' }, - }); - mockInviteUsers.mockResolvedValue({ httpStatusCode: 200, data: null }); - }); - - it('renders 3 initial empty rows and disables the submit button', () => { - render(); - - const emailInputs = screen.getAllByPlaceholderText('john@signoz.io'); - expect(emailInputs).toHaveLength(3); - - expect( - screen.getByRole('button', { name: /invite team members/i }), - ).toBeDisabled(); - }); - - it('adds a row when "Add another" is clicked and removes a row via trash button', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - - render(); - - await user.click(screen.getByRole('button', { name: /add another/i })); - expect(screen.getAllByPlaceholderText('john@signoz.io')).toHaveLength(4); - - const removeButtons = screen.getAllByRole('button', { name: /remove row/i }); - await user.click(removeButtons[0]); - expect(screen.getAllByPlaceholderText('john@signoz.io')).toHaveLength(3); - }); - - describe('validation callout messages', () => { - it('shows combined message when email is invalid and role is missing', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - - render(); - - await user.type( - screen.getAllByPlaceholderText('john@signoz.io')[0], - 'not-an-email', - ); - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await expect( - screen.findByText( - 'Please enter valid emails and select roles for team members', - ), - ).resolves.toBeInTheDocument(); - }); - - it('shows email-only message when email is invalid but role is selected', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - - render(); - - const emailInputs = screen.getAllByPlaceholderText('john@signoz.io'); - await user.type(emailInputs[0], 'not-an-email'); - - await user.click(screen.getAllByText('Select roles')[0]); - await user.click(await screen.findByText('Viewer')); - - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await expect( - screen.findByText('Please enter valid emails for team members'), - ).resolves.toBeInTheDocument(); - }); - - it('shows role-only message when email is valid but role is missing', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - - render(); - - await user.type( - screen.getAllByPlaceholderText('john@signoz.io')[0], - 'valid@signoz.io', - ); - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await expect( - screen.findByText('Please select roles for team members'), - ).resolves.toBeInTheDocument(); - }); - }); - - it('uses sendInvite (single) when only one row is filled', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - const onComplete = jest.fn(); - - render(); - - const emailInputs = screen.getAllByPlaceholderText('john@signoz.io'); - await user.type(emailInputs[0], 'single@signoz.io'); - - const roleSelects = screen.getAllByText('Select roles'); - await user.click(roleSelects[0]); - await user.click(await screen.findByText('Viewer')); - - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await waitFor(() => { - expect(mockSendInvite).toHaveBeenCalledWith( - expect.objectContaining({ email: 'single@signoz.io', role: 'VIEWER' }), - ); - expect(mockInviteUsers).not.toHaveBeenCalled(); - expect(onComplete).toHaveBeenCalled(); - }); - }); - - describe('error handling', () => { - it('shows BE message on single invite 409', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - const error = makeApiError( - 'An invite already exists for this email: single@signoz.io', - ); - mockSendInvite.mockRejectedValue(error); - - render(); - - await user.type( - screen.getAllByPlaceholderText('john@signoz.io')[0], - 'single@signoz.io', - ); - await user.click(screen.getAllByText('Select roles')[0]); - await user.click(await screen.findByText('Viewer')); - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await waitFor(() => { - expect(showErrorModal).toHaveBeenCalledWith(error); - }); - }); - - it('shows BE message on bulk invite 409', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - const error = makeApiError( - 'An invite already exists for this email: alice@signoz.io', - ); - mockInviteUsers.mockRejectedValue(error); - - render(); - - const emailInputs = screen.getAllByPlaceholderText('john@signoz.io'); - await user.type(emailInputs[0], 'alice@signoz.io'); - await user.click(screen.getAllByText('Select roles')[0]); - await user.click(await screen.findByText('Viewer')); - - await user.type(emailInputs[1], 'bob@signoz.io'); - await user.click(screen.getAllByText('Select roles')[0]); - const editorOptions = await screen.findAllByText('Editor'); - await user.click(editorOptions[editorOptions.length - 1]); - - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await waitFor(() => { - expect(showErrorModal).toHaveBeenCalledWith(error); - }); - }); - - it('shows BE message on generic error', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - const error = makeApiError( - 'Internal server error', - StatusCodes.INTERNAL_SERVER_ERROR, - ); - mockSendInvite.mockRejectedValue(error); - - render(); - - await user.type( - screen.getAllByPlaceholderText('john@signoz.io')[0], - 'single@signoz.io', - ); - await user.click(screen.getAllByText('Select roles')[0]); - await user.click(await screen.findByText('Viewer')); - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await waitFor(() => { - expect(showErrorModal).toHaveBeenCalledWith(error); - }); - }); - }); - - it('uses inviteUsers (bulk) when multiple rows are filled', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - const onComplete = jest.fn(); - - render(); - - const emailInputs = screen.getAllByPlaceholderText('john@signoz.io'); - - await user.type(emailInputs[0], 'alice@signoz.io'); - await user.click(screen.getAllByText('Select roles')[0]); - await user.click(await screen.findByText('Viewer')); - - await user.type(emailInputs[1], 'bob@signoz.io'); - await user.click(screen.getAllByText('Select roles')[0]); - const editorOptions = await screen.findAllByText('Editor'); - await user.click(editorOptions[editorOptions.length - 1]); - - await user.click( - screen.getByRole('button', { name: /invite team members/i }), - ); - - await waitFor(() => { - expect(mockInviteUsers).toHaveBeenCalledWith({ - invites: expect.arrayContaining([ - expect.objectContaining({ email: 'alice@signoz.io', role: 'VIEWER' }), - expect.objectContaining({ email: 'bob@signoz.io', role: 'EDITOR' }), - ]), - }); - expect(mockSendInvite).not.toHaveBeenCalled(); - expect(onComplete).toHaveBeenCalled(); - }); - }); -}); diff --git a/frontend/src/components/MembersTable/MembersTable.styles.scss b/frontend/src/components/MembersTable/MembersTable.styles.scss index e25bc5b889..f6d4425134 100644 --- a/frontend/src/components/MembersTable/MembersTable.styles.scss +++ b/frontend/src/components/MembersTable/MembersTable.styles.scss @@ -3,7 +3,6 @@ flex-direction: column; flex: 1; min-height: 0; - overflow: hidden; border-radius: 4px; } diff --git a/frontend/src/components/RolesSelect/RolesSelect.tsx b/frontend/src/components/RolesSelect/RolesSelect.tsx index 050c17ccd7..3eaca36d62 100644 --- a/frontend/src/components/RolesSelect/RolesSelect.tsx +++ b/frontend/src/components/RolesSelect/RolesSelect.tsx @@ -32,10 +32,13 @@ export function useRoles(): { }; } -export function getRoleOptions(roles: AuthtypesRoleDTO[]): RoleOption[] { +export function getRoleOptions( + roles: AuthtypesRoleDTO[], + valueField: 'id' | 'name', +): RoleOption[] { return roles.map((role) => ({ label: role.name ?? '', - value: role.id ?? '', + value: role[valueField] ?? '', })); } @@ -82,6 +85,7 @@ interface BaseProps { error?: APIError; onRefetch?: () => void; disabled?: boolean; + valueField?: 'id' | 'name'; } interface SingleProps extends BaseProps { @@ -113,7 +117,7 @@ function RolesSelect(props: RolesSelectProps): JSX.Element { }); const roles = externalRoles ?? data?.data ?? []; - const options = getRoleOptions(roles); + const options = getRoleOptions(roles, props.valueField || 'id'); const { mode, diff --git a/frontend/src/components/YAxisUnitSelector/__tests__/formatter.test.tsx b/frontend/src/components/YAxisUnitSelector/__tests__/formatter.test.tsx index 140b1cd77d..f0d16d81ed 100644 --- a/frontend/src/components/YAxisUnitSelector/__tests__/formatter.test.tsx +++ b/frontend/src/components/YAxisUnitSelector/__tests__/formatter.test.tsx @@ -680,6 +680,13 @@ describe('formatUniversalUnit', () => { }); describe('Datetime', () => { + beforeAll(() => { + jest.useFakeTimers().setSystemTime(new Date('2026-01-01T00:00:00Z')); + }); + afterAll(() => { + jest.useRealTimers(); + }); + it('formats datetime units', () => { expect(formatUniversalUnit(900, UniversalYAxisUnit.DATETIME_FROM_NOW)).toBe( '56 years ago', diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.module.scss index 7133ffa6a3..369ea10f92 100644 --- a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.module.scss +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.module.scss @@ -1,7 +1,28 @@ +.filtersBar { + display: flex; + gap: var(--spacing-6); + align-items: center; + justify-content: space-between; +} + +.filtersBarLeft { + display: flex; + gap: var(--spacing-6); + align-items: center; +} + +.filtersBarSearch { + width: 280px; +} + +.filtersBarSource { + width: 160px; +} + .pageError { padding: var(--spacing-6) var(--spacing-8); border-radius: var(--radius-2); - background: color-mix(in srgb, var(--bg-cherry-400) 8%, transparent); - color: var(--text-cherry-400); + background: color-mix(in srgb, var(--accent-cherry) 8%, transparent); + color: var(--accent-cherry); font-size: var(--periscope-font-size-base); } diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.tsx b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.tsx index bcd552608f..51ebae9003 100644 --- a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.tsx +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/ModelCostTabPanel.tsx @@ -1,52 +1,164 @@ import { useMemo } from 'react'; +import { Button } from '@signozhq/ui/button'; +import { Input } from '@signozhq/ui/input'; +import { SelectSimple } from '@signozhq/ui/select'; +import { Typography } from '@signozhq/ui/typography'; +import { Plus, Search, X } from '@signozhq/icons'; import { useListLLMPricingRules } from 'api/generated/services/llmpricingrules'; import { type ListLLMPricingRulesParams } from 'api/generated/services/sigNoz.schemas'; import { useTableParams } from 'components/TanStackTableView'; -import { Typography } from '@signozhq/ui/typography'; +import useComponentPermission from 'hooks/useComponentPermission'; +import useDebounce from 'hooks/useDebounce'; +import { parseAsString, parseAsStringEnum, useQueryState } from 'nuqs'; +import { useAppContext } from 'providers/App/App'; -import { LIMIT_KEY, PAGE_KEY, PAGE_SIZE } from '../constants'; -import styles from './ModelCostTabPanel.module.scss'; +import { + LIMIT_KEY, + PAGE_KEY, + PAGE_SIZE, + SEARCH_DEBOUNCE_MS, + SEARCH_KEY, + SOURCE_FILTER_OPTIONS, + SOURCE_FILTER_TO_IS_OVERRIDE, + SOURCE_KEY, + type SourceFilter, +} from '../constants'; +import type { PricingRule } from '../types'; +import DeleteConfirmDialog from './components/DeleteConfirmDialog'; +import ModelCostDrawer, { + useModelCostDrawer, +} from './components/ModelCostDrawer'; import ModelCostsTable from './components/ModelCostsTable'; -import { type LlmpricingruletypesLLMPricingRuleDTO } from 'api/generated/services/sigNoz.schemas'; +import { useModelCostDelete } from './hooks/useModelCostDelete'; +import styles from './ModelCostTabPanel.module.scss'; +// "Model costs" tab: the priced-model listing, search + source filter, the add/ +// edit drawer, and pagination. Page and page size live in the URL (shareable/ +// reload-safe) and are owned by TanStackTable via enableQueryParams — this tab +// reads them back through the same useTableParams hook so the two stay in lockstep. function ModelCostTabPanel(): JSX.Element { - const { page, limit } = useTableParams( + const { page, limit, setPage } = useTableParams( { page: PAGE_KEY, limit: LIMIT_KEY }, { page: 1, limit: PAGE_SIZE }, ); - // Search + source filters are intentionally omitted for now — the list API - // doesn't honour them yet. They'll be reintroduced here once it does. + const [search, setSearch] = useQueryState( + SEARCH_KEY, + parseAsString.withDefault(''), + ); + const debouncedSearch = useDebounce(search, SEARCH_DEBOUNCE_MS); + + const [source, setSource] = useQueryState( + SOURCE_KEY, + parseAsStringEnum( + SOURCE_FILTER_OPTIONS.map((option) => option.value), + ).withDefault('all'), + ); + + const handleSearchChange = ( + event: React.ChangeEvent, + ): void => { + void setSearch(event.target.value || null); + setPage(1); + }; + + const clearSearch = (): void => { + void setSearch(null); + setPage(1); + }; + + const handleSourceChange = (value: string | string[]): void => { + void setSource(value as SourceFilter); + setPage(1); + }; + + const isOverride = SOURCE_FILTER_TO_IS_OVERRIDE[source]; + const listParams: ListLLMPricingRulesParams = { offset: (page - 1) * limit, limit, + ...(debouncedSearch ? { q: debouncedSearch } : {}), + ...(isOverride !== undefined ? { isOverride } : {}), }; - const { data, isLoading, isError } = useListLLMPricingRules(listParams); + const { data, isLoading, isError } = useListLLMPricingRules(listParams, { + query: { + enabled: search === debouncedSearch, + }, + }); - const rules: LlmpricingruletypesLLMPricingRuleDTO[] = useMemo( - () => data?.data?.items || [], - [data], + const { user } = useAppContext(); + const [canManagePricing] = useComponentPermission( + ['manage_llm_pricing'], + user.role, ); + + const rules: PricingRule[] = useMemo(() => data?.data?.items || [], [data]); const total = data?.data?.total ?? 0; + const drawer = useModelCostDrawer(); + const deletion = useModelCostDelete(); + return ( <> +
+
+ } + suffix={ + search ? ( +
+ {canManagePricing && ( + + )} +
+ {isError && (
Failed to load pricing rules. Please try again.
)} - {/* Read-only listing. Edit/Add wiring + the drawer land in the next PR. */} undefined} - onDelete={(): void => undefined} + selectedRuleId={drawer.selectedRuleId} + canManage={canManagePricing} + onEdit={drawer.openForEdit} + onDelete={deletion.requestDelete} />
@@ -54,6 +166,29 @@ function ModelCostTabPanel(): JSX.Element { All prices per 1M tokens (USD)
+ + {drawer.isOpen && ( + + )} + + {deletion.pendingDelete && ( + + )} ); } diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/DeleteConfirmDialog/DeleteConfirmDialog.tsx b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/DeleteConfirmDialog/DeleteConfirmDialog.tsx new file mode 100644 index 0000000000..151fd9e70b --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/DeleteConfirmDialog/DeleteConfirmDialog.tsx @@ -0,0 +1,64 @@ +import { AlertDialog } from '@signozhq/ui/alert-dialog'; +import { Button } from '@signozhq/ui/button'; +import { Trash2, X } from '@signozhq/icons'; + +interface DeleteConfirmDialogProps { + open: boolean; + modelName: string; + isDeleting: boolean; + onConfirm: () => void; + onCancel: () => void; +} + +// Confirmation step before deleting a model cost — deletion is irreversible, so +// the destructive action is gated behind an explicit confirm. AlertDialog blocks +// outside-click dismissal and hides the close button to force an explicit choice. +function DeleteConfirmDialog({ + open, + modelName, + isDeleting, + onConfirm, + onCancel, +}: DeleteConfirmDialogProps): JSX.Element { + return ( + { + if (!isOpen) { + onCancel(); + } + }} + width="narrow" + title="Delete Model Cost Data " + titleIcon={} + footer={ + <> + + + + } + > + Are you sure you want to delete {modelName}? Once deleted, + this action cannot be undone. + + ); +} + +export default DeleteConfirmDialog; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/DeleteConfirmDialog/index.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/DeleteConfirmDialog/index.ts new file mode 100644 index 0000000000..8cca597b94 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/DeleteConfirmDialog/index.ts @@ -0,0 +1 @@ +export { default } from './DeleteConfirmDialog'; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/ModelCostDrawer.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/ModelCostDrawer.module.scss new file mode 100644 index 0000000000..bdda883563 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/ModelCostDrawer.module.scss @@ -0,0 +1,58 @@ +.drawerSection { + composes: drawerSection from './shared.module.scss'; +} + +.fullWidth { + width: 100%; +} + +.required { + composes: required from './shared.module.scss'; +} + +.modelCostDrawer { + // Uniform horizontal padding across header / body / footer. The header and + // footer read these dialog vars; the body (rendered in drawer-description) + // is set directly below. + --dialog-header-padding: var(--spacing-10) var(--spacing-12); + --dialog-footer-padding: var(--spacing-8) var(--spacing-12); + + display: flex; + overflow-y: auto; + + // The drawer body — children render inside [data-slot='drawer-description'] + // (this is the @signozhq drawer, not antd, so .ant-drawer-body was a no-op). + [data-slot='drawer-description'] { + display: flex; + flex-direction: column; + gap: var(--spacing-12); + padding: var(--spacing-10) var(--spacing-12); + } + + [data-slot='select-content'] { + width: var(--radix-select-trigger-width); + } +} + +.title { + h3 { + margin: 0; + font-size: var(--periscope-font-size-medium); + font-weight: var(--font-weight-semibold); + } + + p { + margin: var(--spacing-2) 0 0; + color: var(--l3-foreground); + font-size: 12px; + } +} + +.footer { + display: flex; + align-items: center; + justify-content: space-between; + // Horizontal padding is provided by the drawer-footer slot var above. + padding: 0; + width: 100%; +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/ModelCostDrawer.tsx b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/ModelCostDrawer.tsx new file mode 100644 index 0000000000..8ce1a3a470 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/ModelCostDrawer.tsx @@ -0,0 +1,238 @@ +import { Button } from '@signozhq/ui/button'; +import { DrawerWrapper } from '@signozhq/ui/drawer'; +import { Input } from '@signozhq/ui/input'; +import { SelectSimple } from '@signozhq/ui/select'; +import { Typography } from '@signozhq/ui/typography'; +import { Controller, useForm } from 'react-hook-form'; + +import PatternEditor from './components/PatternEditor'; +import PricingFields from './components/PricingFields'; +import SourceSelector from './components/SourceSelector'; +import { PROVIDER_OPTIONS } from '../../../constants'; +import styles from './ModelCostDrawer.module.scss'; +import { + validateModelName, + validatePricing, + validateProvider, +} from '../../../utils'; +import type { DrawerDraft, DrawerMode } from '../../../types'; + +interface ModelCostDrawerProps { + isOpen: boolean; + mode: DrawerMode; + initialDraft: DrawerDraft; + onClose: () => void; + onSave: (draft: DrawerDraft) => void; + isSaving: boolean; + saveError: string | null; + canManage: boolean; +} + +function ModelCostDrawer({ + isOpen, + mode, + initialDraft, + onClose, + onSave, + isSaving, + saveError, + canManage, +}: ModelCostDrawerProps): JSX.Element { + // Default mode validates on submit, then re-validates on change — so we don't + // flag empty fields before the user has tried to save, but errors clear live + // once they start fixing them. + const { + control, + handleSubmit, + watch, + formState: { isDirty }, + } = useForm({ + defaultValues: initialDraft, + }); + + const isOverride = watch('isOverride'); + + // Metadata (model id / provider / patterns / source) is editable by any + // manager. Pricing fields are editable only once the user picks "User + // override" — auto-populated pricing is managed by SigNoz. Write APIs are + // Admin-only, so non-managers can't edit anything. + const metadataReadOnly = !canManage; + const pricingReadOnly = !canManage || !isOverride; + + // Non-managers can only view (write APIs are Admin-only), so the drawer is a + // read-only "View" rather than "Edit"/"Add". + let drawerTitle = 'Add model cost'; + if (!canManage) { + drawerTitle = 'View model cost'; + } else if (mode === 'edit') { + drawerTitle = 'Edit model cost'; + } + + const footer = ( +
+ + {canManage && ( + + )} +
+ ); + + return ( + { + if (!open) { + onClose(); + } + }} + direction="right" + width="base" + className={styles.modelCostDrawer} + footer={footer} + title={drawerTitle} + drawerHeaderProps={{ className: styles.title }} + > +
+ + validateModelName(value, mode), + }} + render={({ field, fieldState }): JSX.Element => ( + <> + field.onChange(e.target.value)} + testId="drawer-model-id-input" + /> + {fieldState.error && ( + + {fieldState.error.message} + + )} + + )} + /> +
+ +
+ + ( + <> + field.onChange(value as string)} + items={PROVIDER_OPTIONS} + disabled={mode === 'edit' || metadataReadOnly} + className={styles.fullWidth} + withPortal={false} + testId="drawer-provider-select" + /> + {fieldState.error && ( + + {fieldState.error.message} + + )} + + )} + /> +
+ + ( + + )} + /> + + {/* Source is auto vs. override — a choice only a manager can make, so + there's nothing to show a read-only viewer. */} + {canManage && ( + ( + + )} + /> + )} + + + validatePricing(value, values.isOverride), + }} + render={({ field, fieldState }): JSX.Element => ( + <> + field.onChange({ ...field.value, ...patch })} + /> + {fieldState.error && ( + + {fieldState.error.message} + + )} + + )} + /> + + {saveError && ( + + {saveError} + + )} +
+ ); +} + +export default ModelCostDrawer; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/ExtraPricingBuckets.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/ExtraPricingBuckets.module.scss new file mode 100644 index 0000000000..6185c0a806 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/ExtraPricingBuckets.module.scss @@ -0,0 +1,69 @@ +.drawerSection { + composes: drawerSection from '../../shared.module.scss'; +} + +.fullWidth { + width: 100%; +} + +.pricingField { + composes: pricingField from '../../shared.module.scss'; +} + +.cacheModeField { + margin-top: var(--spacing-5); +} + +.extraBucketsSection { + margin-top: var(--spacing-7); + gap: var(--spacing-5); +} + +.extraBucketsSectionHead { + display: flex; + align-items: center; + justify-content: space-between; +} + +.bucketRow { + display: flex; + align-items: center; + gap: var(--spacing-2); + + input { + flex: 1 auto auto; + min-width: 0; + } +} + +.bucketRowName { + flex: 0 0 110px; +} + +.bucketAddBtn { + width: 100%; +} + +.bucketPicker { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: var(--spacing-5); + padding: var(--spacing-6); + border-radius: 6px; + background: var(--l2-background); + border: 1px solid var(--l2-border); +} + +.bucketPickerTitle { + font-size: var(--periscope-font-size-small); + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--l3-foreground); +} + +.bucketPickerChips { + display: flex; + flex-wrap: wrap; + gap: var(--spacing-4); +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/ExtraPricingBuckets.tsx b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/ExtraPricingBuckets.tsx new file mode 100644 index 0000000000..6c3d43e483 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/ExtraPricingBuckets.tsx @@ -0,0 +1,179 @@ +import { useState } from 'react'; +import { Button } from '@signozhq/ui/button'; +import { Input } from '@signozhq/ui/input'; +import { SelectSimple } from '@signozhq/ui/select'; +import { Typography } from '@signozhq/ui/typography'; +import { Plus, Trash2 } from '@signozhq/icons'; +import { LlmpricingruletypesLLMPricingRuleCacheModeDTO as CacheModeDTO } from 'api/generated/services/sigNoz.schemas'; +import cx from 'classnames'; + +import { CACHE_BUCKETS, CACHE_MODE_OPTIONS } from '../../../../../constants'; +import styles from './ExtraPricingBuckets.module.scss'; +import { parsePricingAmount } from '../../../../../utils'; +import type { CacheBucketKey, DrawerDraft } from '../../../../../types'; +import { Tooltip } from 'antd'; + +type Pricing = DrawerDraft['pricing']; + +interface ExtraPricingBucketsProps { + pricing: Pricing; + isReadOnly: boolean; + onChange: (patch: Partial) => void; +} + +function ExtraPricingBuckets({ + pricing, + isReadOnly, + onChange, +}: ExtraPricingBucketsProps): JSX.Element { + const [isExtraPricingBucketOpen, setIsExtraPricingBucketOpen] = + useState(false); + + // Track which buckets are shown separately from their value, so a freshly + // added bucket can start blank (value null) instead of being seeded to 0. + // Seeded from buckets that already carry a value (edit mode). + const [addedKeys, setAddedKeys] = useState>( + () => + new Set( + CACHE_BUCKETS.filter((b) => pricing[b.key] !== null).map((b) => b.key), + ), + ); + + const addedBuckets = CACHE_BUCKETS.filter((b) => addedKeys.has(b.key)); + const availableBuckets = CACHE_BUCKETS.filter((b) => !addedKeys.has(b.key)); + const patchBucket = (key: CacheBucketKey, value: number | null): void => { + const patch: Partial = { [key]: value }; + onChange(patch); + }; + + const addBucket = (key: CacheBucketKey): void => { + // Leave the value null so the field renders blank until the user types. + setAddedKeys((prev) => new Set(prev).add(key)); + // Close the picker once nothing is left to add. + if (availableBuckets.length <= 1) { + setIsExtraPricingBucketOpen(false); + } + }; + + const removeBucket = (key: CacheBucketKey): void => { + setAddedKeys((prev) => { + const next = new Set(prev); + next.delete(key); + return next; + }); + patchBucket(key, null); + }; + + return ( +
+
+ + Extra pricing buckets + + + optional + +
+ + {addedBuckets.map((bucket) => ( +
+ + {bucket.label} + + + // Clearing the field is allowed — the row stays mounted because + // presence is tracked in `addedKeys`, not the value. Removal is + // explicit via the trash button. + patchBucket(bucket.key, parsePricingAmount(e.target.value)) + } + testId={`drawer-${bucket.testId}-cost`} + /> + + + 1M + + + + {!isReadOnly && ( +
+ ))} + + {addedBuckets.length > 0 && ( +
+ + onChange({ cacheMode: v as CacheModeDTO })} + disabled={isReadOnly} + className={styles.fullWidth} + withPortal={false} + testId="drawer-cache-mode" + /> +
+ )} + + {!isReadOnly && !isExtraPricingBucketOpen && availableBuckets.length > 0 && ( + + )} + + {!isReadOnly && isExtraPricingBucketOpen && ( +
+
Add a pricing bucket
+
+ {availableBuckets.map((bucket) => ( + + ))} +
+ +
+ )} +
+ ); +} + +export default ExtraPricingBuckets; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/index.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/index.ts new file mode 100644 index 0000000000..e69969dd47 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/ExtraPricingBuckets/index.ts @@ -0,0 +1 @@ +export { default } from './ExtraPricingBuckets'; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/PatternEditor.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/PatternEditor.module.scss new file mode 100644 index 0000000000..698dc1b55a --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/PatternEditor.module.scss @@ -0,0 +1,49 @@ +.drawerSection { + composes: drawerSection from '../../shared.module.scss'; +} + +.help { + composes: help from '../../shared.module.scss'; +} + +.patternBox { + display: flex; + flex-direction: column; + gap: var(--spacing-6); + padding: var(--spacing-6); + border-radius: 6px; + border: 1px solid var(--l2-border); +} + +.patternChips { + display: flex; + flex-wrap: wrap; + gap: var(--spacing-3); + min-height: 28px; +} + +.patternChip { + display: inline-flex; + align-items: center; + gap: var(--spacing-2); +} + +.patternChipRemove { + background: transparent; + border: none; + padding: 0; + margin-left: 2px; + cursor: pointer; + color: inherit; + display: inline-flex; + align-items: center; + + &:hover { + color: var(--accent-cherry); + } +} + +.patternAdd { + display: flex; + gap: var(--spacing-3); +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/PatternEditor.tsx b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/PatternEditor.tsx new file mode 100644 index 0000000000..8a866a91f5 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/PatternEditor.tsx @@ -0,0 +1,102 @@ +import { useState } from 'react'; +import { Badge } from '@signozhq/ui/badge'; +import { Button } from '@signozhq/ui/button'; +import { Input } from '@signozhq/ui/input'; +import { Typography } from '@signozhq/ui/typography'; +import { X } from '@signozhq/icons'; + +import styles from './PatternEditor.module.scss'; + +interface PatternEditorProps { + patterns: string[]; + isReadOnly: boolean; + onChange: (patterns: string[]) => void; +} + +// Model-name prefix patterns as removable chips + an add input. +function PatternEditor({ + patterns, + isReadOnly, + onChange, +}: PatternEditorProps): JSX.Element { + const [patternInput, setPatternInput] = useState(''); + + const addPattern = (): void => { + const next = patternInput.trim(); + if (!next || patterns.includes(next)) { + setPatternInput(''); + return; + } + onChange([...patterns, next]); + setPatternInput(''); + }; + + const removePattern = (pattern: string): void => { + onChange(patterns.filter((p) => p !== pattern)); + }; + + return ( +
+ + Model name patterns{' '} + + (prefix match) + + +
+
+ {patterns.map((pattern) => ( + + {pattern}* + {!isReadOnly && ( + + )} + + ))} +
+ {!isReadOnly && ( +
+ setPatternInput(e.target.value)} + onKeyDown={(e): void => { + if (e.key === 'Enter') { + e.preventDefault(); + addPattern(); + } + }} + testId="drawer-pattern-input" + /> + +
+ )} +
+ + Each pattern uses prefix matching against{' '} + gen_ai.request.model. + +
+ ); +} + +export default PatternEditor; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/index.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/index.ts new file mode 100644 index 0000000000..e9963976e7 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PatternEditor/index.ts @@ -0,0 +1 @@ +export { default } from './PatternEditor'; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/PricingFields.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/PricingFields.module.scss new file mode 100644 index 0000000000..2036d467dc --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/PricingFields.module.scss @@ -0,0 +1,31 @@ +.drawerSection { + composes: drawerSection from '../../shared.module.scss'; +} + +.drawerSurface { + composes: drawerSurface from '../../shared.module.scss'; +} + +.drawerSurfaceHead { + composes: drawerSurfaceHead from '../../shared.module.scss'; +} + +.managedLabel { + display: flex; + align-items: center; + gap: var(--spacing-2); +} + +.pricingField { + composes: pricingField from '../../shared.module.scss'; +} + +.required { + composes: required from '../../shared.module.scss'; +} + +.pricingGrid { + display: grid; + grid-template-columns: 1fr 1fr; + gap: var(--spacing-6); +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/PricingFields.tsx b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/PricingFields.tsx new file mode 100644 index 0000000000..457b352d9d --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/PricingFields.tsx @@ -0,0 +1,91 @@ +import { Input } from '@signozhq/ui/input'; +import { Lock } from '@signozhq/icons'; +import cx from 'classnames'; + +import ExtraPricingBuckets from '../ExtraPricingBuckets'; +import styles from './PricingFields.module.scss'; +import { parsePricingAmount } from '../../../../../utils'; +import type { DrawerDraft } from '../../../../../types'; +import { Typography } from '@signozhq/ui/typography'; + +type Pricing = DrawerDraft['pricing']; + +interface PricingFieldsProps { + pricing: Pricing; + isReadOnly: boolean; + onChange: (patch: Partial) => void; +} + +function PricingFields({ + pricing, + isReadOnly, + onChange, +}: PricingFieldsProps): JSX.Element { + return ( +
+
+ + Pricing (per 1M tokens, USD) + + + {isReadOnly && ( + + + + Read-only + + )} +
+
+
+ + + onChange({ input: parsePricingAmount(e.target.value) }) + } + testId="drawer-input-cost" + /> +
+
+ + + onChange({ output: parsePricingAmount(e.target.value) }) + } + testId="drawer-output-cost" + /> +
+
+ + +
+ ); +} + +export default PricingFields; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/index.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/index.ts new file mode 100644 index 0000000000..a0c95bb57c --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/PricingFields/index.ts @@ -0,0 +1 @@ +export { default } from './PricingFields'; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/SourceSelector/SourceSelector.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/SourceSelector/SourceSelector.module.scss new file mode 100644 index 0000000000..2fc083b8ff --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/SourceSelector/SourceSelector.module.scss @@ -0,0 +1,115 @@ +.drawerSection { + composes: drawerSection from '../../shared.module.scss'; +} + +.drawerSurface { + composes: drawerSurface from '../../shared.module.scss'; +} + +.drawerSurfaceHead { + composes: drawerSurfaceHead from '../../shared.module.scss'; +} + +.managedLabel { + composes: managedLabel from '../../shared.module.scss'; +} + +.sourceRadioGroup { + --radio-group-item-border-color: var(--l2-border); + + display: flex; + flex-direction: column; + gap: var(--spacing-4); + width: 100%; + .sourceRadio { + display: flex; + flex-direction: row; + align-items: flex-start; + justify-content: flex-start; + gap: var(--spacing-5); + padding: var(--spacing-5) var(--spacing-6); + border-radius: var(--radius-2); + border: 1px solid transparent; + background: var(--l3-background); + margin: 0; + width: 100%; + // Include padding + border in the 100% width so the card fits inside + // the SOURCE surface instead of overflowing its right edge. + box-sizing: border-box; + cursor: pointer; + transition: + background-color 0.12s ease, + border-color 0.12s ease; + + // The radio button itself: keep it fixed-size and aligned with the title + // baseline (margin-top compensates for align-items: flex-start vs the + // title's line-box). + > button[role='radio'] { + flex: 0 0 16px; + width: 16px; + height: 16px; + margin-top: 3px; + } + + // The library wraps children in a
+ + )} + + ); +} + +export default SourceSelector; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/SourceSelector/index.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/SourceSelector/index.ts new file mode 100644 index 0000000000..18a14709d7 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/components/SourceSelector/index.ts @@ -0,0 +1 @@ +export { default } from './SourceSelector'; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/hooks/useModelCostDrawer.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/hooks/useModelCostDrawer.ts new file mode 100644 index 0000000000..b00b9da55f --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/hooks/useModelCostDrawer.ts @@ -0,0 +1,100 @@ +import { useCallback, useState } from 'react'; +import { toast } from '@signozhq/ui/sonner'; +import { useQueryClient } from 'react-query'; +import { + getListLLMPricingRulesQueryKey, + useCreateOrUpdateLLMPricingRules, +} from 'api/generated/services/llmpricingrules'; + +import { EMPTY_DRAFT } from '../../../../constants'; +import type { DrawerDraft, DrawerMode, PricingRule } from '../../../../types'; +import { buildRulePayload, draftFromRule } from '../../../../utils'; + +interface UseModelCostDrawerResult { + isOpen: boolean; + mode: DrawerMode; + initialDraft: DrawerDraft; + openForAdd: (prefillModelName?: string) => void; + openForEdit: (rule: PricingRule) => void; + close: () => void; + save: (draft: DrawerDraft) => Promise; + isSaving: boolean; + saveError: string | null; + selectedRuleId: string | null; +} + +export function useModelCostDrawer(): UseModelCostDrawerResult { + const queryClient = useQueryClient(); + const [isOpen, setIsOpen] = useState(false); + const [mode, setMode] = useState('add'); + const [initialDraft, setInitialDraft] = useState(EMPTY_DRAFT); + const [selectedRuleId, setSelectedRuleId] = useState(null); + const [saveError, setSaveError] = useState(null); + + const { mutateAsync: createOrUpdate, isLoading: isSaving } = + useCreateOrUpdateLLMPricingRules(); + + const invalidateList = useCallback(async (): Promise => { + await queryClient.invalidateQueries({ + queryKey: getListLLMPricingRulesQueryKey(), + }); + }, [queryClient]); + + const openForAdd = useCallback((): void => { + setMode('add'); + setInitialDraft({ + ...EMPTY_DRAFT, + modelName: '', + patterns: [], + }); + setSelectedRuleId(null); + setSaveError(null); + setIsOpen(true); + }, []); + + const openForEdit = useCallback((rule: PricingRule): void => { + setMode('edit'); + setInitialDraft(draftFromRule(rule)); + setSelectedRuleId(rule.id); + setSaveError(null); + setIsOpen(true); + }, []); + + const close = useCallback((): void => { + setIsOpen(false); + setSelectedRuleId(null); + setSaveError(null); + }, []); + + const save = useCallback( + async (draft: DrawerDraft): Promise => { + setSaveError(null); + try { + await createOrUpdate({ + data: { rules: [buildRulePayload(draft)] }, + }); + await invalidateList(); + setIsOpen(false); + setSelectedRuleId(null); + toast.success(mode === 'edit' ? 'Model cost updated' : 'Model cost added'); + } catch (error) { + const message = error instanceof Error ? error.message : 'Save failed'; + setSaveError(message); + } + }, + [createOrUpdate, invalidateList, mode], + ); + + return { + isOpen, + mode, + initialDraft, + openForAdd, + openForEdit, + close, + save, + isSaving, + saveError, + selectedRuleId, + }; +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/index.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/index.ts new file mode 100644 index 0000000000..807dfe3e95 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/index.ts @@ -0,0 +1,2 @@ +export { default } from './ModelCostDrawer'; +export { useModelCostDrawer } from './hooks/useModelCostDrawer'; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/shared.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/shared.module.scss new file mode 100644 index 0000000000..51677e14e9 --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostDrawer/shared.module.scss @@ -0,0 +1,59 @@ +/* Shared drawer selectors used by 2+ of the model-cost drawer components. */ +/* Components pull these in via CSS-modules `composes` from their own module so */ +/* the authored class names in the TSX stay identical. */ +/* NOTE: this file is a `composes` target, so it is parsed as plain CSS (no SCSS */ +/* preprocessing). Keep it flat — no nesting, no slash-slash comments. */ + +.drawerSection { + display: flex; + flex-direction: column; + gap: var(--spacing-3); +} + +.drawerSection .help, +.help { + margin: 0; +} + +.help code { + padding: 1px var(--spacing-2); + border-radius: 3px; + background: var(--l3-background); + font-size: 10px; +} + +.drawerSurface { + padding: var(--spacing-7); + border-radius: 6px; + background: var(--l2-background); + border: 1px solid var(--l2-border); +} + +.drawerSurfaceHead { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: var(--spacing-5); +} + +.managedLabel { + display: inline-flex; + align-items: center; + gap: var(--spacing-2); + font-size: var(--periscope-font-size-small); + color: var(--l3-foreground); +} + +.required { + color: var(--accent-cherry); +} + +.pricingField { + display: flex; + flex-direction: column; + gap: var(--spacing-2); +} + +.pricingField input { + width: 100%; +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostsTable/ModelCostsTable.module.scss b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostsTable/ModelCostsTable.module.scss index 3e29c73043..42b40a1ed5 100644 --- a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostsTable/ModelCostsTable.module.scss +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/components/ModelCostsTable/ModelCostsTable.module.scss @@ -15,6 +15,6 @@ justify-content: center; margin-top: var(--spacing-8); min-height: 400px; - color: var(--text-vanilla-400); + color: var(--l3-foreground); font-size: var(--periscope-font-size-base); } diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/hooks/useModelCostDelete.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/hooks/useModelCostDelete.ts new file mode 100644 index 0000000000..b8de9467bc --- /dev/null +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/ModelCostTabPanel/hooks/useModelCostDelete.ts @@ -0,0 +1,66 @@ +import { useCallback, useState } from 'react'; +import { toast } from '@signozhq/ui/sonner'; +import { useQueryClient } from 'react-query'; +import { + getListLLMPricingRulesQueryKey, + useDeleteLLMPricingRule, +} from 'api/generated/services/llmpricingrules'; + +import type { PricingRule } from '../../types'; + +// The minimal slice of a rule the delete-confirm flow needs: the id to delete +// and the model name to show in the confirmation copy. +type PendingDelete = Pick; + +interface UseModelCostDeleteResult { + requestDelete: (rule: PendingDelete) => void; + confirmDelete: () => Promise; + cancelDelete: () => void; + pendingDelete: PendingDelete | null; + isDeleting: boolean; +} + +// Owns the confirm-then-delete flow for a pricing rule, independent of the +// add/edit drawer — delete is triggered from the table row menu, so this state +// lives at the panel level rather than inside useModelCostDrawer. +export function useModelCostDelete(): UseModelCostDeleteResult { + const queryClient = useQueryClient(); + // The rule queued for deletion. Non-null drives the confirm dialog open. + const [pendingDelete, setPendingDelete] = useState(null); + + const { mutateAsync: deleteRuleApi, isLoading: isDeleting } = + useDeleteLLMPricingRule(); + + const requestDelete = useCallback((rule: PendingDelete): void => { + setPendingDelete({ id: rule.id, modelName: rule.modelName }); + }, []); + + const cancelDelete = useCallback((): void => { + setPendingDelete(null); + }, []); + + const confirmDelete = useCallback(async (): Promise => { + if (!pendingDelete) { + return; + } + try { + await deleteRuleApi({ pathParams: { id: pendingDelete.id } }); + await queryClient.invalidateQueries({ + queryKey: getListLLMPricingRulesQueryKey(), + }); + setPendingDelete(null); + toast.success('Model cost deleted'); + } catch (error) { + const message = error instanceof Error ? error.message : 'Delete failed'; + toast.error(message); + } + }, [deleteRuleApi, pendingDelete, queryClient]); + + return { + requestDelete, + confirmDelete, + cancelDelete, + pendingDelete, + isDeleting, + }; +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/constants.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/constants.ts index f8eba93c1b..2511692db5 100644 --- a/frontend/src/container/LLMObservability/Settings/ModelPricing/constants.ts +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/constants.ts @@ -1,6 +1,68 @@ +import { LlmpricingruletypesLLMPricingRuleCacheModeDTO as CacheModeDTO } from 'api/generated/services/sigNoz.schemas'; + +import type { CacheBucketDef, DrawerDraft } from './types'; + export const PAGE_SIZE = 20; export const PAGE_KEY = 'page'; export const LIMIT_KEY = 'limit'; +export const SEARCH_KEY = 'search'; +export const SEARCH_DEBOUNCE_MS = 300; +export const SOURCE_KEY = 'source'; +export type SourceFilter = 'all' | 'override' | 'auto'; +export const SOURCE_FILTER_OPTIONS: { value: SourceFilter; label: string }[] = [ + { value: 'all', label: 'All sources' }, + { value: 'override', label: 'User override' }, + { value: 'auto', label: 'Auto' }, +]; + +export const SOURCE_FILTER_TO_IS_OVERRIDE: Record< + SourceFilter, + boolean | undefined +> = { + all: undefined, + override: true, + auto: false, +}; + +// Match the page size so the skeleton reserves the same number of rows the +// loaded page renders — otherwise the table height jumps on load. export const SKELETON_ROW_COUNT = PAGE_SIZE; + +export const PROVIDER_OPTIONS = [ + { value: 'OpenAI', label: 'OpenAI' }, + { value: 'Anthropic', label: 'Anthropic' }, + { value: 'Azure OpenAI', label: 'Azure OpenAI' }, + { value: 'Google', label: 'Google' }, + { value: 'Self-hosted', label: 'Self-hosted' }, + { value: 'Other', label: 'Other' }, +]; + +export const CACHE_MODE_OPTIONS = [ + { value: CacheModeDTO.subtract, label: 'Subtract (OpenAI style)' }, + { value: CacheModeDTO.additive, label: 'Additive (Anthropic style)' }, + // https://app.notion.com/p/signoz/LLM-Tokens-Cost-Calculation-330fcc6bcd19805283ccc841d596358e?source=copy_link#33efcc6bcd1980e6a187e442c6ba5996 + { value: CacheModeDTO.unknown, label: 'Unknown' }, +]; + +export const CACHE_BUCKETS: CacheBucketDef[] = [ + { key: 'cacheRead', label: 'cache_read', testId: 'cache-read' }, + { key: 'cacheWrite', label: 'cache_write', testId: 'cache-write' }, +]; + +export const EMPTY_DRAFT: DrawerDraft = { + id: null, + sourceId: null, + modelName: '', + provider: 'OpenAI', + patterns: [], + isOverride: true, + pricing: { + input: null, + output: null, + cacheMode: CacheModeDTO.unknown, + cacheRead: null, + cacheWrite: null, + }, +}; diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/types.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/types.ts index 05d96f7248..dfac6f2eee 100644 --- a/frontend/src/container/LLMObservability/Settings/ModelPricing/types.ts +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/types.ts @@ -1,4 +1,39 @@ +import { + LlmpricingruletypesLLMPricingRuleCacheModeDTO as CacheModeDTO, + type LlmpricingruletypesLLMPricingRuleDTO, +} from 'api/generated/services/sigNoz.schemas'; + +export type PricingRule = LlmpricingruletypesLLMPricingRuleDTO; + export interface ExtraBucket { key: string; pricePerMillion: number; } + +export type DrawerMode = 'add' | 'edit'; + +// Optional pricing buckets the user can add/remove. Keyed by the matching +// DrawerDraft['pricing'] field. +export type CacheBucketKey = 'cacheRead' | 'cacheWrite'; + +export interface CacheBucketDef { + key: CacheBucketKey; + label: string; + testId: string; +} + +export interface DrawerDraft { + id: string | null; + sourceId: string | null; + modelName: string; + provider: string; + patterns: string[]; + isOverride: boolean; + pricing: { + input: number | null; + output: number | null; + cacheMode: CacheModeDTO; + cacheRead: number | null; + cacheWrite: number | null; + }; +} diff --git a/frontend/src/container/LLMObservability/Settings/ModelPricing/utils.ts b/frontend/src/container/LLMObservability/Settings/ModelPricing/utils.ts index a7c9e9831c..16083238f8 100644 --- a/frontend/src/container/LLMObservability/Settings/ModelPricing/utils.ts +++ b/frontend/src/container/LLMObservability/Settings/ModelPricing/utils.ts @@ -1,8 +1,19 @@ +import { + LlmpricingruletypesLLMPricingRuleCacheModeDTO as CacheModeDTO, + LlmpricingruletypesLLMPricingRuleUnitDTO as UnitDTO, + type LlmpricingruletypesLLMPricingCacheCostsDTO, + type LlmpricingruletypesLLMRulePricingDTO, + type LlmpricingruletypesUpdatableLLMPricingRuleDTO, +} from 'api/generated/services/sigNoz.schemas'; import dayjs from 'dayjs'; import relativeTime from 'dayjs/plugin/relativeTime'; -import type { ExtraBucket } from './types'; -import type { LlmpricingruletypesLLMPricingRuleDTO } from 'api/generated/services/sigNoz.schemas'; +import type { + DrawerDraft, + DrawerMode, + ExtraBucket, + PricingRule, +} from './types'; dayjs.extend(relativeTime); @@ -13,6 +24,19 @@ const getRelativeTime = ( return parsed?.isValid() ? parsed.fromNow() : '—'; }; +const hasCacheValue = (value: number | null | undefined): value is number => + typeof value === 'number' && value > 0; + +// ─── Input helpers ─────────────────────────────────────────────────────────── + +export const parsePricingAmount = (raw: string): number | null => { + if (raw.trim() === '') { + return null; + } + const value = Number(raw); + return Number.isFinite(value) ? value : 0; +}; + // ─── Display helpers ───────────────────────────────────────────────────────── export const formatPricePerMillion = (value: number | undefined): string => { @@ -23,38 +47,117 @@ export const formatPricePerMillion = (value: number | undefined): string => { return `$${value.toFixed(2)}`; }; -export const getExtraBuckets = ( - rule: LlmpricingruletypesLLMPricingRuleDTO, -): ExtraBucket[] => { +export const getExtraBuckets = (rule: PricingRule): ExtraBucket[] => { const cache = rule.pricing?.cache; if (!cache) { return []; } const buckets: ExtraBucket[] = []; - if (typeof cache.read === 'number' && cache.read > 0) { + if (hasCacheValue(cache.read)) { buckets.push({ key: 'cache_read', pricePerMillion: cache.read }); } - if (typeof cache.write === 'number' && cache.write > 0) { + if (hasCacheValue(cache.write)) { buckets.push({ key: 'cache_write', pricePerMillion: cache.write }); } return buckets; }; -export const getSourceLabel = ( - rule: LlmpricingruletypesLLMPricingRuleDTO, -): 'Auto' | 'User override' => (rule.isOverride ? 'User override' : 'Auto'); +export const getSourceLabel = (rule: PricingRule): 'Auto' | 'User override' => + rule.isOverride ? 'User override' : 'Auto'; -export const getRelativeLastSeen = ( - rule: LlmpricingruletypesLLMPricingRuleDTO, -): string => getRelativeTime(rule.updatedAt || rule.syncedAt || rule.createdAt); +export const getRelativeLastSeen = (rule: PricingRule): string => + getRelativeTime(rule.updatedAt || rule.syncedAt || rule.createdAt); // Canonical id shown under the model name, e.g. "openai:gpt-4o". Both segments // are lower-cased so the id is consistently normalised (providers/models can // arrive with mixed casing). -export const getCanonicalId = ( - rule: LlmpricingruletypesLLMPricingRuleDTO, -): string => { +export const getCanonicalId = (rule: PricingRule): string => { const provider = rule.provider?.trim().toLowerCase() || 'unknown'; const model = rule.modelName?.trim().toLowerCase() || 'unknown'; return `${provider}:${model}`; }; + +// ─── Drawer draft <-> API helpers ──────────────────────────────────────────── + +export const draftFromRule = (rule: PricingRule): DrawerDraft => ({ + id: rule.id, + sourceId: rule.sourceId ?? null, + modelName: rule.modelName, + provider: rule.provider, + patterns: rule.modelPattern || [], + isOverride: !!rule.isOverride, + pricing: { + input: rule.pricing?.input ?? 0, + output: rule.pricing?.output ?? 0, + cacheMode: rule.pricing?.cache?.mode ?? CacheModeDTO.unknown, + cacheRead: rule.pricing?.cache?.read ?? null, + cacheWrite: rule.pricing?.cache?.write ?? null, + }, +}); + +const buildCacheCosts = ( + pricing: DrawerDraft['pricing'], +): LlmpricingruletypesLLMPricingCacheCostsDTO | undefined => { + const { cacheMode, cacheRead, cacheWrite } = pricing; + if (!hasCacheValue(cacheRead) && !hasCacheValue(cacheWrite)) { + return undefined; + } + return { + mode: cacheMode, + ...(hasCacheValue(cacheRead) && { read: cacheRead }), + ...(hasCacheValue(cacheWrite) && { write: cacheWrite }), + }; +}; + +export const buildPricingPayload = ( + draft: DrawerDraft, +): LlmpricingruletypesLLMRulePricingDTO => { + const cache = buildCacheCosts(draft.pricing); + return { + input: draft.pricing.input ?? 0, + output: draft.pricing.output ?? 0, + ...(cache && { cache }), + }; +}; + +export const buildRulePayload = ( + draft: DrawerDraft, +): LlmpricingruletypesUpdatableLLMPricingRuleDTO => ({ + id: draft.id || undefined, + sourceId: draft.sourceId || undefined, + modelName: draft.modelName.trim(), + provider: draft.provider.trim(), + modelPattern: draft.patterns, + isOverride: draft.isOverride, + enabled: true, + unit: UnitDTO.per_million_tokens, + pricing: buildPricingPayload(draft), +}); + +export const validateModelName = ( + modelName: string, + mode: DrawerMode, +): true | string => + mode === 'add' && !modelName.trim() ? 'Billing model ID is required.' : true; + +export const validateProvider = (provider: string): true | string => + provider.trim() ? true : 'Provider is required.'; + +export const validatePricing = ( + pricing: DrawerDraft['pricing'], + isOverride: boolean, +): true | string => { + if (!isOverride) { + return true; + } + if (pricing.input === null || pricing.input <= 0) { + return 'Input cost must be greater than 0.'; + } + if (pricing.output === null || pricing.output <= 0) { + return 'Output cost must be greater than 0.'; + } + if ((pricing.cacheRead ?? 0) < 0 || (pricing.cacheWrite ?? 0) < 0) { + return 'Cache costs must be non-negative.'; + } + return true; +}; diff --git a/frontend/src/container/MembersSettings/MembersSettings.styles.scss b/frontend/src/container/MembersSettings/MembersSettings.styles.scss index 96979c6191..453f0dfe75 100644 --- a/frontend/src/container/MembersSettings/MembersSettings.styles.scss +++ b/frontend/src/container/MembersSettings/MembersSettings.styles.scss @@ -1,9 +1,14 @@ +@use '../../styles/scrollbar' as *; + .members-settings-page { display: flex; flex-direction: column; gap: var(--spacing-8); padding: var(--padding-4) var(--padding-2) var(--padding-6) var(--padding-4); height: 100%; + overflow-y: auto; + + @include custom-scrollbar; } .members-settings { diff --git a/frontend/src/container/MembersSettings/MembersSettings.tsx b/frontend/src/container/MembersSettings/MembersSettings.tsx index 4847ecd409..b27ca1114b 100644 --- a/frontend/src/container/MembersSettings/MembersSettings.tsx +++ b/frontend/src/container/MembersSettings/MembersSettings.tsx @@ -6,7 +6,7 @@ import { DropdownMenuSimple, type MenuItem } from '@signozhq/ui/dropdown-menu'; import { Input } from '@signozhq/ui/input'; import { useListUsers } from 'api/generated/services/users'; import EditMemberDrawer from 'components/EditMemberDrawer/EditMemberDrawer'; -import InviteMembersModal from 'components/InviteMembersModal/InviteMembersModal'; +import InviteMembersModal from 'container/MembersSettings/components/InviteMembersModal/InviteMembersModal'; import MembersTable, { MemberRow } from 'components/MembersTable/MembersTable'; import useUrlQuery from 'hooks/useUrlQuery'; import { parseAsBoolean, useQueryState } from 'nuqs'; diff --git a/frontend/src/container/MembersSettings/__tests__/MembersSettings.integration.test.tsx b/frontend/src/container/MembersSettings/__tests__/MembersSettings.integration.test.tsx index 7f0b16b309..a684f8ad00 100644 --- a/frontend/src/container/MembersSettings/__tests__/MembersSettings.integration.test.tsx +++ b/frontend/src/container/MembersSettings/__tests__/MembersSettings.integration.test.tsx @@ -110,7 +110,9 @@ describe('MembersSettings (integration)', () => { fireEvent.click(await screen.findByText('Alice Smith')); - await screen.findByText('Member Details'); + await expect( + screen.findByText('Member Details'), + ).resolves.toBeInTheDocument(); }); it('opens EditMemberDrawer when a deleted member row is clicked', async () => { @@ -127,7 +129,7 @@ describe('MembersSettings (integration)', () => { fireEvent.click(screen.getByRole('button', { name: /invite member/i })); await expect( - screen.findAllByPlaceholderText('john@signoz.io'), + screen.findAllByPlaceholderText('e.g. john@signoz.io'), ).resolves.toHaveLength(3); }); @@ -137,7 +139,7 @@ describe('MembersSettings (integration)', () => { }); await expect( - screen.findAllByPlaceholderText('john@signoz.io'), + screen.findAllByPlaceholderText('e.g. john@signoz.io'), ).resolves.toHaveLength(3); }); }); diff --git a/frontend/src/container/MembersSettings/components/InviteMembersModal/InviteMembersModal.styles.scss b/frontend/src/container/MembersSettings/components/InviteMembersModal/InviteMembersModal.styles.scss new file mode 100644 index 0000000000..7e93d7be0c --- /dev/null +++ b/frontend/src/container/MembersSettings/components/InviteMembersModal/InviteMembersModal.styles.scss @@ -0,0 +1,38 @@ +.invite-members-modal { + max-width: 700px; + background: var(--popover); + border: 1px solid var(--secondary); + border-radius: 4px; + box-shadow: 0 4px 9px 0 rgba(0, 0, 0, 0.04); + + [data-slot='dialog-header'] { + padding: var(--padding-4); + border-bottom: 1px solid var(--secondary); + flex-shrink: 0; + background: transparent; + margin: 0; + } + + [data-slot='dialog-title'] { + font-family: Inter, sans-serif; + font-size: var(--label-base-400-font-size); + font-weight: var(--label-base-400-font-weight); + line-height: var(--label-base-400-line-height); + letter-spacing: -0.065px; + color: var(--l1-foreground); + margin: 0; + } + + [data-slot='dialog-description'] { + padding: var(--padding-4); + } +} + +.invite-members-modal__footer { + padding-top: var(--padding-4); + border-top: 1px solid var(--l1-border); + display: flex; + align-items: center; + justify-content: flex-end; + gap: var(--spacing-6); +} diff --git a/frontend/src/container/MembersSettings/components/InviteMembersModal/InviteMembersModal.tsx b/frontend/src/container/MembersSettings/components/InviteMembersModal/InviteMembersModal.tsx new file mode 100644 index 0000000000..f0e45e2c69 --- /dev/null +++ b/frontend/src/container/MembersSettings/components/InviteMembersModal/InviteMembersModal.tsx @@ -0,0 +1,71 @@ +import { useCallback } from 'react'; +import { X } from '@signozhq/icons'; +import { Button } from '@signozhq/ui/button'; +import { DialogWrapper } from '@signozhq/ui/dialog'; +import { toast } from '@signozhq/ui/sonner'; +import InviteMembers from 'components/InviteMembers/InviteMembers'; + +import './InviteMembersModal.styles.scss'; + +export interface InviteMembersModalProps { + open: boolean; + onClose: () => void; + onComplete?: () => void; +} + +function InviteMembersModal({ + open, + onClose, + onComplete, +}: InviteMembersModalProps): JSX.Element { + const handleSuccess = useCallback((): void => { + toast.success('Invites sent successfully', { position: 'top-right' }); + onClose(); + onComplete?.(); + }, [onClose, onComplete]); + + const handlePartialSuccess = useCallback((): void => { + toast.warning('Some invites failed', { position: 'top-right' }); + onComplete?.(); + }, [onComplete]); + + return ( + { + if (!isOpen) { + onClose(); + } + }} + showCloseButton + width="wide" + className="invite-members-modal" + > + ( +
+ + + +
+ )} + /> +
+ ); +} + +export default InviteMembersModal; diff --git a/frontend/src/container/MembersSettings/components/InviteMembersModal/__tests__/InviteMembersModal.test.tsx b/frontend/src/container/MembersSettings/components/InviteMembersModal/__tests__/InviteMembersModal.test.tsx new file mode 100644 index 0000000000..9a0d596757 --- /dev/null +++ b/frontend/src/container/MembersSettings/components/InviteMembersModal/__tests__/InviteMembersModal.test.tsx @@ -0,0 +1,210 @@ +import { toast } from '@signozhq/ui/sonner'; +import { render, screen, userEvent } from 'tests/test-utils'; + +import InviteMembersModal from '../InviteMembersModal'; + +jest.mock('@signozhq/ui/sonner', () => ({ + ...jest.requireActual('@signozhq/ui/sonner'), + toast: { + success: jest.fn(), + warning: jest.fn(), + }, +})); + +interface MockInviteMembersProps { + onSuccess: () => void; + onPartialSuccess: () => void; + onAllFailed?: () => void; + renderFooter: (props: { + submit: () => void; + canSubmit: boolean; + isSubmitting: boolean; + }) => JSX.Element; +} + +let mockInviteMembersProps: MockInviteMembersProps | null = null; + +jest.mock('components/InviteMembers/InviteMembers', () => { + return function MockInviteMembers(props: MockInviteMembersProps): JSX.Element { + mockInviteMembersProps = props; + return ( +
+ {props.renderFooter({ + submit: jest.fn(), + canSubmit: true, + isSubmitting: false, + })} +
+ ); + }; +}); + +const defaultProps = { + open: true, + onClose: jest.fn(), + onComplete: jest.fn(), +}; + +function renderComponent( + props: Partial = {}, +): ReturnType { + return render(); +} + +describe('InviteMembersModal', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockInviteMembersProps = null; + }); + + describe('rendering', () => { + it('renders modal with title and InviteMembers component', () => { + renderComponent(); + + expect(screen.getByRole('dialog')).toBeInTheDocument(); + expect(screen.getByRole('heading', { level: 2 })).toHaveTextContent( + 'Invite Team Members', + ); + expect(screen.getByTestId('mock-invite-members')).toBeInTheDocument(); + }); + + it('does not render when open=false', () => { + renderComponent({ open: false }); + + expect(screen.queryByText('Invite Team Members')).not.toBeInTheDocument(); + }); + }); + + describe('footer buttons', () => { + it('renders Cancel and Invite buttons', () => { + renderComponent(); + + expect(screen.getByRole('button', { name: /cancel/i })).toBeInTheDocument(); + expect( + screen.getByRole('button', { name: /invite team members/i }), + ).toBeInTheDocument(); + }); + + it('disables Invite button when canSubmit=false', () => { + const { unmount } = renderComponent(); + unmount(); + + const { getByRole } = render( + mockInviteMembersProps?.renderFooter({ + submit: jest.fn(), + canSubmit: false, + isSubmitting: false, + }) as JSX.Element, + ); + + expect(getByRole('button', { name: /invite team members/i })).toBeDisabled(); + }); + + it('shows loading state when isSubmitting=true', () => { + const { unmount } = renderComponent(); + unmount(); + + const { getByRole } = render( + mockInviteMembersProps?.renderFooter({ + submit: jest.fn(), + canSubmit: true, + isSubmitting: true, + }) as JSX.Element, + ); + + expect(getByRole('button', { name: /inviting/i })).toBeInTheDocument(); + }); + + it('calls onClose when Cancel is clicked', async () => { + const user = userEvent.setup(); + const onClose = jest.fn(); + renderComponent({ onClose }); + + await user.click(screen.getByRole('button', { name: /cancel/i })); + + expect(onClose).toHaveBeenCalledTimes(1); + }); + + it('calls submit when Invite button is clicked', async () => { + const user = userEvent.setup(); + const mockSubmit = jest.fn(); + + const { unmount } = renderComponent(); + unmount(); + + const { getByRole } = render( + mockInviteMembersProps?.renderFooter({ + submit: mockSubmit, + canSubmit: true, + isSubmitting: false, + }) as JSX.Element, + ); + + await user.click(getByRole('button', { name: /invite team members/i })); + + expect(mockSubmit).toHaveBeenCalledTimes(1); + }); + }); + + describe('handleSuccess callback', () => { + it('shows success toast, calls onClose and onComplete', () => { + const onClose = jest.fn(); + const onComplete = jest.fn(); + renderComponent({ onClose, onComplete }); + + mockInviteMembersProps?.onSuccess(); + + expect(toast.success).toHaveBeenCalledWith('Invites sent successfully', { + position: 'top-right', + }); + expect(onClose).toHaveBeenCalledTimes(1); + expect(onComplete).toHaveBeenCalledTimes(1); + }); + + it('works without onComplete prop', () => { + const onClose = jest.fn(); + renderComponent({ onClose, onComplete: undefined }); + + mockInviteMembersProps?.onSuccess(); + + expect(toast.success).toHaveBeenCalled(); + expect(onClose).toHaveBeenCalledTimes(1); + }); + }); + + describe('handlePartialSuccess callback', () => { + it('shows warning toast and calls onComplete', () => { + const onComplete = jest.fn(); + renderComponent({ onComplete }); + + mockInviteMembersProps?.onPartialSuccess(); + + expect(toast.warning).toHaveBeenCalledWith('Some invites failed', { + position: 'top-right', + }); + expect(onComplete).toHaveBeenCalledTimes(1); + }); + + it('does not call onClose on partial success', () => { + const onClose = jest.fn(); + renderComponent({ onClose }); + + mockInviteMembersProps?.onPartialSuccess(); + + expect(onClose).not.toHaveBeenCalled(); + }); + }); + + describe('dialog close behavior', () => { + it('calls onClose when dialog is closed via close button', async () => { + const user = userEvent.setup(); + const onClose = jest.fn(); + renderComponent({ onClose }); + + const closeButton = screen.getByRole('button', { name: /close/i }); + await user.click(closeButton); + + expect(onClose).toHaveBeenCalled(); + }); + }); +}); diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlBadge.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlBadge.tsx index e22be73c05..fe65d39be3 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlBadge.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlBadge.tsx @@ -1,4 +1,5 @@ import { Gauge } from '@signozhq/icons'; +import { Tooltip } from 'antd'; import { MetricreductionruletypesGettableReductionRuleDTO } from 'api/generated/services/sigNoz.schemas'; import { Badge } from '@signozhq/ui/badge'; @@ -8,16 +9,26 @@ interface VolumeControlBadgeProps { } function VolumeControlBadge({ rule }: VolumeControlBadgeProps): JSX.Element { - return ( + const badge = ( - {!rule.active ? 'Active' : 'Pending'} + {rule.active ? 'Active' : 'Pending'} ); + + if (rule.active) { + return badge; + } + + return ( + + {badge} + + ); } export default VolumeControlBadge; diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.module.scss b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.module.scss index dd8d395ab1..cbe40d7485 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.module.scss +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.module.scss @@ -7,6 +7,12 @@ padding: 12px 16px 0 16px; } +.chartHeader { + display: flex; + flex-direction: column; + gap: 2px; +} + .chartTitle { text-transform: uppercase; letter-spacing: 0.05em; @@ -15,3 +21,11 @@ .chartBody { height: 340px; } + +.chartStatus { + display: flex; + align-items: center; + justify-content: center; + width: 100%; + height: 100%; +} diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.tsx index fec7817588..4fe3fe4168 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlChart/VolumeControlChart.tsx @@ -1,5 +1,6 @@ import { Color } from '@signozhq/design-tokens'; import { Typography } from '@signozhq/ui/typography'; +import { Spin } from 'antd'; import { useGetMetricReductionRuleTimeseries } from 'api/generated/services/metrics'; import { PANEL_TYPES } from 'constants/queryBuilder'; import BarChart from 'container/DashboardContainer/visualization/charts/BarChart/BarChart'; @@ -25,7 +26,9 @@ interface VolumeControlChartProps { } function VolumeControlChart({ enabled }: VolumeControlChartProps): JSX.Element { - const { data } = useGetMetricReductionRuleTimeseries({ query: { enabled } }); + const { data, isLoading, isError } = useGetMetricReductionRuleTimeseries({ + query: { enabled }, + }); const isDarkMode = useIsDarkMode(); const { timezone } = useTimezone(); @@ -65,11 +68,34 @@ function VolumeControlChart({ enabled }: VolumeControlChartProps): JSX.Element { return (
- - Series volume over time · ingested vs retained - +
+ + Sample volume · ingested vs retained + + + Last 6 hours + +
- {dimensions.width > 0 && ( + {isLoading && ( +
+ +
+ )} + {!isLoading && isError && ( +
+ + Failed to load chart + +
+ )} + {!isLoading && !isError && dimensions.width > 0 && ( 0 ? (1 - proposed / current) * 100 : 0; @@ -40,31 +43,59 @@ function ImpactPanel({ {!isLoading && preview && (
- - Current series +
+ + Full series + + + + +
+ + {formatCompact(full)} +
+
+
+ + Current retained + + + + +
{formatCompact(current)}
- - Proposed series - +
+ + Potential retained + + + + +
{formatCompact(proposed)} - -
-
- - Reduction - - = 0 ? 'success' : undefined} - className={styles.meterValue} - > - {reductionLabel} + {deltaPct !== 0 && ( + = 0 ? 'success' : undefined} + > + {reductionLabel} + + )}
diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/ModeSelector/ModeSelector.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/ModeSelector/ModeSelector.tsx index ed450d8334..7d138c315e 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/ModeSelector/ModeSelector.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/ModeSelector/ModeSelector.tsx @@ -18,12 +18,12 @@ const MODE_OPTIONS: ModeOption[] = [ }, { mode: 'include', - title: 'Include attributes', + title: 'Include', description: 'Allowlist: only the selected attributes stay queryable.', }, { mode: 'exclude', - title: 'Exclude attributes', + title: 'Exclude', description: 'Blocklist: the selected attributes are aggregated away.', }, ]; diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/RelatedAssetsWarning/RelatedAssetsWarning.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/RelatedAssetsWarning/RelatedAssetsWarning.tsx index 2f6eaee30a..4663acab9c 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/RelatedAssetsWarning/RelatedAssetsWarning.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/RelatedAssetsWarning/RelatedAssetsWarning.tsx @@ -53,12 +53,12 @@ function RelatedAssetsWarning({
- + This rule affects {impacted.length} related asset {impacted.length > 1 ? 's' : ''}. {impactedLabels.length > 0 && ( - + {impactedLabels.join(', ')} will no longer be queryable; affected panels fall back to aggregated data once the rule applies. @@ -73,7 +73,7 @@ function RelatedAssetsWarning({
  • {href ? ( ) : ( - + {label} )} diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/VolumeControlConfigDrawer.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/VolumeControlConfigDrawer.tsx index 625c9719cb..ad1da496da 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/VolumeControlConfigDrawer.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlConfigDrawer/VolumeControlConfigDrawer.tsx @@ -42,6 +42,10 @@ function VolumeControlConfigDrawer({ const footer = (
    + + Changes take effect about 5 minutes after saving. + +
    -
    {hasExistingRule && (
    ); diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlSection/VolumeControlSection.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlSection/VolumeControlSection.tsx index dbe31afdca..081b917d83 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlSection/VolumeControlSection.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlSection/VolumeControlSection.tsx @@ -22,7 +22,7 @@ function VolumeControlSection({ useVolumeControlFeatureGate(); const [isConfigOpen, setIsConfigOpen] = useState(false); - const { data, isLoading, error } = useListMetricReductionRules( + const { data, isLoading, isError } = useListMetricReductionRules( { metricName }, { query: { @@ -37,7 +37,7 @@ function VolumeControlSection({ } const rule = data?.data.rules?.[0]; - const hasRule = !!rule && !error; + const hasRule = !!rule && !isError; const openConfig = (): void => setIsConfigOpen(true); const closeConfig = (): void => setIsConfigOpen(false); @@ -53,6 +53,16 @@ function VolumeControlSection({ {isLoading && } + {!isLoading && isError && ( + + Failed to load volume control. Please try again. + + )} + {!isLoading && hasRule && rule && !rule.active && ( )} @@ -65,7 +75,7 @@ function VolumeControlSection({ /> )} - {!isLoading && !hasRule && ( + {!isLoading && !isError && !hasRule && ( )} diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.module.scss b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.module.scss index 7e30f93265..bef6a49d21 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.module.scss +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.module.scss @@ -1,3 +1,9 @@ +.statsSection { + display: flex; + flex-direction: column; + gap: 8px; +} + .stats { display: flex; flex-wrap: wrap; @@ -21,11 +27,24 @@ background: var(--callout-success-background); } +.statCardLabelRow { + display: flex; + align-items: center; + gap: 4px; +} + .statCardLabel { text-transform: uppercase; letter-spacing: 0.05em; } +.statCardInfo { + flex-shrink: 0; + color: var(--bg-vanilla-400, #c0c1c3); + line-height: 1; + cursor: help; +} + .statCardValue { display: flex; align-items: baseline; diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.tsx index 0dc9e92b15..588da70744 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlStats/VolumeControlStats.tsx @@ -1,4 +1,6 @@ +import { Info } from '@signozhq/icons'; import { Typography } from '@signozhq/ui/typography'; +import { Skeleton, Tooltip } from 'antd'; import cx from 'classnames'; import { formatCompact, formatUsd } from '../../../configUtils'; @@ -8,12 +10,17 @@ interface VolumeControlStatsProps { activeRules: number; ingestedSeries: number; retainedSeries: number; + ingestedSamples: number; + retainedSamples: number; estimatedMonthlySavingsUsd: number; + isLoading?: boolean; + isError?: boolean; } interface StatItem { label: string; value: string; + tooltip: string; delta?: string; unit?: string; highlighted?: boolean; @@ -24,20 +31,53 @@ function VolumeControlStats({ activeRules, ingestedSeries, retainedSeries, + ingestedSamples, + retainedSamples, estimatedMonthlySavingsUsd, + isLoading = false, + isError = false, }: VolumeControlStatsProps): JSX.Element { const overallReduction = ingestedSeries > 0 ? Math.round((1 - retainedSeries / ingestedSeries) * 100) : 0; + const sampleReduction = + ingestedSamples > 0 + ? Math.round((1 - retainedSamples / ingestedSamples) * 100) + : 0; + const items: StatItem[] = [ - { label: 'Active rules', value: String(activeRules) }, - { label: 'Ingested series', value: formatCompact(ingestedSeries) }, + { + label: 'Configured rules', + value: String(activeRules), + tooltip: 'Volume-control rules currently configured for this workspace.', + }, + { + label: 'Ingested series', + value: formatCompact(ingestedSeries), + tooltip: + 'Distinct time series across all metrics in the last 1 hour, before any reduction.', + }, { label: 'Retained series', value: formatCompact(retainedSeries), delta: overallReduction > 0 ? `−${overallReduction}%` : undefined, + tooltip: + 'Distinct time series kept across all metrics in the last 1 hour; everything except what the rules reduce away. Lower than ingested means more reduction.', + }, + { + label: 'Ingested samples', + value: formatCompact(ingestedSamples), + tooltip: + 'Sample data points across all metrics in the last 1 hour, before any reduction.', + }, + { + label: 'Retained samples', + value: formatCompact(retainedSamples), + delta: sampleReduction > 0 ? `−${sampleReduction}%` : undefined, + tooltip: + 'Sample data points kept across all metrics in the last 1 hour; everything except what the rules reduce. Samples reduce more than series because series do not all carry the same sample volume.', }, { label: 'Est. monthly savings', @@ -45,42 +85,73 @@ function VolumeControlStats({ unit: '/mo', highlighted: true, valueGood: true, + tooltip: + 'Rough monthly estimate: the samples the rules reduced in the last 1 hour, scaled to a month at 1-month standard retention. It is extrapolated from a single rolling hour.', }, ]; return ( -
    - {items.map((item) => ( -
    - - {item.label} - - + + Last 1 hour + +
    + {items.map((item) => ( +
    - {item.value} - {item.delta && ( - - {item.delta} +
    + + {item.label} + + + + +
    + {isLoading && ( + + )} + {!isLoading && isError && ( + + Failed to load )} - {item.unit && ( - - {item.unit} + {!isLoading && !isError && ( + + {item.value} + {item.delta && ( + + {item.delta} + + )} + {item.unit && ( + + {item.unit} + + )} )} - -
    - ))} +
    + ))} +
    ); } diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.module.scss b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.module.scss index 2d16f1121b..47b2bf68ed 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.module.scss +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.module.scss @@ -13,8 +13,10 @@ font-family: 'Geist Mono', monospace; } -.reductionCell { - font-family: 'Geist Mono', monospace; +.volumeCell { + display: flex; + flex-direction: column; + gap: 2px; } .empty { diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.tsx b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.tsx index f16db2aeac..b4153c6175 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.tsx +++ b/frontend/src/container/MetricsExplorer/VolumeControl/components/VolumeControlTab/VolumeControlTab.tsx @@ -36,7 +36,7 @@ type VolumeControlTableParams = Required< >; const DEFAULT_PARAMS: VolumeControlTableParams = { - orderBy: OrderBy.reduction, + orderBy: OrderBy.ingested_volume, order: SortOrder.desc, search: '', offset: 0, @@ -60,11 +60,20 @@ function VolumeControlTab(): JSX.Element { ); }, [debouncedSearch]); - const { data, isLoading } = useListMetricReductionRules(params, { + const { + data, + isLoading, + isError: isListError, + } = useListMetricReductionRules(params, { query: { enabled: isVolumeControlEnabled }, }); - const { data: statsData } = useGetMetricReductionRuleStats({ + const { + data: statsData, + isLoading: isStatsLoading, + isFetching: isStatsFetching, + isError: isStatsError, + } = useGetMetricReductionRuleStats({ query: { enabled: isVolumeControlEnabled }, }); const stats = statsData?.data; @@ -111,7 +120,7 @@ function VolumeControlTab(): JSX.Element { { title: 'MODE', key: 'mode', - width: 160, + width: 110, render: ( _value: unknown, rule: MetricreductionruletypesGettableReductionRuleDTO, @@ -138,7 +147,14 @@ function VolumeControlTab(): JSX.Element { ), }, { - title: 'INGESTED', + title: ( + <> + INGESTED{' '} + + (1h) + + + ), key: OrderBy.ingested_volume, width: 130, sorter: true, @@ -147,13 +163,28 @@ function VolumeControlTab(): JSX.Element { _value: unknown, rule: MetricreductionruletypesGettableReductionRuleDTO, ): JSX.Element => ( - - {formatCompact(rule.ingestedSeries)} - +
    + + {formatCompact(rule.ingestedSeries)}{' '} + + series + + + + {formatCompact(rule.ingestedSamples)} samples + +
    ), }, { - title: 'RETAINED', + title: ( + <> + RETAINED{' '} + + (1h) + + + ), key: OrderBy.reduced_volume, width: 130, sorter: true, @@ -162,22 +193,35 @@ function VolumeControlTab(): JSX.Element { _value: unknown, rule: MetricreductionruletypesGettableReductionRuleDTO, ): JSX.Element => ( - - {formatCompact(rule.retainedSeries)} - +
    + + {formatCompact(rule.retainedSeries)}{' '} + + series + + + + {formatCompact(rule.retainedSamples)} samples + +
    ), }, { title: 'CHANGE', - key: OrderBy.reduction, - width: 110, - sorter: true, - sortOrder: sortOrderFor(OrderBy.reduction), + width: 140, render: ( _value: unknown, rule: MetricreductionruletypesGettableReductionRuleDTO, ): JSX.Element => { - if (rule.reductionPercent <= 0) { + const seriesReduction = + rule.ingestedSeries > 0 + ? (1 - rule.retainedSeries / rule.ingestedSeries) * 100 + : 0; + const samplesReduction = + rule.ingestedSamples > 0 + ? (1 - rule.retainedSamples / rule.ingestedSamples) * 100 + : 0; + if (seriesReduction <= 0 && samplesReduction <= 0) { return ( — @@ -185,14 +229,18 @@ function VolumeControlTab(): JSX.Element { ); } return ( - - −{Math.round(rule.reductionPercent)}% - +
    + + {seriesReduction > 0 ? `−${Math.round(seriesReduction)}%` : '0%'}{' '} + + series + + + + {samplesReduction > 0 ? `−${Math.round(samplesReduction)}%` : '0%'}{' '} + samples + +
    ); }, }, @@ -273,7 +321,11 @@ function VolumeControlTab(): JSX.Element { activeRules={total} ingestedSeries={stats?.ingestedSeries ?? 0} retainedSeries={stats?.retainedSeries ?? 0} + ingestedSamples={stats?.ingestedSamples ?? 0} + retainedSamples={stats?.retainedSamples ?? 0} estimatedMonthlySavingsUsd={stats?.estimatedMonthlySavingsUsd ?? 0} + isLoading={isStatsLoading || isStatsFetching} + isError={isStatsError} /> @@ -293,7 +345,13 @@ function VolumeControlTab(): JSX.Element { showSizeChanger: false, }} locale={{ - emptyText: ( + emptyText: isListError ? ( +
    + + Failed to load volume control rules. Please try again. + +
    + ) : (
    No volume control rules yet. Open a metric and set one up to start diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/hooks/useVolumeControlConfig.ts b/frontend/src/container/MetricsExplorer/VolumeControl/hooks/useVolumeControlConfig.ts index 471185bf7b..4551fc13c0 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/hooks/useVolumeControlConfig.ts +++ b/frontend/src/container/MetricsExplorer/VolumeControl/hooks/useVolumeControlConfig.ts @@ -49,6 +49,7 @@ export interface UseVolumeControlConfigResult { const PREVIEW_DEBOUNCE_MS = 400; const SAVE_ERROR_MESSAGE = 'Failed to save volume control rule'; const REMOVE_ERROR_MESSAGE = 'Failed to remove volume control rule'; +const PREVIEW_ERROR_MESSAGE = 'Failed to preview volume control rule'; export function useVolumeControlConfig({ metricName, @@ -95,11 +96,25 @@ export function useVolumeControlConfig({ const timer = setTimeout(() => { previewMutate( { data: { metricName, matchType: matchTypeForMode(mode), labels } }, - { onSettled: () => setIsPreviewPending(false) }, + { + onError: (error) => + notifications.error({ + message: error.response?.data?.error?.message ?? PREVIEW_ERROR_MESSAGE, + }), + onSettled: () => setIsPreviewPending(false), + }, ); }, PREVIEW_DEBOUNCE_MS); return (): void => clearTimeout(timer); - }, [open, mode, labels, metricName, previewMutate, previewReset]); + }, [ + open, + mode, + labels, + metricName, + previewMutate, + previewReset, + notifications, + ]); const createMutation = useCreateMetricReductionRule(); const updateMutation = useUpdateMetricReductionRuleByID(); @@ -142,7 +157,10 @@ export function useVolumeControlConfig({ } const onSuccess = (): void => { - notifications.success({ message: 'Volume control rule saved' }); + notifications.success({ + message: + 'Volume control rule saved. It takes about 5 minutes to take effect.', + }); invalidate(); onClose(); }; diff --git a/frontend/src/container/MetricsExplorer/VolumeControl/ruleUtils.ts b/frontend/src/container/MetricsExplorer/VolumeControl/ruleUtils.ts index e4e4695d73..a425db1dd7 100644 --- a/frontend/src/container/MetricsExplorer/VolumeControl/ruleUtils.ts +++ b/frontend/src/container/MetricsExplorer/VolumeControl/ruleUtils.ts @@ -9,7 +9,7 @@ export function isKeepMode( export function getMatchTypeLabel( matchType: MetricreductionruletypesMatchTypeDTO, ): string { - return isKeepMode(matchType) ? 'Include attributes' : 'Exclude attributes'; + return isKeepMode(matchType) ? 'Include' : 'Exclude'; } export function getLabelVerb( diff --git a/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/InviteTeamMembers.tsx b/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/InviteTeamMembers.tsx index 2d130747e2..2d43e540ec 100644 --- a/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/InviteTeamMembers.tsx +++ b/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/InviteTeamMembers.tsx @@ -1,26 +1,12 @@ -import { useCallback, useEffect, useState } from 'react'; -import { useMutation } from 'react-query'; +import { useMemo } from 'react'; +import { ArrowRight, LoaderCircle } from '@signozhq/icons'; import { Button } from '@signozhq/ui/button'; -import { Callout } from '@signozhq/ui/callout'; -import { Input } from '@signozhq/ui/input'; -import { Select } from 'antd'; -import { Typography } from '@signozhq/ui/typography'; import logEvent from 'api/common/logEvent'; -import inviteUsers from 'api/v1/invite/bulk/create'; -import AuthError from 'components/AuthError/AuthError'; +import InviteMembers from 'components/InviteMembers/InviteMembers'; +import { InviteMemberRow, InviteResult } from 'components/InviteMembers/types'; +import { useRoles } from 'components/RolesSelect/RolesSelect'; import { useNotifications } from 'hooks/useNotifications'; -import { cloneDeep, debounce } from 'lodash-es'; -import { - ArrowRight, - ChevronDown, - CircleAlert, - LoaderCircle, - Plus, - Trash2, -} from '@signozhq/icons'; -import APIError from 'types/api/error'; import { getBaseUrl } from 'utils/basePath'; -import { v4 as uuid } from 'uuid'; import { OnboardingQuestionHeader } from '../OnboardingQuestionHeader'; @@ -36,101 +22,41 @@ interface TeamMember { interface InviteTeamMembersProps { isLoading: boolean; - teamMembers: TeamMember[] | null; - setTeamMembers: (teamMembers: TeamMember[]) => void; onNext: () => void; } function InviteTeamMembers({ isLoading, - teamMembers, - setTeamMembers, onNext, }: InviteTeamMembersProps): JSX.Element { - const [teamMembersToInvite, setTeamMembersToInvite] = useState< - TeamMember[] | null - >(teamMembers); - const [emailValidity, setEmailValidity] = useState>( - {}, - ); - const [hasInvalidEmails, setHasInvalidEmails] = useState(false); - const [hasInvalidRoles, setHasInvalidRoles] = useState(false); - const [inviteError, setInviteError] = useState(null); const { notifications } = useNotifications(); + const { roles } = useRoles(); - const defaultTeamMember: TeamMember = { - email: '', - role: '', - name: '', - frontendBaseUrl: getBaseUrl(), - id: '', - }; - - useEffect(() => { - if (teamMembers === null) { - const initialTeamMembers = Array.from({ length: 3 }, () => ({ - ...defaultTeamMember, - id: uuid(), - })); - - setTeamMembersToInvite(initialTeamMembers); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [teamMembers]); - - const handleAddTeamMember = (): void => { - const newTeamMember = { - ...defaultTeamMember, - id: uuid(), - }; - setTeamMembersToInvite((prev) => [...(prev || []), newTeamMember]); - }; - - const handleRemoveTeamMember = (id: string): void => { - setTeamMembersToInvite((prev) => (prev || []).filter((m) => m.id !== id)); - }; - - const isMemberTouched = (member: TeamMember): boolean => - member.email.trim() !== '' || - Boolean(member.role && member.role.trim() !== ''); - - const validateAllUsers = (): boolean => { - let isValid = true; - let hasEmailErrors = false; - let hasRoleErrors = false; - - const updatedEmailValidity: Record = {}; - - const touchedMembers = teamMembersToInvite?.filter(isMemberTouched) ?? []; - - touchedMembers?.forEach((member) => { - const emailValid = /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(member.email); - const roleValid = Boolean(member.role && member.role.trim() !== ''); - - if (!emailValid || !member.email) { - isValid = false; - hasEmailErrors = true; - } - if (!roleValid) { - isValid = false; - hasRoleErrors = true; - } - - if (member.id) { - updatedEmailValidity[member.id] = emailValid; + const roleIdToName = useMemo(() => { + const map: Record = {}; + roles.forEach((role) => { + if (role.id && role.name) { + map[role.id] = role.name; } }); + return map; + }, [roles]); - setEmailValidity(updatedEmailValidity); - setHasInvalidEmails(hasEmailErrors); - setHasInvalidRoles(hasRoleErrors); + const toTeamMembers = (rows: InviteMemberRow[]): TeamMember[] => + rows.map((row) => ({ + email: row.email, + role: roleIdToName[row.roleId] ?? row.roleId, + name: '', + frontendBaseUrl: getBaseUrl(), + id: row.id, + })); - return isValid; - }; - - const handleInviteUsersSuccess = (): void => { + const handleSuccess = ( + _results: InviteResult[], + rows: InviteMemberRow[], + ): void => { logEvent('Org Onboarding: Invite Team Members Success', { - teamMembers: teamMembersToInvite, + teamMembers: toTeamMembers(rows), }); notifications.success({ message: 'Invites sent successfully!', @@ -140,125 +66,34 @@ function InviteTeamMembers({ }, 1000); }; - const { mutate: sendInvites, isLoading: isSendingInvites } = useMutation( - inviteUsers, - { - onSuccess: (): void => { - handleInviteUsersSuccess(); - }, - onError: (error: APIError): void => { - logEvent('Org Onboarding: Invite Team Members Failed', { - teamMembers: teamMembersToInvite, - }); - setInviteError(error); - }, - }, - ); - - const handleNext = (): void => { - if (validateAllUsers()) { - setTeamMembers(teamMembersToInvite?.filter(isMemberTouched) ?? []); - setHasInvalidEmails(false); - setHasInvalidRoles(false); - setInviteError(null); - sendInvites({ - invites: teamMembersToInvite?.filter(isMemberTouched) ?? [], - }); - } + const handlePartialSuccess = ( + _results: InviteResult[], + rows: InviteMemberRow[], + ): void => { + logEvent('Org Onboarding: Invite Team Members Partial Success', { + teamMembers: toTeamMembers(rows), + }); + notifications.warning({ + message: 'Some invites failed. Check the errors above.', + }); }; - // eslint-disable-next-line react-hooks/exhaustive-deps - const debouncedValidateEmail = useCallback( - debounce((email: string, memberId: string, updatedMembers: TeamMember[]) => { - const isValid = /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email); - setEmailValidity((prev) => ({ ...prev, [memberId]: isValid })); - - // Clear hasInvalidEmails only when ALL emails are valid - if (hasInvalidEmails) { - const allEmailsValid = updatedMembers.every( - (m) => m.email && /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(m.email), - ); - if (allEmailsValid) { - setHasInvalidEmails(false); - } - } - }, 500), - [hasInvalidEmails], - ); - - const handleEmailChange = useCallback( - (e: React.ChangeEvent, member: TeamMember): void => { - const { value } = e.target; - const updatedMembers = cloneDeep(teamMembersToInvite || []); - - const memberToUpdate = updatedMembers.find((m) => m.id === member.id); - if (memberToUpdate && member.id) { - memberToUpdate.email = value; - setTeamMembersToInvite(updatedMembers); - debouncedValidateEmail(value, member.id, updatedMembers); - // Clear API error when user starts typing - if (inviteError) { - setInviteError(null); - } - } - }, - [debouncedValidateEmail, inviteError, teamMembersToInvite], - ); - - const createEmailChangeHandler = useCallback( - (member: TeamMember) => - (e: React.ChangeEvent): void => { - handleEmailChange(e, member); - }, - [handleEmailChange], - ); - - const handleRoleChange = (role: string, member: TeamMember): void => { - const updatedMembers = cloneDeep(teamMembersToInvite || []); - const memberToUpdate = updatedMembers.find((m) => m.id === member.id); - if (memberToUpdate && member.id) { - memberToUpdate.role = role; - setTeamMembersToInvite(updatedMembers); - - // Clear errors when user selects a role - if (hasInvalidRoles) { - // Check if all roles are now valid - const allRolesValid = updatedMembers.every( - (m) => m.role && m.role.trim() !== '', - ); - if (allRolesValid) { - setHasInvalidRoles(false); - } - } - if (inviteError) { - setInviteError(null); - } - } - }; - - const getValidationErrorMessage = (): string => { - if (hasInvalidEmails && hasInvalidRoles) { - return 'Please enter valid emails and select roles for team members'; - } - if (hasInvalidEmails) { - return 'Please enter valid emails for team members'; - } - return 'Please select roles for team members'; + const handleAllFailed = ( + _results: InviteResult[], + rows: InviteMemberRow[], + ): void => { + logEvent('Org Onboarding: Invite Team Members Failed', { + teamMembers: toTeamMembers(rows), + }); }; const handleDoLater = (): void => { logEvent('Org Onboarding: Clicked Do Later', { currentPageID: 4, }); - onNext(); }; - const hasInvites = - (teamMembersToInvite?.filter(isMemberTouched) ?? []).length > 0; - const isButtonDisabled = isSendingInvites || isLoading; - const isInviteButtonDisabled = isButtonDisabled || !hasInvites; - return (
    -
    -
    -
    Email address
    -
    Roles
    -
    -
    + { + const isButtonDisabled = isSubmitting || isLoading; + const isInviteButtonDisabled = isButtonDisabled || !canSubmit; -
    - {teamMembersToInvite?.map((member) => ( -
    -
    - - {member.id && - emailValidity[member.id] === false && - member.email.trim() !== '' && ( - - Invalid email address - - )} -
    -
    - -
    -
    - {teamMembersToInvite && teamMembersToInvite.length > 1 && ( - - )} -
    + return ( +
    + +
    - ))} -
    - -
    - -
    -
    + ); + }} + />
    - - {(hasInvalidEmails || hasInvalidRoles) && ( - } - className="invite-team-members-error-callout" - > - {getValidationErrorMessage()} - - )} - - {inviteError && !hasInvalidEmails && !hasInvalidRoles && ( - - )} - -
    - - -
    ); diff --git a/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/__tests__/InviteTeamMembers.test.tsx b/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/__tests__/InviteTeamMembers.test.tsx index faba3b9986..843455ba95 100644 --- a/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/__tests__/InviteTeamMembers.test.tsx +++ b/frontend/src/container/OnboardingQuestionaire/InviteTeamMembers/__tests__/InviteTeamMembers.test.tsx @@ -1,97 +1,86 @@ -import { rest, server } from 'mocks-server/server'; import { - fireEvent, - render, - screen, - userEvent, - waitFor, -} from 'tests/test-utils'; + InviteMemberRow, + InviteMembersProps, + InviteResult, +} from 'components/InviteMembers/types'; +import logEvent from 'api/common/logEvent'; +import { render, screen, userEvent } from 'tests/test-utils'; import InviteTeamMembers from '../InviteTeamMembers'; -const mockNotificationSuccess = jest.fn() as jest.MockedFunction< - (args: { message: string }) => void ->; -const mockNotificationError = jest.fn() as jest.MockedFunction< - (args: { message: string }) => void ->; +const mockNotificationSuccess = jest.fn(); +const mockNotificationWarning = jest.fn(); jest.mock('hooks/useNotifications', () => ({ useNotifications: (): any => ({ notifications: { success: mockNotificationSuccess, - error: mockNotificationError, + warning: mockNotificationWarning, }, }), })); -const INVITE_USERS_ENDPOINT = '*/api/v1/invite/bulk'; +jest.mock('api/common/logEvent', () => jest.fn()); -interface TeamMember { - email: string; - role: string; - name: string; - frontendBaseUrl: string; - id: string; -} +jest.mock('components/RolesSelect/RolesSelect', () => ({ + useRoles: (): any => ({ + roles: [ + { id: 'role-viewer-id', name: 'VIEWER' }, + { id: 'role-editor-id', name: 'EDITOR' }, + { id: 'role-admin-id', name: 'ADMIN' }, + ], + isLoading: false, + isError: false, + error: undefined, + refetch: jest.fn(), + }), +})); -interface InviteRequestBody { - invites: { email: string; role: string }[]; -} +jest.mock('utils/basePath', () => ({ + ...jest.requireActual('utils/basePath'), + getBaseUrl: (): string => 'http://localhost:3301', +})); -interface RenderProps { - isLoading?: boolean; - teamMembers?: TeamMember[] | null; -} +let mockInviteMembersProps: InviteMembersProps | null = null; -const mockOnNext = jest.fn() as jest.MockedFunction<() => void>; -const mockSetTeamMembers = jest.fn() as jest.MockedFunction< - (members: TeamMember[]) => void ->; +jest.mock('components/InviteMembers/InviteMembers', () => { + return function MockInviteMembers(props: InviteMembersProps): JSX.Element { + mockInviteMembersProps = props; + return ( +
    + {props.renderFooter?.({ + submit: jest.fn().mockResolvedValue([]), + reset: jest.fn(), + canSubmit: true, + isSubmitting: false, + touchedCount: 0, + })} +
    + ); + }; +}); + +const mockOnNext = jest.fn(); function renderComponent({ isLoading = false, - teamMembers = null, -}: RenderProps = {}): ReturnType { - return render( - , - ); -} - -async function selectRole( - user: ReturnType, - selectIndex: number, - optionLabel: string, -): Promise { - const placeholders = screen.getAllByText(/select roles/i); - await user.click(placeholders[selectIndex]); - const optionContent = await screen.findByText(optionLabel); - fireEvent.click(optionContent); +}: { isLoading?: boolean } = {}): ReturnType { + return render(); } describe('InviteTeamMembers', () => { beforeEach(() => { jest.clearAllMocks(); - - server.use( - rest.post(INVITE_USERS_ENDPOINT, (_, res, ctx) => - res(ctx.status(200), ctx.json({ status: 'success' })), - ), - ); + jest.useFakeTimers(); + mockInviteMembersProps = null; }); afterEach(() => { jest.useRealTimers(); - server.resetHandlers(); }); - describe('Initial rendering', () => { - it('renders the page header, column labels, default rows, and action buttons', () => { + describe('rendering', () => { + it('renders header and InviteMembers component', () => { renderComponent(); expect( @@ -100,11 +89,20 @@ describe('InviteTeamMembers', () => { expect( screen.getByText(/signoz is a lot more useful with collaborators/i), ).toBeInTheDocument(); - expect( - screen.getAllByPlaceholderText(/e\.g\. john@signoz\.io/i), - ).toHaveLength(3); - expect(screen.getByText('Email address')).toBeInTheDocument(); - expect(screen.getByText('Roles')).toBeInTheDocument(); + expect(screen.getByTestId('mock-invite-members')).toBeInTheDocument(); + }); + + it('passes showHeader=true to InviteMembers', () => { + renderComponent(); + + expect(mockInviteMembersProps?.showHeader).toBe(true); + }); + }); + + describe('footer buttons', () => { + it('renders Send Invites and Do Later buttons', () => { + renderComponent(); + expect( screen.getByRole('button', { name: /send invites/i }), ).toBeInTheDocument(); @@ -113,7 +111,7 @@ describe('InviteTeamMembers', () => { ).toBeInTheDocument(); }); - it('disables both action buttons while isLoading is true', () => { + it('disables buttons when isLoading=true', () => { renderComponent({ isLoading: true }); expect(screen.getByRole('button', { name: /send invites/i })).toBeDisabled(); @@ -121,355 +119,181 @@ describe('InviteTeamMembers', () => { screen.getByRole('button', { name: /i'll do this later/i }), ).toBeDisabled(); }); - }); - describe('Row management', () => { - it('adds a new empty row when "Add another" is clicked', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - renderComponent(); + it('disables Send Invites when canSubmit=false from InviteMembers', () => { + const { unmount } = renderComponent(); + unmount(); - expect( - screen.getAllByPlaceholderText(/e\.g\. john@signoz\.io/i), - ).toHaveLength(3); - - await user.click(screen.getByRole('button', { name: /add another/i })); - - expect( - screen.getAllByPlaceholderText(/e\.g\. john@signoz\.io/i), - ).toHaveLength(4); - }); - - it('removes the correct row when its trash icon is clicked', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - renderComponent(); - - const emailInputs = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - await user.type(emailInputs[0], 'first@example.com'); - await screen.findByDisplayValue('first@example.com'); - - await user.click( - screen.getAllByRole('button', { name: /remove team member/i })[0], + const { getByTestId } = render( + mockInviteMembersProps?.renderFooter?.({ + submit: jest.fn().mockResolvedValue([]), + reset: jest.fn(), + canSubmit: false, + isSubmitting: false, + touchedCount: 0, + }) as JSX.Element, ); - await waitFor(() => { - expect( - screen.queryByDisplayValue('first@example.com'), - ).not.toBeInTheDocument(); - expect( - screen.getAllByPlaceholderText(/e\.g\. john@signoz\.io/i), - ).toHaveLength(2); - }); + expect(getByTestId('send-invites-button')).toBeDisabled(); + expect(getByTestId('do-later-button')).not.toBeDisabled(); }); - it('hides remove buttons when only one row remains', async () => { - renderComponent(); - const user = userEvent.setup({ pointerEventsCheck: 0 }); + it('disables buttons when isSubmitting=true from InviteMembers', () => { + const { unmount } = renderComponent(); + unmount(); - let removeButtons = screen.getAllByRole('button', { - name: /remove team member/i, - }); - while (removeButtons.length > 0) { - await user.click(removeButtons[0]); - removeButtons = screen.queryAllByRole('button', { - name: /remove team member/i, - }); - } + const { getByTestId } = render( + mockInviteMembersProps?.renderFooter?.({ + submit: jest.fn().mockResolvedValue([]), + reset: jest.fn(), + canSubmit: true, + isSubmitting: true, + touchedCount: 0, + }) as JSX.Element, + ); - expect( - screen.queryByRole('button', { name: /remove team member/i }), - ).not.toBeInTheDocument(); + expect(getByTestId('send-invites-button')).toBeDisabled(); + expect(getByTestId('do-later-button')).toBeDisabled(); }); }); - describe('Inline email validation', () => { - it('shows an inline error after typing an invalid email and clears it when a valid email is entered', async () => { - jest.useFakeTimers(); - const user = userEvent.setup({ - advanceTimers: (ms) => jest.advanceTimersByTime(ms), - }); + describe('handleSuccess callback', () => { + it('logs event with teamMembers in correct shape, shows success notification, and calls onNext after delay', () => { renderComponent(); - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); + const mockResults: InviteResult[] = [ + { email: 'user1@test.com', success: true }, + { email: 'user2@test.com', success: true }, + ]; + const mockRows: InviteMemberRow[] = [ + { id: 'row-1', email: 'user1@test.com', roleId: 'role-viewer-id' }, + { id: 'row-2', email: 'user2@test.com', roleId: 'role-editor-id' }, + ]; + mockInviteMembersProps?.onSuccess?.(mockResults, mockRows); - await user.type(firstInput, 'not-an-email'); - jest.advanceTimersByTime(600); - await waitFor(() => { - expect(screen.getByText(/invalid email address/i)).toBeInTheDocument(); - }); - - await user.clear(firstInput); - await user.type(firstInput, 'good@example.com'); - jest.advanceTimersByTime(600); - await waitFor(() => { - expect( - screen.queryByText(/invalid email address/i), - ).not.toBeInTheDocument(); - }); - }); - - it('does not show an inline error when the field is cleared back to empty', async () => { - jest.useFakeTimers(); - const user = userEvent.setup({ - advanceTimers: (ms) => jest.advanceTimersByTime(ms), - }); - renderComponent(); - - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - await user.type(firstInput, 'a'); - await user.clear(firstInput); - jest.advanceTimersByTime(600); - - await waitFor(() => { - expect( - screen.queryByText(/invalid email address/i), - ).not.toBeInTheDocument(); - }); - }); - }); - - describe('Validation callout on Complete', () => { - it('shows the correct callout message for each combination of email/role validity', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0, delay: null }); - renderComponent(); - - const removeButtons = screen.getAllByRole('button', { - name: /remove team member/i, - }); - await user.click(removeButtons[0]); - await user.click( - screen.getAllByRole('button', { name: /remove team member/i })[0], - ); - - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - - await user.type(firstInput, 'bad-email'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - await waitFor(() => { - expect( - screen.getByText( - /please enter valid emails and select roles for team members/i, - ), - ).toBeInTheDocument(); - expect( - screen.queryByText(/please enter valid emails for team members/i), - ).not.toBeInTheDocument(); - expect( - screen.queryByText(/please select roles for team members/i), - ).not.toBeInTheDocument(); - }); - - await selectRole(user, 0, 'Viewer'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - await waitFor(() => { - expect( - screen.getByText(/please enter valid emails for team members/i), - ).toBeInTheDocument(); - expect( - screen.queryByText(/please select roles for team members/i), - ).not.toBeInTheDocument(); - expect( - screen.queryByText(/please enter valid emails and select roles/i), - ).not.toBeInTheDocument(); - }); - - await user.clear(firstInput); - await user.type(firstInput, 'valid@example.com'); - await user.click(screen.getByRole('button', { name: /add another/i })); - const allInputs = screen.getAllByPlaceholderText(/e\.g\. john@signoz\.io/i); - await user.type(allInputs[1], 'norole@example.com'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - await waitFor(() => { - expect( - screen.getByText(/please select roles for team members/i), - ).toBeInTheDocument(); - expect( - screen.queryByText(/please enter valid emails for team members/i), - ).not.toBeInTheDocument(); - expect( - screen.queryByText(/please enter valid emails and select roles/i), - ).not.toBeInTheDocument(); - }); - }, 15000); - - it('treats whitespace as untouched, clears the callout on fix-and-resubmit, and clears role error on role select', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0, delay: null }); - renderComponent(); - - const removeButtons = screen.getAllByRole('button', { - name: /remove team member/i, - }); - await user.click(removeButtons[0]); - await user.click( - screen.getAllByRole('button', { name: /remove team member/i })[0], - ); - - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - - await user.type(firstInput, ' '); - await user.click(screen.getByRole('button', { name: /send invites/i })); - await waitFor(() => { - expect( - screen.queryByText(/please enter valid emails/i), - ).not.toBeInTheDocument(); - expect(screen.queryByText(/please select roles/i)).not.toBeInTheDocument(); - }); - - await user.clear(firstInput); - await user.type(firstInput, 'bad-email'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - await waitFor(() => { - expect( - screen.getByText( - /please enter valid emails and select roles for team members/i, - ), - ).toBeInTheDocument(); - expect( - screen.queryByText(/please enter valid emails for team members/i), - ).not.toBeInTheDocument(); - expect( - screen.queryByText(/please select roles for team members/i), - ).not.toBeInTheDocument(); - }); - - await user.clear(firstInput); - await user.type(firstInput, 'good@example.com'); - await selectRole(user, 0, 'Admin'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - await waitFor(() => { - expect( - screen.queryByText(/please enter valid emails and select roles/i), - ).not.toBeInTheDocument(); - expect( - screen.queryByText(/please enter valid emails for team members/i), - ).not.toBeInTheDocument(); - expect( - screen.queryByText(/please select roles for team members/i), - ).not.toBeInTheDocument(); - }); - - await waitFor(() => expect(mockOnNext).toHaveBeenCalledTimes(1), { - timeout: 1200, - }); - }, 15000); - - it('disables the Send Invites button when all rows are untouched (empty)', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - renderComponent(); - - const sendInvitesBtn = screen.getByRole('button', { name: /send invites/i }); - expect(sendInvitesBtn).toBeDisabled(); - - // Type something to make a row touched - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - await user.type(firstInput, 'a'); - - expect(sendInvitesBtn).not.toBeDisabled(); - }); - }); - - describe('API integration', () => { - it('only sends touched (non-empty) rows — empty rows are excluded from the invite payload', async () => { - let capturedBody: InviteRequestBody | null = null; - - server.use( - rest.post(INVITE_USERS_ENDPOINT, async (req, res, ctx) => { - capturedBody = await req.json(); - return res(ctx.status(200), ctx.json({ status: 'success' })); - }), - ); - - const user = userEvent.setup({ pointerEventsCheck: 0 }); - renderComponent(); - - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - await user.type(firstInput, 'only@example.com'); - await selectRole(user, 0, 'Admin'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - - await waitFor(() => { - expect(capturedBody).not.toBeNull(); - expect(capturedBody?.invites).toHaveLength(1); - expect(capturedBody?.invites[0]).toMatchObject({ - email: 'only@example.com', - role: 'ADMIN', - }); - }); - await waitFor(() => expect(mockOnNext).toHaveBeenCalled(), { - timeout: 1200, - }); - }); - - it('calls the invite API, shows a success notification, and calls onNext after the 1 s delay', async () => { - const user = userEvent.setup({ pointerEventsCheck: 0 }); - renderComponent(); - - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, - ); - await user.type(firstInput, 'alice@example.com'); - await selectRole(user, 0, 'Admin'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - - await waitFor(() => { - expect(mockNotificationSuccess).toHaveBeenCalledWith( - expect.objectContaining({ message: 'Invites sent successfully!' }), - ); - }); - - await waitFor( - () => { - expect(mockOnNext).toHaveBeenCalledTimes(1); + expect(logEvent).toHaveBeenCalledWith( + 'Org Onboarding: Invite Team Members Success', + { + teamMembers: [ + { + email: 'user1@test.com', + role: 'VIEWER', + name: '', + frontendBaseUrl: 'http://localhost:3301', + id: 'row-1', + }, + { + email: 'user2@test.com', + role: 'EDITOR', + name: '', + frontendBaseUrl: 'http://localhost:3301', + id: 'row-2', + }, + ], }, - { timeout: 1200 }, ); + expect(mockNotificationSuccess).toHaveBeenCalledWith({ + message: 'Invites sent successfully!', + }); + + expect(mockOnNext).not.toHaveBeenCalled(); + jest.advanceTimersByTime(1000); + expect(mockOnNext).toHaveBeenCalledTimes(1); }); + }); - it('renders an API error container when the invite request fails', async () => { - server.use( - rest.post(INVITE_USERS_ENDPOINT, (_, res, ctx) => - res( - ctx.status(500), - ctx.json({ - errors: [{ code: 'INTERNAL_ERROR', msg: 'Something went wrong' }], - }), - ), - ), - ); - - const user = userEvent.setup({ pointerEventsCheck: 0 }); + describe('handlePartialSuccess callback', () => { + it('logs event with teamMembers in correct shape and shows warning notification', () => { renderComponent(); - const [firstInput] = screen.getAllByPlaceholderText( - /e\.g\. john@signoz\.io/i, + const mockResults: InviteResult[] = [ + { email: 'user1@test.com', success: true }, + { email: 'user2@test.com', success: false, error: 'Already exists' }, + ]; + const mockRows: InviteMemberRow[] = [ + { id: 'row-1', email: 'user1@test.com', roleId: 'role-viewer-id' }, + { id: 'row-2', email: 'user2@test.com', roleId: 'role-admin-id' }, + ]; + mockInviteMembersProps?.onPartialSuccess?.(mockResults, mockRows); + + expect(logEvent).toHaveBeenCalledWith( + 'Org Onboarding: Invite Team Members Partial Success', + { + teamMembers: [ + { + email: 'user1@test.com', + role: 'VIEWER', + name: '', + frontendBaseUrl: 'http://localhost:3301', + id: 'row-1', + }, + { + email: 'user2@test.com', + role: 'ADMIN', + name: '', + frontendBaseUrl: 'http://localhost:3301', + id: 'row-2', + }, + ], + }, ); - await user.type(firstInput, 'fail@example.com'); - await selectRole(user, 0, 'Viewer'); - await user.click(screen.getByRole('button', { name: /send invites/i })); - - await waitFor(() => { - expect(document.querySelector('.auth-error-container')).toBeInTheDocument(); + expect(mockNotificationWarning).toHaveBeenCalledWith({ + message: 'Some invites failed. Check the errors above.', }); + }); + }); - await user.type(firstInput, 'x'); - await waitFor(() => { - expect( - document.querySelector('.auth-error-container'), - ).not.toBeInTheDocument(); + describe('handleAllFailed callback', () => { + it('logs event with teamMembers in correct shape', () => { + renderComponent(); + + const mockResults: InviteResult[] = [ + { email: 'user1@test.com', success: false, error: 'Error 1' }, + { email: 'user2@test.com', success: false, error: 'Error 2' }, + ]; + const mockRows: InviteMemberRow[] = [ + { id: 'row-1', email: 'user1@test.com', roleId: 'role-editor-id' }, + { id: 'row-2', email: 'user2@test.com', roleId: 'role-viewer-id' }, + ]; + mockInviteMembersProps?.onAllFailed?.(mockResults, mockRows); + + expect(logEvent).toHaveBeenCalledWith( + 'Org Onboarding: Invite Team Members Failed', + { + teamMembers: [ + { + email: 'user1@test.com', + role: 'EDITOR', + name: '', + frontendBaseUrl: 'http://localhost:3301', + id: 'row-1', + }, + { + email: 'user2@test.com', + role: 'VIEWER', + name: '', + frontendBaseUrl: 'http://localhost:3301', + id: 'row-2', + }, + ], + }, + ); + }); + }); + + describe('handleDoLater', () => { + it('logs event and calls onNext immediately', async () => { + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }); + renderComponent(); + + await user.click( + screen.getByRole('button', { name: /i'll do this later/i }), + ); + + expect(logEvent).toHaveBeenCalledWith('Org Onboarding: Clicked Do Later', { + currentPageID: 4, }); + expect(mockOnNext).toHaveBeenCalledTimes(1); }); }); }); diff --git a/frontend/src/container/OnboardingQuestionaire/OnboardingQuestionaire.styles.scss b/frontend/src/container/OnboardingQuestionaire/OnboardingQuestionaire.styles.scss index c2c6d5ea3c..bbc9672378 100644 --- a/frontend/src/container/OnboardingQuestionaire/OnboardingQuestionaire.styles.scss +++ b/frontend/src/container/OnboardingQuestionaire/OnboardingQuestionaire.styles.scss @@ -143,6 +143,8 @@ } &.invite-team-members-form { + --invite-members-field-background: var(--l3-background); + padding-right: 12px; .form-group { diff --git a/frontend/src/container/OnboardingQuestionaire/__tests__/OnboardingQuestionaire.test.tsx b/frontend/src/container/OnboardingQuestionaire/__tests__/OnboardingQuestionaire.test.tsx index 8eb5437035..94d1f0a8fa 100644 --- a/frontend/src/container/OnboardingQuestionaire/__tests__/OnboardingQuestionaire.test.tsx +++ b/frontend/src/container/OnboardingQuestionaire/__tests__/OnboardingQuestionaire.test.tsx @@ -22,7 +22,8 @@ const ORG_PREFERENCES_ENDPOINT = '*/api/v1/org/preferences/list'; const UPDATE_ORG_PREFERENCE_ENDPOINT = '*/api/v1/org/preferences/name/update'; const UPDATE_PROFILE_ENDPOINT = '*/api/v2/zeus/profiles'; const EDIT_ORG_ENDPOINT = '*/api/v2/orgs/me'; -const INVITE_USERS_ENDPOINT = '*/api/v1/invite/bulk/create'; +const CREATE_USER_ENDPOINT = '*/api/v2/users'; +const LIST_ROLES_ENDPOINT = '*/api/v1/roles'; const mockOrgPreferences = { data: { @@ -31,6 +32,12 @@ const mockOrgPreferences = { status: 'success', }; +const MOCK_ROLES = [ + { id: 'role-admin', name: 'Admin', description: 'Admin role' }, + { id: 'role-editor', name: 'Editor', description: 'Editor role' }, + { id: 'role-viewer', name: 'Viewer', description: 'Viewer role' }, +]; + describe('OnboardingQuestionaire Component', () => { beforeEach(() => { jest.clearAllMocks(); @@ -48,8 +55,11 @@ describe('OnboardingQuestionaire Component', () => { rest.post(UPDATE_ORG_PREFERENCE_ENDPOINT, (_, res, ctx) => res(ctx.status(200), ctx.json({ status: 'success' })), ), - rest.post(INVITE_USERS_ENDPOINT, (_, res, ctx) => - res(ctx.status(200), ctx.json({ status: 'success' })), + rest.get(LIST_ROLES_ENDPOINT, (_, res, ctx) => + res(ctx.status(200), ctx.json({ data: MOCK_ROLES })), + ), + rest.post(CREATE_USER_ENDPOINT, (_, res, ctx) => + res(ctx.status(201), ctx.json({ data: { id: 'user-123' } })), ), ); }); diff --git a/frontend/src/container/OnboardingQuestionaire/index.tsx b/frontend/src/container/OnboardingQuestionaire/index.tsx index 5dfc03d1c0..9da7fe63e6 100644 --- a/frontend/src/container/OnboardingQuestionaire/index.tsx +++ b/frontend/src/container/OnboardingQuestionaire/index.tsx @@ -11,7 +11,6 @@ import { AxiosError } from 'axios'; import { SOMETHING_WENT_WRONG } from 'constants/api'; import { ORG_PREFERENCES } from 'constants/orgPreferences'; import ROUTES from 'constants/routes'; -import { InviteTeamMembersProps } from 'container/OrganizationSettings/utils'; import { useNotifications } from 'hooks/useNotifications'; import history from 'lib/history'; import { useAppContext } from 'providers/App/App'; @@ -71,9 +70,6 @@ function OnboardingQuestionaire(): JSX.Element { const [optimiseSignozDetails, setOptimiseSignozDetails] = useState(INITIAL_OPTIMISE_SIGNOZ_DETAILS); - const [teamMembers, setTeamMembers] = useState< - InviteTeamMembersProps[] | null - >(null); const [updatingOrgOnboardingStatus, setUpdatingOrgOnboardingStatus] = useState(false); @@ -232,8 +228,6 @@ function OnboardingQuestionaire(): JSX.Element { {currentStep === 4 && ( )} diff --git a/frontend/src/container/OnboardingV2Container/AddDataSource/AddDataSource.tsx b/frontend/src/container/OnboardingV2Container/AddDataSource/AddDataSource.tsx index f92cff989f..81b4246835 100644 --- a/frontend/src/container/OnboardingV2Container/AddDataSource/AddDataSource.tsx +++ b/frontend/src/container/OnboardingV2Container/AddDataSource/AddDataSource.tsx @@ -1,5 +1,5 @@ import React, { useCallback, useEffect, useRef, useState } from 'react'; -import { Check, Goal, Search, UserPlus, X } from '@signozhq/icons'; +import { ArrowRight, Check, Goal, Search, UserPlus, X } from '@signozhq/icons'; import { Button, Flex, @@ -10,6 +10,8 @@ import { Space, Steps, } from 'antd'; +import { Button as SignozButton } from '@signozhq/ui/button'; +import { toast } from '@signozhq/ui/sonner'; import { Typography } from '@signozhq/ui/typography'; import logEvent from 'api/common/logEvent'; import LaunchChatSupport from 'components/LaunchChatSupport/LaunchChatSupport'; @@ -27,7 +29,7 @@ import { isModifierKeyPressed } from 'utils/app'; import signozBrandLogoUrl from '@/assets/Logos/signoz-brand-logo.svg'; import OnboardingIngestionDetails from '../IngestionDetails/IngestionDetails'; -import InviteTeamMembers from '../InviteTeamMembers/InviteTeamMembers'; +import InviteMembers from 'components/InviteMembers/InviteMembers'; import onboardingConfigWithLinks from '../onboarding-configs/onboarding-config-with-links'; import '../OnboardingV2.styles.scss'; @@ -119,6 +121,10 @@ const ONBOARDING_V3_ANALYTICS_EVENTS_MAP = { GET_HELP_BUTTON_CLICKED: 'Get help clicked', GET_EXPERT_ASSISTANCE_BUTTON_CLICKED: 'Get expert assistance clicked', INVITE_TEAM_MEMBER_BUTTON_CLICKED: 'Invite team member clicked', + INVITE_TEAM_MEMBER_SEND_CLICKED: 'Send invites clicked', + INVITE_TEAM_MEMBER_SUCCESS: 'Invite team members success', + INVITE_TEAM_MEMBER_PARTIAL_SUCCESS: 'Invite team members partial success', + INVITE_TEAM_MEMBER_FAILED: 'Invite team members failed', CLOSE_ONBOARDING_CLICKED: 'Close onboarding clicked', DATA_SOURCE_REQUESTED: 'Datasource requested', DATA_SOURCE_SEARCHED: 'Searched', @@ -1147,12 +1153,54 @@ function OnboardingAddDataSource(): JSX.Element { destroyOnClose >
    - {}} - onNext={(): void => setShowInviteTeamMembersModal(false)} - onClose={(): void => setShowInviteTeamMembersModal(false)} + { + void logEvent( + `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_SUCCESS}`, + {}, + ); + setShowInviteTeamMembersModal(false); + + toast.success('Invites sent successfully', { position: 'top-center' }); + }} + onPartialSuccess={(): void => { + void logEvent( + `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_PARTIAL_SUCCESS}`, + {}, + ); + }} + onAllFailed={(): void => { + void logEvent( + `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_FAILED}`, + {}, + ); + }} + renderFooter={({ submit, canSubmit, isSubmitting }): JSX.Element => ( +
    + setShowInviteTeamMembersModal(false)} + > + Cancel + + { + void logEvent( + `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_SEND_CLICKED}`, + {}, + ); + void submit(); + }} + disabled={!canSubmit} + loading={isSubmitting} + suffix={} + > + Send Invites + +
    + )} />
    diff --git a/frontend/src/container/OnboardingV2Container/InviteTeamMembers/InviteTeamMembers.styles.scss b/frontend/src/container/OnboardingV2Container/InviteTeamMembers/InviteTeamMembers.styles.scss deleted file mode 100644 index db407ddc6b..0000000000 --- a/frontend/src/container/OnboardingV2Container/InviteTeamMembers/InviteTeamMembers.styles.scss +++ /dev/null @@ -1,116 +0,0 @@ -.team-member-container { - display: flex; - align-items: center; - - .invite-team-members-form { - padding: 16px 0px; - } - - .team-member-email-input { - width: 80%; - background-color: var(--l1-background); - border-top-right-radius: 0px; - border-bottom-right-radius: 0px; - - .ant-input, - .ant-input-group-addon { - background-color: var(--l1-background) !important; - border-right: 0px; - border-top-right-radius: 0px; - border-bottom-right-radius: 0px; - } - } - - .team-member-role-select { - width: 20%; - - .ant-select-selector { - border: 1px solid var(--l1-border); - border-top-left-radius: 0px; - border-bottom-left-radius: 0px; - - border-top-right-radius: 0px; - border-bottom-right-radius: 0px; - } - } - - .remove-team-member-button { - border-top-left-radius: 0px; - border-bottom-left-radius: 0px; - } -} - -.invite-team-members-container { - display: flex; - flex-direction: column; - gap: 8px; - - .invite-team-members-add-another-member-container { - margin: 16px 0px; - display: flex; - justify-content: flex-end; - } - - .next-prev-container { - display: flex; - justify-content: flex-end; - gap: 8px; - - margin-top: 16px; - } - - .error-message-container, - .success-message-container, - .partially-sent-invites-container { - border-radius: 4px; - width: 100%; - display: flex; - align-items: center; - - .error-message, - .success-message { - font-size: 12px; - font-weight: 400; - - display: flex; - align-items: center; - gap: 8px; - } - } - - .invite-users-error-message-container, - .invite-users-success-message-container { - display: flex; - flex-direction: column; - align-items: flex-start; - gap: 8px; - - .success-message { - color: var(--bg-success-500, #00b37e); - } - } - - .partially-sent-invites-container { - margin-top: 16px; - padding: 8px; - border: 1px solid var(--l1-border); - background-color: var(--l1-background); - - display: flex; - flex-direction: column; - align-items: flex-start; - gap: 8px; - box-sizing: border-box; - - .partially-sent-invites-message { - color: var(--bg-warning-500, #fbbd23); - - font-size: 12px; - font-weight: 400; - - display: flex; - align-items: center; - gap: 8px; - } - } -} diff --git a/frontend/src/container/OnboardingV2Container/InviteTeamMembers/InviteTeamMembers.tsx b/frontend/src/container/OnboardingV2Container/InviteTeamMembers/InviteTeamMembers.tsx deleted file mode 100644 index 11153a0be8..0000000000 --- a/frontend/src/container/OnboardingV2Container/InviteTeamMembers/InviteTeamMembers.tsx +++ /dev/null @@ -1,298 +0,0 @@ -import { useCallback, useEffect, useState } from 'react'; -import { useMutation } from 'react-query'; -import { Color } from '@signozhq/design-tokens'; -import { Button, Input, Select } from 'antd'; -import { Typography } from '@signozhq/ui/typography'; -import logEvent from 'api/common/logEvent'; -import inviteUsers from 'api/v1/invite/bulk/create'; -import { useNotifications } from 'hooks/useNotifications'; -import { cloneDeep, debounce, isEmpty } from 'lodash-es'; -import { - ArrowRight, - CircleCheck, - Plus, - TriangleAlert, - X, -} from '@signozhq/icons'; -import APIError from 'types/api/error'; -import { getBaseUrl } from 'utils/basePath'; -import { v4 as uuid } from 'uuid'; - -import './InviteTeamMembers.styles.scss'; - -interface TeamMember { - email: string; - role: string; - name: string; - frontendBaseUrl: string; - id: string; -} - -interface InviteTeamMembersProps { - isLoading: boolean; - teamMembers: TeamMember[] | null; - setTeamMembers: (teamMembers: TeamMember[]) => void; - onNext: () => void; - onClose: () => void; -} - -const ONBOARDING_V3_ANALYTICS_EVENTS_MAP = { - BASE: 'Onboarding V3', - INVITE_TEAM_MEMBER_BUTTON_CLICKED: 'Send invites clicked', - INVITE_TEAM_MEMBER_SUCCESS: 'Invite team members success', - INVITE_TEAM_MEMBER_PARTIAL_SUCCESS: 'Invite team members partial success', - INVITE_TEAM_MEMBER_FAILED: 'Invite team members failed', -}; - -function InviteTeamMembers({ - isLoading, - teamMembers, - setTeamMembers, - onNext, - onClose, -}: InviteTeamMembersProps): JSX.Element { - const [teamMembersToInvite, setTeamMembersToInvite] = useState< - TeamMember[] | null - >(teamMembers); - const [emailValidity, setEmailValidity] = useState>( - {}, - ); - const [hasInvalidEmails, setHasInvalidEmails] = useState(false); - const { notifications } = useNotifications(); - - const defaultTeamMember: TeamMember = { - email: '', - role: 'EDITOR', - name: '', - frontendBaseUrl: getBaseUrl(), - id: '', - }; - - useEffect(() => { - if (isEmpty(teamMembers)) { - const teamMember = { - ...defaultTeamMember, - id: uuid(), - }; - - setTeamMembersToInvite([teamMember]); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [teamMembers]); - - const handleAddTeamMember = (): void => { - const newTeamMember = { - ...defaultTeamMember, - id: uuid(), - }; - setTeamMembersToInvite((prev) => [...(prev || []), newTeamMember]); - }; - - const handleRemoveTeamMember = (id: string): void => { - setTeamMembersToInvite((prev) => (prev || []).filter((m) => m.id !== id)); - }; - - // Validation function to check all users - const validateAllUsers = (): boolean => { - let isValid = true; - - const updatedValidity: Record = {}; - - teamMembersToInvite?.forEach((member) => { - const emailValid = /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(member.email); - if (!emailValid || !member.email) { - isValid = false; - setHasInvalidEmails(true); - } - updatedValidity[member.id!] = emailValid; - }); - - setEmailValidity(updatedValidity); - - return isValid; - }; - - const handleInviteUsersSuccess = (): void => { - logEvent( - `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_SUCCESS}`, - { - teamMembers: teamMembersToInvite, - }, - ); - setTimeout(() => { - onNext(); - }, 1000); - }; - - const { mutate: sendInvites, isLoading: isSendingInvites } = useMutation( - inviteUsers, - { - onSuccess: (): void => { - handleInviteUsersSuccess(); - notifications.success({ - message: 'Invites sent successfully!', - }); - }, - onError: (error: APIError): void => { - logEvent( - `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_FAILED}`, - { - teamMembers: teamMembersToInvite, - error, - }, - ); - notifications.error({ - message: error.getErrorCode(), - description: error.getErrorMessage(), - }); - }, - }, - ); - - const handleNext = (): void => { - if (validateAllUsers()) { - setTeamMembers(teamMembersToInvite || []); - - logEvent( - `${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.BASE}: ${ONBOARDING_V3_ANALYTICS_EVENTS_MAP?.INVITE_TEAM_MEMBER_BUTTON_CLICKED}`, - { - teamMembers: teamMembersToInvite, - }, - ); - - setHasInvalidEmails(false); - sendInvites({ - invites: teamMembersToInvite || [], - }); - } - }; - - // eslint-disable-next-line react-hooks/exhaustive-deps - const debouncedValidateEmail = useCallback( - debounce((email: string, memberId: string) => { - const isValid = /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email); - setEmailValidity((prev) => ({ ...prev, [memberId]: isValid })); - }, 500), - [], - ); - - const handleEmailChange = ( - e: React.ChangeEvent, - member: TeamMember, - ): void => { - const { value } = e.target; - const updatedMembers = cloneDeep(teamMembersToInvite || []); - - const memberToUpdate = updatedMembers.find((m) => m.id === member.id); - if (memberToUpdate) { - memberToUpdate.email = value; - setTeamMembersToInvite(updatedMembers); - debouncedValidateEmail(value, member.id!); - } - }; - - const handleRoleChange = (role: string, member: TeamMember): void => { - const updatedMembers = cloneDeep(teamMembersToInvite || []); - const memberToUpdate = updatedMembers.find((m) => m.id === member.id); - if (memberToUpdate) { - memberToUpdate.role = role; - setTeamMembersToInvite(updatedMembers); - } - }; - - return ( -
    -
    -
    -
    - {teamMembersToInvite?.map((member) => ( -
    - ): void => - handleEmailChange(e, member) - } - addonAfter={ - emailValidity[member.id!] === undefined ? null : emailValidity[ - member.id! - ] ? ( - - ) : ( - - ) - } - /> - - - {teamMembersToInvite?.length > 1 && ( -
    - ))} -
    - -
    - -
    -
    - - {hasInvalidEmails && ( -
    - - Please enter valid emails for all team - members - -
    - )} -
    -
    - - - -
    -
    - ); -} - -export default InviteTeamMembers; diff --git a/frontend/src/container/OnboardingV2Container/OnboardingV2.styles.scss b/frontend/src/container/OnboardingV2Container/OnboardingV2.styles.scss index 82b8b82848..aca65e9dc5 100644 --- a/frontend/src/container/OnboardingV2Container/OnboardingV2.styles.scss +++ b/frontend/src/container/OnboardingV2Container/OnboardingV2.styles.scss @@ -1220,6 +1220,14 @@ .request-data-source-modal-input { margin-top: 8px; } + + .invite-team-member-modal-footer { + display: flex; + justify-content: flex-end; + gap: var(--spacing-8); + border-top: 1px solid var(--l1-border); + padding-top: var(--spacing-6); + } } .request-data-source-modal { diff --git a/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/Providers.styles.scss b/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/Providers.styles.scss index 68348967fb..1246bfc48b 100644 --- a/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/Providers.styles.scss +++ b/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/Providers.styles.scss @@ -79,7 +79,7 @@ margin-bottom: 12px; } - input, + input:not(.ant-select-selection-search-input), textarea { height: 32px; background: var(--l2-background) !important; diff --git a/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.styles.scss b/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.styles.scss index 9db3b087d1..71593fec10 100644 --- a/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.styles.scss +++ b/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.styles.scss @@ -111,31 +111,9 @@ &__select { width: 100%; - &.ant-select { - .ant-select-selector { - height: 32px; - background: var(--l2-background) !important; - border: 1px solid var(--l2-border) !important; - border-radius: 2px; - color: var(--l2-foreground) !important; - - .ant-select-selection-item { - color: var(--l2-foreground) !important; - } - } - - &:hover .ant-select-selector { - border-color: var(--l2-border) !important; - } - - &.ant-select-focused .ant-select-selector { - border-color: var(--primary) !important; - box-shadow: none !important; - } - - .ant-select-arrow { - color: var(--l2-foreground); - } + .ant-select-selection-search { + inset-inline-start: var(--padding-2) !important; + inset-inline-end: var(--padding-2) !important; } } @@ -185,7 +163,7 @@ &--role { flex: 1; - min-width: 120px; + min-width: 180px; } } @@ -272,7 +250,7 @@ } // todo: https://github.com/SigNoz/components/issues/116 - input { + input:not(.ant-select-selection-search-input) { height: 32px; background: var(--l2-background) !important; border: 1px solid var(--l2-border) !important; diff --git a/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.tsx b/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.tsx index 76f2c3e3b5..4c4f3629e3 100644 --- a/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.tsx +++ b/frontend/src/container/OrganizationSettings/AuthDomain/CreateEdit/Providers/components/RoleMappingSection.tsx @@ -11,23 +11,20 @@ import { import { Button } from '@signozhq/ui/button'; import { Checkbox } from '@signozhq/ui/checkbox'; import { Input } from '@signozhq/ui/input'; -import { Collapse, Form, Select, Tooltip } from 'antd'; +import { Collapse, Form, Tooltip } from 'antd'; +import RolesSelect, { useRoles } from 'components/RolesSelect'; import { useCollapseSectionErrors } from 'hooks/useCollapseSectionErrors'; import './RoleMappingSection.styles.scss'; -const ROLE_OPTIONS = [ - { value: 'VIEWER', label: 'VIEWER' }, - { value: 'EDITOR', label: 'EDITOR' }, - { value: 'ADMIN', label: 'ADMIN' }, -]; - interface RoleMappingSectionProps { fieldNamePrefix: string[]; isExpanded?: boolean; onExpandChange?: (expanded: boolean) => void; } +const SIGNOZ_VIEWER_ROLE = 'signoz-viewer'; + function RoleMappingSection({ fieldNamePrefix, isExpanded, @@ -38,6 +35,7 @@ function RoleMappingSection({ [...fieldNamePrefix, 'useRoleAttribute'], form, ); + const { roles, isLoading, isError, error, refetch } = useRoles(); // Support both controlled and uncontrolled modes const [internalExpanded, setInternalExpanded] = useState(false); @@ -108,19 +106,26 @@ function RoleMappingSection({
    - document.body} /> @@ -197,7 +208,9 @@ function RoleMappingSection({ + ), +})); + +// These are heavy real-timer integration tests (antd Select dropdown render + +// form.validateFields() + a react-query mutation, all driven through userEvent). +// Under a CPU-saturated parallel `jest` run the wall-clock roughly triples, which +// pushes the longest tests past the 5000ms default and makes them flaky. Give the +// whole file a wider budget (matches LogsPanelComponent.test.tsx). +jest.setTimeout(20000); + +const ROLES_ENDPOINT = '*/api/v1/roles'; + +type User = ReturnType; + +// antd renders pointer-events:none on parts of its Select, so disable the +// userEvent pointer-events guard (mirrors CreateEdit.test.tsx). +const setupUser = (): User => userEvent.setup({ pointerEventsCheck: 0 }); + +function getRole(name: string): (typeof managedRoles)[number] { + const role = managedRoles.find((r) => r.name === name); + if (!role) { + throw new Error(`missing mock role: ${name}`); + } + return role; +} + +const viewerRole = getRole('signoz-viewer'); +const editorRole = getRole('signoz-editor'); + +function mockRoles( + response: Record = listRolesSuccessResponse, + status = 200, +): { count: () => number } { + let requested = 0; + server.use( + rest.get(ROLES_ENDPOINT, (_req, res, ctx) => { + requested += 1; + return res(ctx.status(status), ctx.json(response)); + }), + ); + return { count: (): number => requested }; +} + +function captureUpdatePayload(): { get: () => any } { + let payload: unknown = null; + server.use( + rest.put(AUTH_DOMAINS_UPDATE_ENDPOINT, async (req, res, ctx) => { + payload = await req.json(); + return res(ctx.status(200), ctx.json(mockUpdateSuccessResponse)); + }), + ); + return { get: (): any => payload }; +} + +const expandRoleMapping = (user: User): Promise => + user.click(screen.getByText(/role mapping \(advanced\)/i)); + +const openDefaultRoleSelect = (user: User): Promise => + user.click(screen.getByLabelText(/default role/i)); + +const saveChanges = (user: User): Promise => + user.click(screen.getByRole('button', { name: /save changes/i })); + +describe('CreateEdit — role mapping uses API roles', () => { + afterEach(() => { + server.resetHandlers(); + }); + + it('fetches the roles list from the API when the form mounts', async () => { + const roles = mockRoles(); + + render( + , + ); + + await waitFor(() => expect(roles.count()).toBeGreaterThan(0)); + }); + + it('renders the default-role options from the API (managed + custom), not the old hardcoded VIEWER/EDITOR/ADMIN', async () => { + const user = setupUser(); + mockRoles(); + + // mockSamlAuthDomain has no stored defaultRole, so nothing stale (e.g. + // "VIEWER") is rendered as a selected tag to pollute the title lookups. + render( + , + ); + + await expandRoleMapping(user); + + // Open the Select and wait for the async roles fetch to populate it. + await openDefaultRoleSelect(user); + await screen.findByTitle(allRoles[0].name); + + // Every role returned by the API is offered as an option, including the + // custom (non-managed) roles — the whole point of the refactor. Use + // getAllByTitle: the preselected default role also renders its name on + // the selection item, so a role may legitimately appear more than once. + allRoles.forEach((role) => { + expect(screen.getAllByTitle(role.name).length).toBeGreaterThan(0); + }); + + // The old hardcoded uppercase role values must NOT appear as options. + expect(screen.queryByTitle('VIEWER')).not.toBeInTheDocument(); + expect(screen.queryByTitle('EDITOR')).not.toBeInTheDocument(); + expect(screen.queryByTitle('ADMIN')).not.toBeInTheDocument(); + }); + + it('submits the selected role name (not the role id) as defaultRole', async () => { + const user = setupUser(); + mockRoles(); + const payload = captureUpdatePayload(); + + render( + , + ); + + await expandRoleMapping(user); + + await openDefaultRoleSelect(user); + await user.click(await screen.findByTitle(editorRole.name)); + + await saveChanges(user); + + await waitFor(() => expect(payload.get()).not.toBeNull()); + + // SSO role mapping matches roles by name, so the payload carries the + // role *name*, not the opaque id. + expect(payload.get().config.roleMapping.defaultRole).toBe(editorRole.name); + expect(payload.get().config.roleMapping.defaultRole).not.toBe(editorRole.id); + }); + + it('defaults a fresh role mapping to the signoz-viewer role name', async () => { + const user = setupUser(); + const roles = mockRoles(); + const payload = captureUpdatePayload(); + + // mockSamlAuthDomain has no roleMapping, so the defaultRole field falls + // back to the Form.Item initialValue (viewerRole.name). That initialValue + // is only applied when the field mounts, so the roles fetch MUST resolve + // before the panel is expanded — otherwise viewerRole is still undefined. + render( + , + ); + + await waitFor(() => expect(roles.count()).toBeGreaterThan(0)); + // Flush the react-query commit so `useRoles` exposes the loaded roles + // before the collapse panel (and thus the default-role field) mounts. + await screen.findByText(/edit saml authentication/i); + + await expandRoleMapping(user); + await screen.findByText(/default role/i); + + await saveChanges(user); + + await waitFor(() => expect(payload.get()).not.toBeNull()); + + expect(payload.get().config.roleMapping.defaultRole).toBe(viewerRole.name); + expect(payload.get().config.roleMapping.defaultRole).not.toBe(viewerRole.id); + }); + + it('still defaults to signoz-viewer when the roles fetch returns empty', async () => { + const user = setupUser(); + // signoz-viewer is a managed role that always exists server-side, so even + // a degenerate/empty roles response must not strip the hardcoded default. + mockRoles({ status: 'success', data: [] }); + const payload = captureUpdatePayload(); + + render( + , + ); + + // Section still renders without crashing even though the fetch was empty. + await expandRoleMapping(user); + await expect(screen.findByText(/default role/i)).resolves.toBeInTheDocument(); + + await saveChanges(user); + + await waitFor(() => expect(payload.get()).not.toBeNull()); + + // The Form.Item initialValue (signoz-viewer) survives an empty roles list. + expect(payload.get().config.roleMapping.defaultRole).toBe(viewerRole.name); + }); + + it('loads a stored role mapping by role name and round-trips it on save', async () => { + const user = setupUser(); + mockRoles(); + const payload = captureUpdatePayload(); + + // mockDomainWithRoleMapping stores defaultRole "signoz-editor" plus three + // group mappings, all keyed by role *name*. Editing must surface each + // stored value as the matching option and submit it unchanged — the + // backward-compatible read path for already-saved SSO domains. + render( + , + ); + + await expandRoleMapping(user); + + // The stored default role renders as a real selection, not a raw token. + await waitFor(() => + expect(screen.getAllByTitle(editorRole.name).length).toBeGreaterThan(0), + ); + + await saveChanges(user); + + await waitFor(() => expect(payload.get()).not.toBeNull()); + + expect(payload.get().config.roleMapping.defaultRole).toBe(editorRole.name); + expect(payload.get().config.roleMapping.groupMappings).toStrictEqual({ + 'admin-group': 'signoz-admin', + 'dev-team': 'signoz-editor', + viewers: 'signoz-viewer', + }); + }); + + it('shows an error state in the default-role select when the roles request fails', async () => { + const user = setupUser(); + mockRoles( + { error: { code: 'internal_error', message: 'boom', url: '' } }, + 500, + ); + + render( + , + ); + + await expandRoleMapping(user); + + // Open the select and confirm the error UI (with retry) is surfaced + // instead of crashing the form. The error message comes straight from + // the failed request; the Retry affordance is always present. + await openDefaultRoleSelect(user); + + await expect(screen.findByTitle('Retry')).resolves.toBeInTheDocument(); + expect(screen.getByText('boom')).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/CreateEdit.sanitization.test.tsx b/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/CreateEdit.sanitization.test.tsx index 3f279d19f8..4eb22699f3 100644 --- a/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/CreateEdit.sanitization.test.tsx +++ b/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/CreateEdit.sanitization.test.tsx @@ -186,9 +186,9 @@ describe('CreateEdit — payload sanitization', () => { expect(payload.config.roleMapping?.useRoleAttribute).toBe(false); expect(payload.config.roleMapping?.groupMappings).toStrictEqual({ - 'admin-group': 'ADMIN', - 'dev-team': 'EDITOR', - viewers: 'VIEWER', + 'admin-group': 'signoz-admin', + 'dev-team': 'signoz-editor', + viewers: 'signoz-viewer', }); }); }); diff --git a/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/mocks.ts b/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/mocks.ts index 324641571e..3a247a3fda 100644 --- a/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/mocks.ts +++ b/frontend/src/container/OrganizationSettings/AuthDomain/__tests__/mocks.ts @@ -75,12 +75,12 @@ export const mockDomainWithRoleMapping: AuthtypesGettableAuthDomainDTO = { samlCert: 'MOCK_CERTIFICATE', }, roleMapping: { - defaultRole: 'EDITOR', + defaultRole: 'signoz-editor', useRoleAttribute: false, groupMappings: { - 'admin-group': 'ADMIN', - 'dev-team': 'EDITOR', - viewers: 'VIEWER', + 'admin-group': 'signoz-admin', + 'dev-team': 'signoz-editor', + viewers: 'signoz-viewer', }, }, }, @@ -103,7 +103,7 @@ export const mockDomainWithDirectRoleAttribute: AuthtypesGettableAuthDomainDTO = clientSecret: 'direct-role-client-secret', }, roleMapping: { - defaultRole: 'VIEWER', + defaultRole: 'signoz-viewer', useRoleAttribute: true, }, }, diff --git a/frontend/src/container/PanelWrapper/PiePanelWrapper.tsx b/frontend/src/container/PanelWrapper/PiePanelWrapper.tsx index ad60cfe661..d772e3275e 100644 --- a/frontend/src/container/PanelWrapper/PiePanelWrapper.tsx +++ b/frontend/src/container/PanelWrapper/PiePanelWrapper.tsx @@ -1,4 +1,4 @@ -import { useRef, useState } from 'react'; +import { useMemo, useRef, useState } from 'react'; import { Color } from '@signozhq/design-tokens'; import { Group } from '@visx/group'; import { Pie } from '@visx/shape'; @@ -8,12 +8,10 @@ import { themeColors } from 'constants/theme'; import { getPieChartClickData } from 'container/QueryTable/Drilldown/drilldownUtils'; import useGraphContextMenu from 'container/QueryTable/Drilldown/useGraphContextMenu'; import { useIsDarkMode } from 'hooks/useDarkMode'; -import getLabelName from 'lib/getLabelName'; -import { generateColor } from 'lib/uPlotLib/utils/generateColor'; -import { isNaN } from 'lodash-es'; import ContextMenu, { useCoordinates } from 'periscope/components/ContextMenu'; import { PanelWrapperProps, TooltipData } from './panelWrapper.types'; +import { preparePieChartData } from './preparePieChartData'; import { lightenColor, tooltipStyles } from './utils'; import './PiePanelWrapper.styles.scss'; @@ -44,37 +42,15 @@ function PiePanelWrapper({ detectBounds: true, }); - const panelData = queryResponse.data?.payload?.data?.result || []; - const isDarkMode = useIsDarkMode(); - let pieChartData: { - label: string; - value: string; - color: string; - record: any; - }[] = [].concat( - ...(panelData - .map((d) => { - const label = getLabelName(d.metric, d.queryName || '', d.legend || ''); - return { - label, - value: d?.values?.[0]?.[1], - record: d, - color: - widget?.customLegendColors?.[label] || - generateColor( - label, - isDarkMode ? themeColors.chartcolors : themeColors.lightModeColor, - ), - }; - }) - .filter((d) => d !== undefined) as never[]), - ); - - pieChartData = pieChartData.filter( - (arc) => - arc.value && !isNaN(parseFloat(arc.value)) && parseFloat(arc.value) > 0, + const pieChartData = useMemo( + () => + preparePieChartData(queryResponse.data?.payload, { + customLegendColors: widget?.customLegendColors, + colorMap: isDarkMode ? themeColors.chartcolors : themeColors.lightModeColor, + }), + [queryResponse.data?.payload, widget?.customLegendColors, isDarkMode], ); let size = 0; diff --git a/frontend/src/container/PanelWrapper/__tests__/preparePieChartData.test.ts b/frontend/src/container/PanelWrapper/__tests__/preparePieChartData.test.ts new file mode 100644 index 0000000000..cfe8c01978 --- /dev/null +++ b/frontend/src/container/PanelWrapper/__tests__/preparePieChartData.test.ts @@ -0,0 +1,185 @@ +import { themeColors } from 'constants/theme'; +import { MetricRangePayloadProps } from 'types/api/metrics/getQueryRange'; +import { QueryData, QueryDataV3 } from 'types/api/widgets/getQuery'; + +import { preparePieChartData } from '../preparePieChartData'; + +const options = { colorMap: themeColors.chartcolors }; + +/** + * Mirrors a query-range payload: the (possibly collapsed) time-series `result` + * plus the scalar table nested under `newResult` (as getQueryResults produces it). + */ +function makePayload( + result: QueryData[], + tables: QueryDataV3[], +): MetricRangePayloadProps { + return { + data: { + result, + resultType: 'scalar', + newResult: { data: { result: tables, resultType: 'scalar' } }, + }, + } as MetricRangePayloadProps; +} + +function tableEntry( + columns: NonNullable['columns'], + rows: NonNullable['rows'], + overrides: Partial = {}, +): QueryDataV3 { + return { + queryName: 'A', + legend: '', + series: null, + list: null, + table: { columns, rows }, + ...overrides, + } as QueryDataV3; +} + +describe('preparePieChartData', () => { + it('renders a slice per value column for a multi-column ClickHouse scalar', () => { + // SELECT count() AS col1, sum(value) AS col2 — the backend collapses the + // time-series result onto col1; the full data lives in the scalar table. + const payload = makePayload( + [ + { + metric: {}, + queryName: 'A', + legend: '', + values: [[0, '23399927']], + } as QueryData, + ], + [ + tableEntry( + [ + { name: 'col1', queryName: 'A', isValueColumn: true, id: 'col1' }, + { name: 'col2', queryName: 'A', isValueColumn: true, id: 'col2' }, + ], + [{ data: { col1: 23399927, col2: 588691297 } }], + ), + ], + ); + + const slices = preparePieChartData(payload, options); + + expect(slices).toHaveLength(2); + expect(slices.map((s) => [s.label, s.value])).toStrictEqual([ + ['col1', '23399927'], + ['col2', '588691297'], + ]); + }); + + it('prefixes the group when multiple value columns are grouped', () => { + const payload = makePayload( + [], + [ + tableEntry( + [ + { name: 'env', queryName: 'A', isValueColumn: false, id: 'env' }, + { name: 'col1', queryName: 'A', isValueColumn: true, id: 'col1' }, + { name: 'col2', queryName: 'A', isValueColumn: true, id: 'col2' }, + ], + [{ data: { env: 'prod', col1: 10, col2: 20 } }], + ), + ], + ); + + const slices = preparePieChartData(payload, options); + + expect(slices.map((s) => s.label)).toStrictEqual([ + 'prod · col1', + 'prod · col2', + ]); + expect(slices[0].record.metric).toStrictEqual({ env: 'prod' }); + }); + + it('drops non-positive and non-numeric values', () => { + const payload = makePayload( + [], + [ + tableEntry( + [ + { name: 'col1', queryName: 'A', isValueColumn: true, id: 'col1' }, + { name: 'col2', queryName: 'A', isValueColumn: true, id: 'col2' }, + { name: 'col3', queryName: 'A', isValueColumn: true, id: 'col3' }, + ], + [{ data: { col1: 5, col2: 0, col3: 'n/a' } }], + ), + ], + ); + + const slices = preparePieChartData(payload, options); + + expect(slices.map((s) => s.label)).toStrictEqual(['col1']); + }); + + it('keeps the series path for a single value column (grouped panel)', () => { + // One value column → the time-series result is authoritative (one slice per + // group), so existing behaviour is preserved. + const payload = makePayload( + [ + { + metric: { 'service.name': 'adservice' }, + queryName: 'A', + legend: 'adservice', + values: [[0, '100']], + } as QueryData, + { + metric: { 'service.name': 'cartservice' }, + queryName: 'A', + legend: 'cartservice', + values: [[0, '200']], + } as QueryData, + ], + [ + tableEntry( + [ + { + name: 'service.name', + queryName: 'A', + isValueColumn: false, + id: 'service.name', + }, + { name: 'count', queryName: 'A', isValueColumn: true, id: 'A' }, + ], + [ + { data: { 'service.name': 'adservice', A: 100 } }, + { data: { 'service.name': 'cartservice', A: 200 } }, + ], + ), + ], + ); + + const slices = preparePieChartData(payload, options); + + expect(slices.map((s) => [s.label, s.value])).toStrictEqual([ + ['adservice', '100'], + ['cartservice', '200'], + ]); + }); + + it('uses the legacy series result when there is no scalar table', () => { + const payload = makePayload( + [ + { + metric: { 'service.name': 'adservice' }, + queryName: 'A', + legend: '{{service.name}}', + values: [[1000, '42']], + } as QueryData, + ], + [], + ); + + const slices = preparePieChartData(payload, options); + + expect(slices).toHaveLength(1); + expect(slices[0].value).toBe('42'); + }); + + it('returns no slices for an empty payload', () => { + expect(preparePieChartData(undefined, options)).toStrictEqual([]); + }); +}); diff --git a/frontend/src/container/PanelWrapper/preparePieChartData.ts b/frontend/src/container/PanelWrapper/preparePieChartData.ts new file mode 100644 index 0000000000..6455a1f006 --- /dev/null +++ b/frontend/src/container/PanelWrapper/preparePieChartData.ts @@ -0,0 +1,144 @@ +import getLabelName from 'lib/getLabelName'; +import { generateColor } from 'lib/uPlotLib/utils/generateColor'; +import { isNaN } from 'lodash-es'; +import { MetricRangePayloadProps } from 'types/api/metrics/getQueryRange'; +import { QueryData, QueryDataV3 } from 'types/api/widgets/getQuery'; + +export interface PieChartSlice { + label: string; + value: string; + color: string; + record: { + queryName: string; + legend?: string; + /** Group-by labels, used for drilldown; absent when the slice has no group. */ + metric?: QueryData['metric']; + }; +} + +interface PreparePieChartDataOptions { + customLegendColors?: Record; + colorMap: Record; +} + +const colorFor = ( + label: string, + { customLegendColors, colorMap }: PreparePieChartDataOptions, +): string => customLegendColors?.[label] || generateColor(label, colorMap); + +const isPositive = (value: string): boolean => + !!value && !isNaN(parseFloat(value)) && parseFloat(value) > 0; + +/** + * Time-series result: one slice per series, value = first datapoint. This is the + * original pie behaviour — kept verbatim (same label/value/colour/record) so + * single-value and grouped panels are unaffected. + */ +function slicesFromSeries( + result: QueryData[], + options: PreparePieChartDataOptions, +): PieChartSlice[] { + return result + .filter((d) => d?.values?.[0]?.[1] !== undefined) + .map((d) => { + const label = getLabelName(d.metric, d.queryName || '', d.legend || ''); + return { + label, + value: d.values[0][1], + color: colorFor(label, options), + record: d, + }; + }); +} + +/** + * V5 scalar table: one slice per (row × value column). With more than one value + * column the column name keeps the slices distinct, so a ClickHouse query like + * `count() AS col1, sum() AS col2` renders a slice per column instead of + * collapsing onto the first; group-by columns become the slice label. + */ +function slicesFromTables( + tables: QueryDataV3[], + options: PreparePieChartDataOptions, +): PieChartSlice[] { + const slices: PieChartSlice[] = []; + + tables.forEach((entry) => { + const { table } = entry; + if (!table?.columns?.length || !table?.rows?.length) { + return; + } + + const valueColumns = table.columns.filter((column) => column.isValueColumn); + if (valueColumns.length === 0) { + return; + } + const labelColumns = table.columns.filter((column) => !column.isValueColumn); + const hasMultipleValueColumns = valueColumns.length > 1; + + table.rows.forEach((row) => { + const groupLabel = labelColumns + .map((column) => row.data[column.id || column.name]) + .filter((part) => part != null) + .map(String) + .join(', '); + // Drilldown filters by group-by labels; leave it undefined when there + // are none (e.g. a ClickHouse query) so no filterless menu is offered. + const metric = labelColumns.length + ? labelColumns.reduce>((acc, column) => { + acc[column.name] = String(row.data[column.id || column.name]); + return acc; + }, {}) + : undefined; + + valueColumns.forEach((column) => { + let label: string; + if (hasMultipleValueColumns) { + label = groupLabel ? `${groupLabel} · ${column.name}` : column.name; + } else { + label = groupLabel || entry.legend || entry.queryName || ''; + } + + slices.push({ + label, + value: String(row.data[column.id || column.name]), + color: colorFor(label, options), + record: { queryName: entry.queryName, legend: entry.legend, metric }, + }); + }); + }); + }); + + return slices; +} + +/** + * Builds pie slices from a query-range payload, dropping non-positive/non-numeric + * values. + * + * A scalar response with several value columns (e.g. a ClickHouse + * `count() AS col1, sum() AS col2`) collapses to a single series in + * `data.result` — only the first value column survives. The full data is kept in + * the scalar table under `newResult`, so in that case slices are built from the + * table (one per value column). Otherwise the legacy time-series result is used, + * preserving existing behaviour for single-value and grouped panels. + */ +export function preparePieChartData( + payload: MetricRangePayloadProps | undefined, + options: PreparePieChartDataOptions, +): PieChartSlice[] { + const tables = (payload?.data?.newResult?.data?.result || []).filter( + (entry) => entry?.table?.rows?.length, + ); + const hasMultipleValueColumns = tables.some( + (entry) => + (entry.table?.columns || []).filter((column) => column.isValueColumn) + .length > 1, + ); + + const slices = hasMultipleValueColumns + ? slicesFromTables(tables, options) + : slicesFromSeries(payload?.data?.result || [], options); + + return slices.filter((slice) => isPositive(slice.value)); +} diff --git a/frontend/src/container/RolesSettings/RolesComponents/RolesListingTable.module.scss b/frontend/src/container/RolesSettings/RolesComponents/RolesListingTable.module.scss index 2b04702f6b..4b36f28957 100644 --- a/frontend/src/container/RolesSettings/RolesComponents/RolesListingTable.module.scss +++ b/frontend/src/container/RolesSettings/RolesComponents/RolesListingTable.module.scss @@ -1,7 +1,6 @@ .rolesListingTable { margin-top: 12px; border-radius: 4px; - overflow: hidden; } .scrollContainer { diff --git a/frontend/src/container/RolesSettings/RolesSettings.module.scss b/frontend/src/container/RolesSettings/RolesSettings.module.scss index 175a759228..ec4b1a5973 100644 --- a/frontend/src/container/RolesSettings/RolesSettings.module.scss +++ b/frontend/src/container/RolesSettings/RolesSettings.module.scss @@ -40,6 +40,7 @@ .rolesSettingsContent { padding: 0 16px; + padding-bottom: 16px; } .rolesSettingsToolbar { diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/__tests__/useResolvedVariables.test.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/__tests__/useResolvedVariables.test.tsx new file mode 100644 index 0000000000..192fc2c512 --- /dev/null +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/__tests__/useResolvedVariables.test.tsx @@ -0,0 +1,67 @@ +import { renderHook } from '@testing-library/react'; +import type { + DashboardtypesGettableDashboardV2DTO, + DashboardtypesVariableDTO, +} from 'api/generated/services/sigNoz.schemas'; + +import { selectResolvedVariables } from '../../store/slices/variableSelectionSlice'; +import { useDashboardStore } from '../../store/useDashboardStore'; +import { useResolvedVariables } from '../useResolvedVariables'; + +// A text variable is the simplest envelope (no list plugin); the builder's full +// type/value matrix is covered in buildVariablesPayload.test.ts. The envelope is +// cast at the boundary — its kind discriminant is the literal 'TextVariable'. +function textVariable(name: string, value: string): DashboardtypesVariableDTO { + return { + kind: 'TextVariable', + spec: { name, value, display: { name } }, + } as unknown as DashboardtypesVariableDTO; +} + +function dashboard( + id: string, + variables: DashboardtypesVariableDTO[], +): DashboardtypesGettableDashboardV2DTO { + return { + id, + spec: { variables }, + } as unknown as DashboardtypesGettableDashboardV2DTO; +} + +describe('useResolvedVariables', () => { + afterEach(() => { + useDashboardStore.setState({ variableValues: {}, resolvedVariables: {} }); + }); + + it('publishes the resolved V5 payload for the dashboard to the store', () => { + renderHook(() => + useResolvedVariables(dashboard('d1', [textVariable('env', 'prod')])), + ); + + expect( + selectResolvedVariables('d1')(useDashboardStore.getState()), + ).toStrictEqual({ env: { type: 'text', value: 'prod' } }); + }); + + it('reflects the runtime selection over the configured default', () => { + useDashboardStore + .getState() + .setVariableValues('d2', { env: { value: 'staging', allSelected: false } }); + + renderHook(() => + useResolvedVariables(dashboard('d2', [textVariable('env', 'prod')])), + ); + + expect( + selectResolvedVariables('d2')(useDashboardStore.getState()), + ).toStrictEqual({ env: { type: 'text', value: 'staging' } }); + }); + + it('publishes an empty payload when the dashboard has no variables', () => { + renderHook(() => useResolvedVariables(dashboard('d3', []))); + + expect( + selectResolvedVariables('d3')(useDashboardStore.getState()), + ).toStrictEqual({}); + }); +}); diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/usePanelQuery.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/usePanelQuery.ts index e7cc207e2e..b5c8d99369 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/usePanelQuery.ts +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/usePanelQuery.ts @@ -17,6 +17,8 @@ import type { PanelPagination, PanelQueryData } from '../queryV5/types'; import { getRawResults } from '../queryV5/v5ResponseData'; import { getBuilderQueries } from '../Panels/utils/getBuilderQueries'; import { PANEL_KIND_TO_PANEL_TYPE } from '../Panels/types/panelKind'; +import { selectResolvedVariables } from '../store/slices/variableSelectionSlice'; +import { useDashboardStore } from '../store/useDashboardStore'; import { resolvePanelTimeWindow } from './resolvePanelTimeWindow'; import { useGetQueryRangeV5 } from './useGetQueryRangeV5'; @@ -65,8 +67,9 @@ export interface UsePanelQueryResult { /** * Fetches query-range data for a V2 panel over the pure-V5 contract: builds the request DTO * from the panel's perses queries (no V1 `Query` intermediary), reads global time from Redux, - * and posts via `useGetQueryRangeV5`. Variable substitution is deferred until V2 has its own - * variable plumbing. Renderers consume the raw response through the `queryV5` prep utils. + * substitutes the dashboard's resolved variable values (published to the store by + * `useResolvedVariables`), and posts via `useGetQueryRangeV5`. Renderers consume the raw + * response through the `queryV5` prep utils. */ export function usePanelQuery({ panel, @@ -105,6 +108,11 @@ export function usePanelQuery({ minTime, } = useSelector((state) => state.globalTime); + // Resolved variable values for this dashboard, published by useResolvedVariables. + // Substituted into the request and keyed into the cache so a selection change refetches. + const dashboardId = useDashboardStore((s) => s.dashboardId); + const variables = useDashboardStore(selectResolvedVariables(dashboardId)); + // `visualization` exists only on variants that declare it — read via `in` narrowing over the // generated union (no cast). `fillSpans` (TimeSeries/Bar only) → formatOptions.fillGaps. const pluginSpec = panel.spec.plugin.spec; @@ -141,8 +149,19 @@ export function usePanelQuery({ endMs, fillGaps, pagination: isPaginated ? { offset, limit: pageSize } : undefined, + variables, }), - [queries, panelType, startMs, endMs, fillGaps, isPaginated, offset, pageSize], + [ + queries, + panelType, + startMs, + endMs, + fillGaps, + isPaginated, + offset, + pageSize, + variables, + ], ); const legendMap = useMemo(() => extractLegendMap(queries), [queries]); @@ -167,6 +186,8 @@ export function usePanelQuery({ // Each page is its own cache entry (0/default for non-paged kinds). offset, pageSize, + // Variable selection changes the request, so it must re-key the cache (refetch). + variables, ], [ panelId, @@ -182,6 +203,7 @@ export function usePanelQuery({ queries, offset, pageSize, + variables, ], ); diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/useResolvedVariables.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/useResolvedVariables.ts new file mode 100644 index 0000000000..16d4a3398a --- /dev/null +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/hooks/useResolvedVariables.ts @@ -0,0 +1,42 @@ +import { useEffect, useMemo } from 'react'; +import type { DashboardtypesGettableDashboardV2DTO } from 'api/generated/services/sigNoz.schemas'; + +import { dtoToFormModel } from '../DashboardSettings/Variables/variableAdapters'; +import { buildVariablesPayload } from '../queryV5/buildVariablesPayload'; +import { selectVariableValues } from '../store/slices/variableSelectionSlice'; +import { useDashboardStore } from '../store/useDashboardStore'; + +/** + * Resolves the dashboard's variable selection into the V5 query payload and + * publishes it to the store, so `usePanelQuery` reads it by dashboardId without + * the spec being threaded through the panel tree (the `setEditContext` pattern). + * + * Definitions come from the spec; values come from the runtime selection (seeded + * by the variable bar). Re-publishes whenever either changes, which re-keys the + * panel queries and triggers a refetch with the new values. + */ +export function useResolvedVariables( + dashboard: DashboardtypesGettableDashboardV2DTO, +): void { + const dashboardId = dashboard.id ?? ''; + + const definitions = useMemo( + () => (dashboard.spec?.variables ?? []).map(dtoToFormModel), + [dashboard.spec?.variables], + ); + + const selection = useDashboardStore(selectVariableValues(dashboardId)); + const setResolvedVariables = useDashboardStore((s) => s.setResolvedVariables); + + const resolved = useMemo( + () => buildVariablesPayload(definitions, selection), + [definitions, selection], + ); + + useEffect(() => { + if (!dashboardId) { + return; + } + setResolvedVariables(dashboardId, resolved); + }, [dashboardId, resolved, setResolvedVariables]); +} diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/index.tsx b/frontend/src/pages/DashboardPageV2/DashboardContainer/index.tsx index b130555e41..bfdbb97f3e 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/index.tsx +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/index.tsx @@ -7,6 +7,7 @@ import { useAppContext } from 'providers/App/App'; import DashboardPageToolbar from './DashboardPageToolbar'; import PanelsAndSectionsLayout from './PanelsAndSectionsLayout'; +import { useResolvedVariables } from './hooks/useResolvedVariables'; import { useDashboardStore } from './store/useDashboardStore'; import styles from './DashboardContainer.module.scss'; import DashboardPageHeader from './components/DashboardPageHeader/DashboardPageHeader'; @@ -50,6 +51,10 @@ function DashboardContainer({ setEditContext, ]); + // Resolve the variable selection into the V5 query payload and publish it to + // the store, so each panel's query substitutes the bar's selected values. + useResolvedVariables(dashboard); + const spec = dashboard.spec; const image = dashboard.image || Base64Icons[0]; const name = spec.display.name; diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/__tests__/buildVariablesPayload.test.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/__tests__/buildVariablesPayload.test.ts new file mode 100644 index 0000000000..5167b238d6 --- /dev/null +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/__tests__/buildVariablesPayload.test.ts @@ -0,0 +1,103 @@ +import { + emptyVariableFormModel, + type VariableFormModel, + type VariableType, +} from '../../DashboardSettings/Variables/variableFormModel'; +import type { VariableSelectionMap } from '../../VariablesBar/selectionTypes'; +import { buildVariablesPayload } from '../buildVariablesPayload'; + +function variable( + name: string, + type: VariableType, + overrides: Partial = {}, +): VariableFormModel { + return { ...emptyVariableFormModel(), name, type, ...overrides }; +} + +describe('buildVariablesPayload', () => { + it('returns an empty map when there are no definitions', () => { + expect(buildVariablesPayload([], {})).toStrictEqual({}); + }); + + it('maps each UI variable type to its V5 wire type', () => { + const definitions = [ + variable('q', 'QUERY'), + variable('c', 'CUSTOM'), + variable('t', 'TEXT'), + variable('d', 'DYNAMIC'), + ]; + const selection: VariableSelectionMap = { + q: { value: 'a', allSelected: false }, + c: { value: 'b', allSelected: false }, + t: { value: 'c', allSelected: false }, + d: { value: 'e', allSelected: false }, + }; + expect(buildVariablesPayload(definitions, selection)).toStrictEqual({ + q: { type: 'query', value: 'a' }, + c: { type: 'custom', value: 'b' }, + t: { type: 'text', value: 'c' }, + d: { type: 'dynamic', value: 'e' }, + }); + }); + + it('passes a multi-select array value through verbatim', () => { + const definitions = [variable('svc', 'QUERY', { multiSelect: true })]; + const selection: VariableSelectionMap = { + svc: { value: ['a', 'b'], allSelected: false }, + }; + expect(buildVariablesPayload(definitions, selection)).toStrictEqual({ + svc: { type: 'query', value: ['a', 'b'] }, + }); + }); + + it('collapses a multi-select dynamic ALL selection to the __all__ sentinel', () => { + const definitions = [variable('pod', 'DYNAMIC', { multiSelect: true })]; + const selection: VariableSelectionMap = { + pod: { value: null, allSelected: true }, + }; + expect(buildVariablesPayload(definitions, selection)).toStrictEqual({ + pod: { type: 'dynamic', value: '__all__' }, + }); + }); + + it('does NOT collapse a query ALL selection — it sends the full value array', () => { + const definitions = [variable('svc', 'QUERY', { multiSelect: true })]; + const selection: VariableSelectionMap = { + svc: { value: ['a', 'b'], allSelected: true }, + }; + expect(buildVariablesPayload(definitions, selection)).toStrictEqual({ + svc: { type: 'query', value: ['a', 'b'] }, + }); + }); + + it('falls back to a text variable configured value when unselected', () => { + const definitions = [variable('env', 'TEXT', { textValue: 'prod' })]; + expect(buildVariablesPayload(definitions, {})).toStrictEqual({ + env: { type: 'text', value: 'prod' }, + }); + }); + + it('falls back to a list variable configured default when unselected', () => { + const definitions = [ + variable('region', 'QUERY', { + defaultValue: { value: 'us-east' }, + } as unknown as Partial), + ]; + expect(buildVariablesPayload(definitions, {})).toStrictEqual({ + region: { type: 'query', value: 'us-east' }, + }); + }); + + it('omits a variable with no selection and no default', () => { + const definitions = [variable('q', 'QUERY')]; + expect(buildVariablesPayload(definitions, {})).toStrictEqual({}); + }); + + it('omits an unnamed variable', () => { + const definitions = [variable('', 'QUERY')]; + const selection: VariableSelectionMap = { + '': { value: 'x', allSelected: false }, + }; + expect(buildVariablesPayload(definitions, selection)).toStrictEqual({}); + }); +}); diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildQueryRangeRequest.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildQueryRangeRequest.ts index 20478d9efa..7c2b49e821 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildQueryRangeRequest.ts +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildQueryRangeRequest.ts @@ -6,6 +6,7 @@ import type { Querybuildertypesv5PromQueryDTO, Querybuildertypesv5QueryEnvelopeDTO, Querybuildertypesv5QueryRangeRequestDTO, + Querybuildertypesv5QueryRangeRequestDTOVariables, } from 'api/generated/services/sigNoz.schemas'; import { Querybuildertypesv5QueryEnvelopeBuilderDTOType, @@ -202,11 +203,13 @@ export interface BuildQueryRangeRequestArgs { fillGaps?: boolean; /** Server-side paging for raw/list panels, written onto the builder queries' `offset`/`limit`. */ pagination?: { offset: number; limit: number }; + /** Runtime variable values (name → {type,value}) substituted server-side; built by `buildVariablesPayload`. */ + variables?: Querybuildertypesv5QueryRangeRequestDTOVariables; } /** * Builds the V5 query-range request DTO directly from the panel's perses queries (no V1 `Query` - * intermediary). Variables are absent (`variables: {}`) until V2 grows its own variable plumbing. + * intermediary). `variables` carries the runtime selection (empty when the dashboard has none). */ export function buildQueryRangeRequest({ queries, @@ -215,6 +218,7 @@ export function buildQueryRangeRequest({ endMs, fillGaps = false, pagination, + variables = {}, }: BuildQueryRangeRequestArgs): Querybuildertypesv5QueryRangeRequestDTO { let envelopes = toQueryEnvelopes(queries); if (panelType === PANEL_TYPES.BAR) { @@ -234,7 +238,7 @@ export function buildQueryRangeRequest({ formatTableResultForUI: panelType === PANEL_TYPES.TABLE, fillGaps, }, - variables: {}, + variables, }; } diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildVariablesPayload.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildVariablesPayload.ts new file mode 100644 index 0000000000..2111b99973 --- /dev/null +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/queryV5/buildVariablesPayload.ts @@ -0,0 +1,105 @@ +import type { + Querybuildertypesv5QueryRangeRequestDTOVariables, + Querybuildertypesv5VariableItemDTOValue, +} from 'api/generated/services/sigNoz.schemas'; +import { Querybuildertypesv5VariableTypeDTO } from 'api/generated/services/sigNoz.schemas'; + +import type { + VariableFormModel, + VariableType, +} from '../DashboardSettings/Variables/variableFormModel'; +import type { + SelectedVariableValue, + VariableSelection, + VariableSelectionMap, +} from '../VariablesBar/selectionTypes'; + +/** + * Backend sentinel for "every value selected" on a multi-select dynamic variable. + * V1 parity (`getDashboardVariables`): only dynamic vars collapse to `__all__`; + * query/custom multi-selects send the full value array instead. Lowercase — the + * URL/store `__ALL__` sentinel is a separate serialization concern. + */ +const ALL_VALUES_SENTINEL = '__all__'; + +/** UI variable grouping → the V5 wire `variables[].type`. */ +const VARIABLE_TYPE_TO_DTO: Record< + VariableType, + Querybuildertypesv5VariableTypeDTO +> = { + QUERY: Querybuildertypesv5VariableTypeDTO.query, + CUSTOM: Querybuildertypesv5VariableTypeDTO.custom, + TEXT: Querybuildertypesv5VariableTypeDTO.text, + DYNAMIC: Querybuildertypesv5VariableTypeDTO.dynamic, +}; + +/** The variable's configured default, used when nothing is selected yet. */ +function configuredDefault( + definition: VariableFormModel, +): SelectedVariableValue | undefined { + if (definition.type === 'TEXT') { + return definition.textValue || undefined; + } + return ( + definition.defaultValue as { value?: SelectedVariableValue } | undefined + )?.value; +} + +/** + * Resolves the wire value for one variable: the dynamic "ALL" sentinel, else the + * user's selection, else the configured default. Returns `undefined` when there + * is nothing meaningful to send (the variable is then omitted from the payload). + */ +function resolveValue( + definition: VariableFormModel, + selection: VariableSelection | undefined, +): Querybuildertypesv5VariableItemDTOValue | undefined { + if ( + definition.type === 'DYNAMIC' && + definition.multiSelect && + selection?.allSelected + ) { + return ALL_VALUES_SENTINEL; + } + + const selected = selection?.value; + const hasSelection = + selected !== null && + selected !== undefined && + !(typeof selected === 'string' && selected === ''); + if (hasSelection) { + return selected as Querybuildertypesv5VariableItemDTOValue; + } + + const fallback = configuredDefault(definition); + return fallback == null + ? undefined + : (fallback as Querybuildertypesv5VariableItemDTOValue); +} + +/** + * Builds the V5 `variables` map from the dashboard's variable definitions and the + * runtime selection, so a panel query substitutes the values the user picked in + * the variable bar (V1 parity with `getDashboardVariables` + the V5 prep). The + * definition list supplies the wire `type` (the selection map carries only values). + */ +export function buildVariablesPayload( + definitions: VariableFormModel[], + selection: VariableSelectionMap, +): Querybuildertypesv5QueryRangeRequestDTOVariables { + const payload: Querybuildertypesv5QueryRangeRequestDTOVariables = {}; + definitions.forEach((definition) => { + if (!definition.name) { + return; + } + const value = resolveValue(definition, selection[definition.name]); + if (value === undefined) { + return; + } + payload[definition.name] = { + type: VARIABLE_TYPE_TO_DTO[definition.type], + value, + }; + }); + return payload; +} diff --git a/frontend/src/pages/DashboardPageV2/DashboardContainer/store/slices/variableSelectionSlice.ts b/frontend/src/pages/DashboardPageV2/DashboardContainer/store/slices/variableSelectionSlice.ts index 7c172a0f74..f6dbb00bc1 100644 --- a/frontend/src/pages/DashboardPageV2/DashboardContainer/store/slices/variableSelectionSlice.ts +++ b/frontend/src/pages/DashboardPageV2/DashboardContainer/store/slices/variableSelectionSlice.ts @@ -1,3 +1,4 @@ +import type { Querybuildertypesv5QueryRangeRequestDTOVariables } from 'api/generated/services/sigNoz.schemas'; import type { StateCreator } from 'zustand'; import type { @@ -12,9 +13,19 @@ import type { DashboardStore } from '../useDashboardStore'; * localStorage (mirrored to the URL by the bar for shareable links); it is * deliberately NOT part of the dashboard spec, so selecting a value never * patches the dashboard. + * + * `resolvedVariables` is the same selection resolved into the V5 query payload + * shape (`{ name: { type, value } }`), published by `useResolvedVariables` so + * `usePanelQuery` reads it without threading the dashboard spec down the tree + * (the edit-context publish pattern). Transient — not persisted (it is derived + * from `variableValues` + the spec on every load). */ export interface VariableSelectionSlice { variableValues: Record; + resolvedVariables: Record< + string, + Querybuildertypesv5QueryRangeRequestDTOVariables + >; setVariableValue: ( dashboardId: string, name: string, @@ -22,6 +33,11 @@ export interface VariableSelectionSlice { ) => void; /** Bulk set (used to seed from URL/localStorage/defaults on load). */ setVariableValues: (dashboardId: string, values: VariableSelectionMap) => void; + /** Publish the resolved V5 variables payload for a dashboard. */ + setResolvedVariables: ( + dashboardId: string, + variables: Querybuildertypesv5QueryRangeRequestDTOVariables, + ) => void; } export const createVariableSelectionSlice: StateCreator< @@ -31,6 +47,7 @@ export const createVariableSelectionSlice: StateCreator< VariableSelectionSlice > = (set, get) => ({ variableValues: {}, + resolvedVariables: {}, setVariableValue: (dashboardId, name, selection): void => { const { variableValues } = get(); set({ @@ -46,6 +63,12 @@ export const createVariableSelectionSlice: StateCreator< variableValues: { ...variableValues, [dashboardId]: values }, }); }, + setResolvedVariables: (dashboardId, variables): void => { + const { resolvedVariables } = get(); + set({ + resolvedVariables: { ...resolvedVariables, [dashboardId]: variables }, + }); + }, }); /** @@ -60,3 +83,13 @@ export const selectVariableValues = (dashboardId: string) => (state: DashboardStore): VariableSelectionMap => state.variableValues[dashboardId] ?? EMPTY_SELECTION_MAP; + +/** Stable empty payload — same rationale as {@link EMPTY_SELECTION_MAP}. */ +const EMPTY_RESOLVED_VARIABLES: Querybuildertypesv5QueryRangeRequestDTOVariables = + {}; + +/** Selector: the resolved V5 variables payload for a dashboard (empty if none). */ +export const selectResolvedVariables = + (dashboardId: string) => + (state: DashboardStore): Querybuildertypesv5QueryRangeRequestDTOVariables => + state.resolvedVariables[dashboardId] ?? EMPTY_RESOLVED_VARIABLES; diff --git a/frontend/src/schemas/generated/webSettings.schema.json b/frontend/src/schemas/generated/webSettings.schema.json index e48f88ca45..a14271b32a 100644 --- a/frontend/src/schemas/generated/webSettings.schema.json +++ b/frontend/src/schemas/generated/webSettings.schema.json @@ -14,6 +14,9 @@ ], "additionalProperties": false, "properties": { + "appId": { + "type": "string" + }, "enabled": { "type": "boolean" } @@ -26,8 +29,17 @@ ], "additionalProperties": false, "properties": { + "apiHost": { + "type": "string" + }, "enabled": { "type": "boolean" + }, + "key": { + "type": "string" + }, + "uiHost": { + "type": "string" } }, "type": "object" @@ -38,8 +50,14 @@ ], "additionalProperties": false, "properties": { + "appId": { + "type": "string" + }, "enabled": { "type": "boolean" + }, + "identitySecret": { + "type": "string" } }, "type": "object" @@ -50,8 +68,14 @@ ], "additionalProperties": false, "properties": { + "dsn": { + "type": "string" + }, "enabled": { "type": "boolean" + }, + "tunnel": { + "type": "string" } }, "type": "object" diff --git a/frontend/src/types/generated/webSettings.ts b/frontend/src/types/generated/webSettings.ts index 115e05edbb..6b3580e50b 100644 --- a/frontend/src/types/generated/webSettings.ts +++ b/frontend/src/types/generated/webSettings.ts @@ -7,14 +7,22 @@ export interface WebSettings { sentry: Sentry; } export interface Appcues { + appId?: string; enabled: boolean; } export interface Posthog { + apiHost?: string; enabled: boolean; + key?: string; + uiHost?: string; } export interface Pylon { + appId?: string; enabled: boolean; + identitySecret?: string; } export interface Sentry { + dsn?: string; enabled: boolean; + tunnel?: string; } diff --git a/frontend/src/utils/permission/index.ts b/frontend/src/utils/permission/index.ts index d6c8e9357b..23d80c8d82 100644 --- a/frontend/src/utils/permission/index.ts +++ b/frontend/src/utils/permission/index.ts @@ -20,7 +20,8 @@ export type ComponentTypes = | 'add_panel' | 'page_pipelines' | 'edit_locked_dashboard' - | 'add_panel_locked_dashboard'; + | 'add_panel_locked_dashboard' + | 'manage_llm_pricing'; export const componentPermission: Record = { current_org_settings: ['ADMIN'], @@ -42,6 +43,7 @@ export const componentPermission: Record = { page_pipelines: ['ADMIN', 'EDITOR'], edit_locked_dashboard: ['ADMIN', 'AUTHOR'], add_panel_locked_dashboard: ['ADMIN', 'AUTHOR'], + manage_llm_pricing: ['ADMIN'], }; export const routePermission: Record = { diff --git a/pkg/apiserver/signozapiserver/querier.go b/pkg/apiserver/signozapiserver/querier.go index d11ff9c577..a75c7904af 100644 --- a/pkg/apiserver/signozapiserver/querier.go +++ b/pkg/apiserver/signozapiserver/querier.go @@ -451,6 +451,23 @@ func (provider *provider) addQuerierRoutes(router *mux.Router) error { return err } + if err := router.Handle("/api/v5/query_range/preview", handler.New(provider.authzMiddleware.ViewAccess(provider.querierHandler.QueryRangePreview), handler.OpenAPIDef{ + ID: "QueryRangePreviewV5", + Tags: []string{"querier"}, + Summary: "Query range preview", + Description: "Validate a composite query without executing it. Accepts the same payload as the query range endpoint. By default (verbose=true) returns, for each query, the rendered underlying ClickHouse statement(s) with each statement's EXPLAIN ESTIMATE (per-table parts/rows/marks) and granule index analysis (candidate/surviving granules and the per-index pruning funnel). Pass ?verbose=false for the lightweight per-query verdict (valid/error/warnings) with no rendered SQL and no ClickHouse round trips. Intended for agentic/dry-run consumption: per-query errors are reported in the response rather than failing the whole request.", + Request: new(qbtypes.QueryRangeRequest), + RequestQuery: new(qbtypes.QueryRangePreviewParams), + RequestContentType: "application/json", + Response: new(qbtypes.QueryRangePreviewResponse), + ResponseContentType: "application/json", + SuccessStatusCode: http.StatusOK, + ErrorStatusCodes: []int{http.StatusBadRequest}, + SecuritySchemes: newSecuritySchemes(types.RoleViewer), + })).Methods(http.MethodPost).GetError(); err != nil { + return err + } + if err := router.Handle("/api/v5/substitute_vars", handler.New(provider.authzMiddleware.ViewAccess(provider.querierHandler.ReplaceVariables), handler.OpenAPIDef{ ID: "ReplaceVariables", Tags: []string{"querier"}, diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/assets/dashboards/overview.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/assets/dashboards/overview.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/assets/dashboards/overview.json @@ -0,0 +1 @@ +{} diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/icon.svg b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/icon.svg new file mode 100644 index 0000000000..9f29818450 --- /dev/null +++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/icon.svg @@ -0,0 +1 @@ +Icon_24px_SQL_Color \ No newline at end of file diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/integration.json b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/integration.json new file mode 100644 index 0000000000..4306a5a9ce --- /dev/null +++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/integration.json @@ -0,0 +1,27 @@ +{ + "id": "cloudsql", + "title": "GCP Cloud SQL", + "icon": "file://icon.svg", + "overview": "file://overview.md", + "supportedSignals": { + "metrics": true, + "logs": true + }, + "dataCollected": { + "metrics": [], + "logs": [] + }, + "telemetryCollectionStrategy": { + "gcp": {} + }, + "assets": { + "dashboards": [ + { + "id": "overview", + "title": "GCP Cloud SQL Overview", + "description": "Overview of GCP Cloud SQL metrics", + "definition": "file://assets/dashboards/overview.json" + } + ] + } +} diff --git a/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/overview.md b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/overview.md new file mode 100644 index 0000000000..697cb95304 --- /dev/null +++ b/pkg/modules/cloudintegration/implcloudintegration/fs/definitions/gcp/cloudsql/overview.md @@ -0,0 +1,3 @@ +### Monitor GCP Cloud SQL with SigNoz + +Collect key GCP Cloud SQL metrics and view them with an out of the box dashboard. diff --git a/pkg/modules/cloudintegration/implcloudintegration/handler.go b/pkg/modules/cloudintegration/implcloudintegration/handler.go index 32435d4ae0..8f75434190 100644 --- a/pkg/modules/cloudintegration/implcloudintegration/handler.go +++ b/pkg/modules/cloudintegration/implcloudintegration/handler.go @@ -481,6 +481,7 @@ func (handler *handler) UpdateService(rw http.ResponseWriter, r *http.Request) { render.Success(rw, http.StatusNoContent, nil) } +// TODO: Rename AgentCheckIn to just CheckIn. func (handler *handler) AgentCheckIn(rw http.ResponseWriter, r *http.Request) { ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) defer cancel() diff --git a/pkg/modules/dashboard/dashboard.go b/pkg/modules/dashboard/dashboard.go index c7527473b4..bb724af05d 100644 --- a/pkg/modules/dashboard/dashboard.go +++ b/pkg/modules/dashboard/dashboard.go @@ -49,7 +49,7 @@ type Module interface { // DeleteUnsafe deletes a dashboard bypassing the guards. Intended for internal system callers. DeleteUnsafe(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error - GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]map[string]string, error) + GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]dashboardtypes.DashboardPanelRef, error) statsreporter.StatsCollector diff --git a/pkg/modules/dashboard/impldashboard/module.go b/pkg/modules/dashboard/impldashboard/module.go index 2accb044ca..d86c0f8fef 100644 --- a/pkg/modules/dashboard/impldashboard/module.go +++ b/pkg/modules/dashboard/impldashboard/module.go @@ -11,6 +11,7 @@ import ( "github.com/SigNoz/signoz/pkg/modules/dashboard" "github.com/SigNoz/signoz/pkg/modules/organization" "github.com/SigNoz/signoz/pkg/modules/tag" + "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/queryparser" "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/coretypes" @@ -168,13 +169,13 @@ func (module *module) DeleteUnsafe(ctx context.Context, orgID valuer.UUID, id va return module.store.Delete(ctx, orgID, id) } -func (module *module) GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]map[string]string, error) { +func (module *module) GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]dashboardtypes.DashboardPanelRef, error) { dashboards, err := module.List(ctx, orgID) if err != nil { return nil, err } - result := make(map[string][]map[string]string) + result := make(map[string][]dashboardtypes.DashboardPanelRef) for _, dashboard := range dashboards { dashData := dashboard.Data @@ -198,21 +199,27 @@ func (module *module) GetByMetricNames(ctx context.Context, orgID valuer.UUID, m continue } - // Track which metrics were found in this widget + // Track which metrics were found in this widget, along with the + // group-by and filter labels referenced for each metric. CH/PromQL + // paths are presence-only and leave the label sets empty. foundMetrics := make(map[string]bool) + groupByByMetric := make(map[string][]string) + filterByByMetric := make(map[string][]string) // Check all three query types - module.checkBuilderQueriesForMetricNames(query, metricNames, foundMetrics) + module.checkBuilderQueriesForMetricNames(query, metricNames, foundMetrics, groupByByMetric, filterByByMetric) module.checkClickHouseQueriesForMetricNames(ctx, query, metricNames, foundMetrics) module.checkPromQLQueriesForMetricNames(ctx, query, metricNames, foundMetrics) // Add widget to results for all found metrics for metricName := range foundMetrics { - result[metricName] = append(result[metricName], map[string]string{ - "dashboard_id": dashboard.ID, - "widget_name": widgetTitle, - "widget_id": widgetID, - "dashboard_name": dashTitle, + result[metricName] = append(result[metricName], dashboardtypes.DashboardPanelRef{ + DashboardID: dashboard.ID, + DashboardName: dashTitle, + PanelID: widgetID, + PanelName: widgetTitle, + GroupBy: groupByByMetric[metricName], + FilterBy: filterByByMetric[metricName], }) } } @@ -260,7 +267,10 @@ func (module *module) DeletePublic(_ context.Context, _ valuer.UUID, _ valuer.UU } // checkBuilderQueriesForMetricNames checks builder.queryData[] for aggregations[].metricName. -func (module *module) checkBuilderQueriesForMetricNames(query map[string]interface{}, metricNames []string, foundMetrics map[string]bool) { +// For each queryData entry whose dataSource is "metrics" and that references a +// target metric, it accumulates (deduped) the group-by and filter labels used +// by that entry into groupByByMetric/filterByByMetric, keyed by metric name. +func (module *module) checkBuilderQueriesForMetricNames(query map[string]interface{}, metricNames []string, foundMetrics map[string]bool, groupByByMetric, filterByByMetric map[string][]string) { builder, ok := query["builder"].(map[string]interface{}) if !ok { return @@ -288,6 +298,7 @@ func (module *module) checkBuilderQueriesForMetricNames(query map[string]interfa continue } + entryMetrics := make([]string, 0, len(aggregations)) for _, agg := range aggregations { aggMap, ok := agg.(map[string]interface{}) if !ok { @@ -301,9 +312,98 @@ func (module *module) checkBuilderQueriesForMetricNames(query map[string]interfa if slices.Contains(metricNames, metricName) { foundMetrics[metricName] = true + entryMetrics = append(entryMetrics, metricName) + } + } + + if len(entryMetrics) == 0 { + continue + } + + groupBy := extractBuilderGroupByLabels(data) + filterBy := extractBuilderFilterLabels(data) + for _, metricName := range entryMetrics { + groupByByMetric[metricName] = appendDedup(groupByByMetric[metricName], groupBy...) + filterByByMetric[metricName] = appendDedup(filterByByMetric[metricName], filterBy...) + } + } +} + +func extractBuilderGroupByLabels(data map[string]interface{}) []string { + gb, ok := data["groupBy"].([]interface{}) + if !ok { + return nil + } + + out := make([]string, 0, len(gb)) + for _, g := range gb { + gm, ok := g.(map[string]interface{}) + if !ok { + continue + } + if name, ok := gm["name"].(string); ok && name != "" { + out = append(out, name) + continue + } + // v3: groupBy[].key may be a plain string ... + if key, ok := gm["key"].(string); ok && key != "" { + out = append(out, key) + continue + } + // ... or a nested object {key: ""}. + if km, ok := gm["key"].(map[string]interface{}); ok { + if key, ok := km["key"].(string); ok && key != "" { + out = append(out, key) } } } + return out +} + +func extractBuilderFilterLabels(data map[string]interface{}) []string { + out := []string{} + + // v5: filter.expression + if f, ok := data["filter"].(map[string]interface{}); ok { + if expr, ok := f["expression"].(string); ok && expr != "" { + for _, sel := range querybuilder.QueryStringToKeysSelectors(expr) { + if sel != nil && sel.Name != "" { + out = append(out, sel.Name) + } + } + } + } + + // v3: filters.items[].key.key + if f, ok := data["filters"].(map[string]interface{}); ok { + if items, ok := f["items"].([]interface{}); ok { + for _, it := range items { + im, ok := it.(map[string]interface{}) + if !ok { + continue + } + km, ok := im["key"].(map[string]interface{}) + if !ok { + continue + } + if key, ok := km["key"].(string); ok && key != "" { + out = append(out, key) + } + } + } + } + + return out +} + +func appendDedup(dst []string, values ...string) []string { + for _, v := range values { + if v == "" || slices.Contains(dst, v) { + continue + } + dst = append(dst, v) + } + return dst } // checkClickHouseQueriesForMetricNames checks clickhouse_sql[] array for metric names in query strings. diff --git a/pkg/modules/fields/implfields/handler.go b/pkg/modules/fields/implfields/handler.go index e083b6bf6a..e77711ebee 100644 --- a/pkg/modules/fields/implfields/handler.go +++ b/pkg/modules/fields/implfields/handler.go @@ -7,7 +7,9 @@ import ( "github.com/SigNoz/signoz/pkg/http/binding" "github.com/SigNoz/signoz/pkg/http/render" "github.com/SigNoz/signoz/pkg/modules/fields" + "github.com/SigNoz/signoz/pkg/types/authtypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/SigNoz/signoz/pkg/valuer" ) type handler struct { @@ -54,7 +56,13 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques fieldValueSelector := telemetrytypes.NewFieldValueSelectorFromPostableFieldValueParams(params) - allValues, allComplete, err := handler.telemetryMetadataStore.GetAllValues(ctx, fieldValueSelector) + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + render.Error(rw, err) + return + } + + allValues, allComplete, err := handler.telemetryMetadataStore.GetAllValues(ctx, valuer.MustNewUUID(claims.OrgID), fieldValueSelector) if err != nil { render.Error(rw, err) return diff --git a/pkg/modules/inframonitoring/implinframonitoring/clusters.go b/pkg/modules/inframonitoring/implinframonitoring/clusters.go index 569cb4f911..7929f05aa7 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/clusters.go +++ b/pkg/modules/inframonitoring/implinframonitoring/clusters.go @@ -137,12 +137,12 @@ func (m *module) getTopClusterGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getClustersTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableClusters) (map[string]map[string]string, error) { +func (m *module) getClustersTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableClusters) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range clusterAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, clustersTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, clustersTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/inframonitoring/implinframonitoring/daemonsets.go b/pkg/modules/inframonitoring/implinframonitoring/daemonsets.go index cae7706d38..dbf22b670a 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/daemonsets.go +++ b/pkg/modules/inframonitoring/implinframonitoring/daemonsets.go @@ -145,12 +145,12 @@ func (m *module) getTopDaemonSetGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getDaemonSetsTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableDaemonSets) (map[string]map[string]string, error) { +func (m *module) getDaemonSetsTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableDaemonSets) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range daemonSetAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, daemonSetsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, daemonSetsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/inframonitoring/implinframonitoring/deployments.go b/pkg/modules/inframonitoring/implinframonitoring/deployments.go index 9c657ccd80..5d17c7e500 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/deployments.go +++ b/pkg/modules/inframonitoring/implinframonitoring/deployments.go @@ -145,12 +145,12 @@ func (m *module) getTopDeploymentGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getDeploymentsTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableDeployments) (map[string]map[string]string, error) { +func (m *module) getDeploymentsTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableDeployments) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range deploymentAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, deploymentsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, deploymentsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/inframonitoring/implinframonitoring/helpers.go b/pkg/modules/inframonitoring/implinframonitoring/helpers.go index 072348d95d..b5d3ab26f6 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/helpers.go +++ b/pkg/modules/inframonitoring/implinframonitoring/helpers.go @@ -7,11 +7,14 @@ import ( "strings" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/flagger" "github.com/SigNoz/signoz/pkg/querybuilder" "github.com/SigNoz/signoz/pkg/telemetrymetrics" + "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/metrictypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" + "github.com/SigNoz/signoz/pkg/valuer" "github.com/huandu/go-sqlbuilder" ) @@ -367,6 +370,33 @@ func (m *module) buildSamplesTblFingerprintSubQuery(metricNames []string, sample return fpSB } +// buildReducedSamplesTblFingerprintSubQuery is like buildSamplesTblFingerprintSubQuery +// but for the reduced tables. +func (m *module) buildReducedSamplesTblFingerprintSubQuery(metricNames []string, flooredStart, flooredEnd uint64) *sqlbuilder.SelectBuilder { + lastSB := sqlbuilder.NewSelectBuilder() + lastSB.Select("reduced_fingerprint") + lastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName)) + lastSB.Where( + lastSB.In("metric_name", sqlbuilder.List(metricNames)), + lastSB.GE("unix_milli", flooredStart), + lastSB.L("unix_milli", flooredEnd), + ) + + sumSB := sqlbuilder.NewSelectBuilder() + sumSB.Select("reduced_fingerprint") + sumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName)) + sumSB.Where( + sumSB.In("metric_name", sqlbuilder.List(metricNames)), + sumSB.GE("unix_milli", flooredStart), + sumSB.L("unix_milli", flooredEnd), + ) + + fpSB := sqlbuilder.NewSelectBuilder() + fpSB.Select("DISTINCT reduced_fingerprint AS fingerprint") + fpSB.From(fpSB.BuilderAs(sqlbuilder.UnionAll(lastSB, sumSB), "reduced_samples")) + return fpSB +} + func (m *module) buildFilterClause(ctx context.Context, filter *qbtypes.Filter, startMillis, endMillis int64) (*sqlbuilder.WhereClause, error) { expression := "" if filter != nil { @@ -557,6 +587,7 @@ func (m *module) getAttributesExistence(ctx context.Context, metricNames, attrNa // mapping to a flat map of attr_name -> attr_value (includes both groupBy and additional cols). func (m *module) getMetadata( ctx context.Context, + orgID valuer.UUID, metricNames []string, groupBy []qbtypes.GroupByKey, additionalCols []string, @@ -570,6 +601,8 @@ func (m *module) getMetadata( return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "groupBy must not be empty") } + reductionEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + // Step-floor the window and pick the right tables — matches the bounds the // QB v5 metric querier uses, so metadataMap covers the same universe the // ranking sees (see alignedMetricWindow doc). @@ -607,22 +640,64 @@ func (m *module) getMetadata( } innerSB.Select(innerSelectCols...) - innerSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName)) - innerSB.Where( - innerSB.In("metric_name", sqlbuilder.List(metricNames)), - innerSB.GE("unix_milli", tsAdjustedStartMs), - innerSB.LE("unix_milli", flooredEndMs), - fmt.Sprintf("fingerprint IN (%s)", innerSB.Var(fpSB)), - ) - // Apply optional filter expression - if filter != nil && strings.TrimSpace(filter.Expression) != "" { - filterClause, err := m.buildFilterClause(ctx, filter, startMs, endMs) - if err != nil { - return nil, err + if reductionEnabled { + var filterClause *sqlbuilder.WhereClause + if filter != nil && strings.TrimSpace(filter.Expression) != "" { + var err error + filterClause, err = m.buildFilterClause(ctx, filter, startMs, endMs) + if err != nil { + return nil, err + } } + + reducedFpSB := m.buildReducedSamplesTblFingerprintSubQuery(metricNames, samplesStartMs, flooredEndMs) + + rawSrc := sqlbuilder.NewSelectBuilder() + rawSrc.Select("labels", "unix_milli") + rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName)) + rawSrc.Where( + rawSrc.In("metric_name", sqlbuilder.List(metricNames)), + rawSrc.GE("unix_milli", tsAdjustedStartMs), + rawSrc.LE("unix_milli", flooredEndMs), + fmt.Sprintf("fingerprint IN (%s)", rawSrc.Var(fpSB)), + ) if filterClause != nil { - innerSB.AddWhereClause(filterClause) + rawSrc.AddWhereClause(sqlbuilder.CopyWhereClause(filterClause)) + } + + reducedSrc := sqlbuilder.NewSelectBuilder() + reducedSrc.Select("labels", "unix_milli") + reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedSrc.Where( + reducedSrc.In("metric_name", sqlbuilder.List(metricNames)), + reducedSrc.GE("unix_milli", tsAdjustedStartMs), + reducedSrc.LE("unix_milli", flooredEndMs), + fmt.Sprintf("fingerprint IN (%s)", reducedSrc.Var(reducedFpSB)), + ) + if filterClause != nil { + reducedSrc.AddWhereClause(sqlbuilder.CopyWhereClause(filterClause)) + } + + // Inner query reads over the union of raw + reduced series. + innerSB.From(innerSB.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series")) + } else { + innerSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTableName)) + innerSB.Where( + innerSB.In("metric_name", sqlbuilder.List(metricNames)), + innerSB.GE("unix_milli", tsAdjustedStartMs), + innerSB.LE("unix_milli", flooredEndMs), + fmt.Sprintf("fingerprint IN (%s)", innerSB.Var(fpSB)), + ) + + if filter != nil && strings.TrimSpace(filter.Expression) != "" { + filterClause, err := m.buildFilterClause(ctx, filter, startMs, endMs) + if err != nil { + return nil, err + } + if filterClause != nil { + innerSB.AddWhereClause(filterClause) + } } } diff --git a/pkg/modules/inframonitoring/implinframonitoring/hosts.go b/pkg/modules/inframonitoring/implinframonitoring/hosts.go index 5ab36a634f..d64cb73d1c 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/hosts.go +++ b/pkg/modules/inframonitoring/implinframonitoring/hosts.go @@ -6,7 +6,9 @@ import ( "slices" "strings" + "github.com/SigNoz/signoz/pkg/flagger" "github.com/SigNoz/signoz/pkg/telemetrymetrics" + "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/inframonitoringtypes" qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" "github.com/SigNoz/signoz/pkg/valuer" @@ -20,12 +22,15 @@ import ( // and a simple uniqExactIf for total count. Inactive = total - active (computed in Go). func (m *module) getPerGroupHostStatusCounts( ctx context.Context, + orgID valuer.UUID, req *inframonitoringtypes.PostableHosts, metricNames []string, pageGroups []map[string]string, sinceUnixMilli int64, ) (map[string]groupHostStatusCounts, error) { + reductionEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + // Build the full filter expression from req (user filter + status filter) and page groups. reqFilterExpr := "" if req.Filter != nil { @@ -53,27 +58,65 @@ func (m *module) getPerGroupHostStatusCounts( fmt.Sprintf("uniqExactIf(%s, %s != '') AS total_host_count", hostNameExpr, hostNameExpr), ) - // Build a fingerprint subquery to restrict to fingerprints with actual sample - // data in the floored time range. - fpSB := m.buildSamplesTblFingerprintSubQuery(metricNames, localSamplesTable, samplesStartMs, flooredEndMs) - sb.Select(selectCols...) - sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName)) - sb.Where( - sb.In("metric_name", sqlbuilder.List(metricNames)), - sb.GE("unix_milli", tsAdjustedStartMs), - sb.LE("unix_milli", flooredEndMs), - fmt.Sprintf("fingerprint IN (%s)", sb.Var(fpSB)), - ) - // Apply the combined filter expression (user filter + status filter + page groups IN). - if filterExpr != "" { - filterClause, err := m.buildFilterClause(ctx, &qbtypes.Filter{Expression: filterExpr}, req.Start, req.End) - if err != nil { - return nil, err + if reductionEnabled { + var filterClause *sqlbuilder.WhereClause + if filterExpr != "" { + var err error + filterClause, err = m.buildFilterClause(ctx, &qbtypes.Filter{Expression: filterExpr}, req.Start, req.End) + if err != nil { + return nil, err + } } + + rawSrc := sqlbuilder.NewSelectBuilder() + rawSrc.Select("labels") + rawSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName)) + rawSrc.Where( + rawSrc.In("metric_name", sqlbuilder.List(metricNames)), + rawSrc.GE("unix_milli", tsAdjustedStartMs), + rawSrc.LE("unix_milli", flooredEndMs), + fmt.Sprintf("fingerprint IN (%s)", rawSrc.Var(m.buildSamplesTblFingerprintSubQuery(metricNames, localSamplesTable, samplesStartMs, flooredEndMs))), + ) if filterClause != nil { - sb.AddWhereClause(filterClause) + rawSrc.AddWhereClause(sqlbuilder.CopyWhereClause(filterClause)) + } + + reducedSrc := sqlbuilder.NewSelectBuilder() + reducedSrc.Select("labels") + reducedSrc.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedSrc.Where( + reducedSrc.In("metric_name", sqlbuilder.List(metricNames)), + reducedSrc.GE("unix_milli", tsAdjustedStartMs), + reducedSrc.LE("unix_milli", flooredEndMs), + fmt.Sprintf("fingerprint IN (%s)", reducedSrc.Var(m.buildReducedSamplesTblFingerprintSubQuery(metricNames, samplesStartMs, flooredEndMs))), + ) + if filterClause != nil { + reducedSrc.AddWhereClause(sqlbuilder.CopyWhereClause(filterClause)) + } + + sb.From(sb.BuilderAs(sqlbuilder.UnionAll(rawSrc, reducedSrc), "series")) + } else { + + fpSB := m.buildSamplesTblFingerprintSubQuery(metricNames, localSamplesTable, samplesStartMs, flooredEndMs) + + sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTimeSeriesTableName)) + sb.Where( + sb.In("metric_name", sqlbuilder.List(metricNames)), + sb.GE("unix_milli", tsAdjustedStartMs), + sb.LE("unix_milli", flooredEndMs), + fmt.Sprintf("fingerprint IN (%s)", sb.Var(fpSB)), + ) + + if filterExpr != "" { + filterClause, err := m.buildFilterClause(ctx, &qbtypes.Filter{Expression: filterExpr}, req.Start, req.End) + if err != nil { + return nil, err + } + if filterClause != nil { + sb.AddWhereClause(filterClause) + } } } @@ -288,7 +331,7 @@ func (m *module) applyHostsActiveStatusFilter(req *inframonitoringtypes.Postable return false } -func (m *module) getHostsTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableHosts) (map[string]map[string]string, error) { +func (m *module) getHostsTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableHosts) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range hostAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { @@ -299,7 +342,7 @@ func (m *module) getHostsTableMetadata(ctx context.Context, req *inframonitoring if req.Filter != nil { filter = &req.Filter.Filter } - metadataMap, err := m.getMetadata(ctx, hostsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, filter, req.Start, req.End) + metadataMap, err := m.getMetadata(ctx, orgID, hostsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, filter, req.Start, req.End) if err != nil { return nil, err } diff --git a/pkg/modules/inframonitoring/implinframonitoring/jobs.go b/pkg/modules/inframonitoring/implinframonitoring/jobs.go index 63211f5a83..0464318914 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/jobs.go +++ b/pkg/modules/inframonitoring/implinframonitoring/jobs.go @@ -153,12 +153,12 @@ func (m *module) getTopJobGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getJobsTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableJobs) (map[string]map[string]string, error) { +func (m *module) getJobsTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableJobs) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range jobAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, jobsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, jobsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/inframonitoring/implinframonitoring/module.go b/pkg/modules/inframonitoring/implinframonitoring/module.go index ff93b3e3f8..0528499d1d 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/module.go +++ b/pkg/modules/inframonitoring/implinframonitoring/module.go @@ -6,6 +6,7 @@ import ( "time" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/flagger" "github.com/SigNoz/signoz/pkg/modules/inframonitoring" "github.com/SigNoz/signoz/pkg/querier" "github.com/SigNoz/signoz/pkg/telemetrymetrics" @@ -26,6 +27,7 @@ type module struct { condBuilder qbtypes.ConditionBuilder logger *slog.Logger config inframonitoring.Config + fl flagger.Flagger } // NewModule constructs the inframonitoring module with the provided dependencies. @@ -33,6 +35,7 @@ func NewModule( telemetryStore telemetrystore.TelemetryStore, telemetryMetadataStore telemetrytypes.MetadataStore, querier querier.Querier, + fl flagger.Flagger, providerSettings factory.ProviderSettings, cfg inframonitoring.Config, ) inframonitoring.Module { @@ -46,6 +49,7 @@ func NewModule( condBuilder: condBuilder, logger: providerSettings.Logger, config: cfg, + fl: fl, } } @@ -186,7 +190,7 @@ func (m *module) ListHosts(ctx context.Context, orgID valuer.UUID, req *inframon return resp, nil } - metadataMap, err := m.getHostsTableMetadata(ctx, req) + metadataMap, err := m.getHostsTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -222,7 +226,7 @@ func (m *module) ListHosts(ctx context.Context, orgID valuer.UUID, req *inframon hostCounts := make(map[string]groupHostStatusCounts) isHostNameInGroupBy := isKeyInGroupByAttrs(req.GroupBy, inframonitoringtypes.HostNameAttrKey) if !isHostNameInGroupBy { - hostCounts, err = m.getPerGroupHostStatusCounts(ctx, req, hostsTableMetricNamesList, pageGroups, sinceUnixMilli) + hostCounts, err = m.getPerGroupHostStatusCounts(ctx, orgID, req, hostsTableMetricNamesList, pageGroups, sinceUnixMilli) if err != nil { return nil, err } @@ -272,7 +276,7 @@ func (m *module) ListPods(ctx context.Context, orgID valuer.UUID, req *inframoni return resp, nil } - metadataMap, err := m.getPodsTableMetadata(ctx, req) + metadataMap, err := m.getPodsTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -360,7 +364,7 @@ func (m *module) ListNodes(ctx context.Context, orgID valuer.UUID, req *inframon return resp, nil } - metadataMap, err := m.getNodesTableMetadata(ctx, req) + metadataMap, err := m.getNodesTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -448,7 +452,7 @@ func (m *module) ListNamespaces(ctx context.Context, orgID valuer.UUID, req *inf return resp, nil } - metadataMap, err := m.getNamespacesTableMetadata(ctx, req) + metadataMap, err := m.getNamespacesTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -530,7 +534,7 @@ func (m *module) ListClusters(ctx context.Context, orgID valuer.UUID, req *infra return resp, nil } - metadataMap, err := m.getClustersTableMetadata(ctx, req) + metadataMap, err := m.getClustersTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -625,7 +629,7 @@ func (m *module) ListVolumes(ctx context.Context, orgID valuer.UUID, req *infram return resp, nil } - metadataMap, err := m.getVolumesTableMetadata(ctx, req) + metadataMap, err := m.getVolumesTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -703,7 +707,7 @@ func (m *module) ListDeployments(ctx context.Context, orgID valuer.UUID, req *in return resp, nil } - metadataMap, err := m.getDeploymentsTableMetadata(ctx, req) + metadataMap, err := m.getDeploymentsTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -791,7 +795,7 @@ func (m *module) ListStatefulSets(ctx context.Context, orgID valuer.UUID, req *i return resp, nil } - metadataMap, err := m.getStatefulSetsTableMetadata(ctx, req) + metadataMap, err := m.getStatefulSetsTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -881,7 +885,7 @@ func (m *module) ListJobs(ctx context.Context, orgID valuer.UUID, req *inframoni return resp, nil } - metadataMap, err := m.getJobsTableMetadata(ctx, req) + metadataMap, err := m.getJobsTableMetadata(ctx, orgID, req) if err != nil { return nil, err } @@ -971,7 +975,7 @@ func (m *module) ListDaemonSets(ctx context.Context, orgID valuer.UUID, req *inf return resp, nil } - metadataMap, err := m.getDaemonSetsTableMetadata(ctx, req) + metadataMap, err := m.getDaemonSetsTableMetadata(ctx, orgID, req) if err != nil { return nil, err } diff --git a/pkg/modules/inframonitoring/implinframonitoring/namespaces.go b/pkg/modules/inframonitoring/implinframonitoring/namespaces.go index 97a25df0cb..fca270d254 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/namespaces.go +++ b/pkg/modules/inframonitoring/implinframonitoring/namespaces.go @@ -120,12 +120,12 @@ func (m *module) getTopNamespaceGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getNamespacesTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableNamespaces) (map[string]map[string]string, error) { +func (m *module) getNamespacesTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNamespaces) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range namespaceAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, namespacesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, namespacesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/inframonitoring/implinframonitoring/nodes.go b/pkg/modules/inframonitoring/implinframonitoring/nodes.go index e3bddd059e..e88acc4aa2 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/nodes.go +++ b/pkg/modules/inframonitoring/implinframonitoring/nodes.go @@ -154,14 +154,14 @@ func (m *module) getTopNodeGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getNodesTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableNodes) (map[string]map[string]string, error) { +func (m *module) getNodesTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNodes) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range nodeAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, nodesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, nodesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } // getPerGroupNodeConditionCounts computes per-group node counts bucketed by each diff --git a/pkg/modules/inframonitoring/implinframonitoring/pods.go b/pkg/modules/inframonitoring/implinframonitoring/pods.go index ede77ba8b9..d6cd7f0390 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/pods.go +++ b/pkg/modules/inframonitoring/implinframonitoring/pods.go @@ -225,14 +225,14 @@ func (m *module) getTopPodGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getPodsTableMetadata(ctx context.Context, req *inframonitoringtypes.PostablePods) (map[string]map[string]string, error) { +func (m *module) getPodsTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostablePods) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range podAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, podsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, podsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } // getPerGroupPodPhaseCounts computes per-group pod counts bucketed by each diff --git a/pkg/modules/inframonitoring/implinframonitoring/statefulsets.go b/pkg/modules/inframonitoring/implinframonitoring/statefulsets.go index f429978700..f842f73790 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/statefulsets.go +++ b/pkg/modules/inframonitoring/implinframonitoring/statefulsets.go @@ -145,12 +145,12 @@ func (m *module) getTopStatefulSetGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getStatefulSetsTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableStatefulSets) (map[string]map[string]string, error) { +func (m *module) getStatefulSetsTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableStatefulSets) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range statefulSetAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, statefulSetsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, statefulSetsTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/inframonitoring/implinframonitoring/volumes.go b/pkg/modules/inframonitoring/implinframonitoring/volumes.go index 890404b630..51ed168515 100644 --- a/pkg/modules/inframonitoring/implinframonitoring/volumes.go +++ b/pkg/modules/inframonitoring/implinframonitoring/volumes.go @@ -120,12 +120,12 @@ func (m *module) getTopVolumeGroups( return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil } -func (m *module) getVolumesTableMetadata(ctx context.Context, req *inframonitoringtypes.PostableVolumes) (map[string]map[string]string, error) { +func (m *module) getVolumesTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableVolumes) (map[string]map[string]string, error) { var nonGroupByAttrs []string for _, key := range volumeAttrKeysForMetadata { if !isKeyInGroupByAttrs(req.GroupBy, key) { nonGroupByAttrs = append(nonGroupByAttrs, key) } } - return m.getMetadata(ctx, volumesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) + return m.getMetadata(ctx, orgID, volumesTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End) } diff --git a/pkg/modules/metricsexplorer/implmetricsexplorer/module.go b/pkg/modules/metricsexplorer/implmetricsexplorer/module.go index aae4d1658f..5cbb4781b3 100644 --- a/pkg/modules/metricsexplorer/implmetricsexplorer/module.go +++ b/pkg/modules/metricsexplorer/implmetricsexplorer/module.go @@ -15,6 +15,7 @@ import ( "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/flagger" "github.com/SigNoz/signoz/pkg/modules/dashboard" "github.com/SigNoz/signoz/pkg/modules/metricsexplorer" "github.com/SigNoz/signoz/pkg/querybuilder" @@ -22,6 +23,8 @@ import ( "github.com/SigNoz/signoz/pkg/telemetrymetrics" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types/ctxtypes" + "github.com/SigNoz/signoz/pkg/types/dashboardtypes" + "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/instrumentationtypes" "github.com/SigNoz/signoz/pkg/types/metricsexplorertypes" "github.com/SigNoz/signoz/pkg/types/metrictypes" @@ -40,11 +43,12 @@ type module struct { cache cache.Cache ruleStore ruletypes.RuleStore dashboardModule dashboard.Module + fl flagger.Flagger config metricsexplorer.Config } // NewModule constructs the metrics module with the provided dependencies. -func NewModule(ts telemetrystore.TelemetryStore, telemetryMetadataStore telemetrytypes.MetadataStore, cache cache.Cache, ruleStore ruletypes.RuleStore, dashboardModule dashboard.Module, providerSettings factory.ProviderSettings, cfg metricsexplorer.Config) metricsexplorer.Module { +func NewModule(ts telemetrystore.TelemetryStore, telemetryMetadataStore telemetrytypes.MetadataStore, cache cache.Cache, ruleStore ruletypes.RuleStore, dashboardModule dashboard.Module, fl flagger.Flagger, providerSettings factory.ProviderSettings, cfg metricsexplorer.Config) metricsexplorer.Module { fieldMapper := telemetrymetrics.NewFieldMapper() condBuilder := telemetrymetrics.NewConditionBuilder(fieldMapper) return &module{ @@ -56,6 +60,7 @@ func NewModule(ts telemetrystore.TelemetryStore, telemetryMetadataStore telemetr cache: cache, ruleStore: ruleStore, dashboardModule: dashboardModule, + fl: fl, config: cfg, } } @@ -137,31 +142,56 @@ func (m *module) listMeterMetrics(ctx context.Context, params *metricsexplorerty } func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *metricsexplorertypes.ListMetricsParams) (*metricsexplorertypes.ListMetricsResponse, error) { - sb := sqlbuilder.NewSelectBuilder() - sb.Select("DISTINCT metric_name") + reductionEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) - if params.Start != nil && params.End != nil { - start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil) - sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable)) - sb.Where(sb.Between("unix_milli", start, end)) + applyListFilters := func(b *sqlbuilder.SelectBuilder) { + if params.Search != "" { + searchLower := strings.ToLower(params.Search) + searchLower = strings.ReplaceAll(searchLower, "%", "\\%") + searchLower = strings.ReplaceAll(searchLower, "_", "\\_") + b.Where(b.Like("lower(metric_name)", fmt.Sprintf("%%%s%%", searchLower))) + } + } + + rawSB := sqlbuilder.NewSelectBuilder() + rawSB.Select("DISTINCT metric_name") + + var rangeStart, rangeEnd uint64 + hasRange := params.Start != nil && params.End != nil + if hasRange { + var distributedTsTable string + rangeStart, rangeEnd, distributedTsTable, _ = telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), false, nil) + rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable)) + rawSB.Where(rawSB.Between("unix_milli", rangeStart, rangeEnd)) } else { - sb.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName)) + rawSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV41weekTableName)) } - sb.Where(sb.E("__normalized", false)) + applyListFilters(rawSB) - if params.Search != "" { - searchLower := strings.ToLower(params.Search) - searchLower = strings.ReplaceAll(searchLower, "%", "\\%") - searchLower = strings.ReplaceAll(searchLower, "_", "\\_") - sb.Where(sb.Like("lower(metric_name)", fmt.Sprintf("%%%s%%", searchLower))) + var query string + var args []any + if reductionEnabled { + reducedSB := sqlbuilder.NewSelectBuilder() + reducedSB.Select("DISTINCT metric_name") + reducedSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + if hasRange { + reducedSB.Where(reducedSB.Between("unix_milli", rangeStart, rangeEnd)) + } + applyListFilters(reducedSB) + + sb := sqlbuilder.NewSelectBuilder() + sb.Select("DISTINCT metric_name") + sb.From(sb.BuilderAs(sqlbuilder.UnionAll(rawSB, reducedSB), "__all_metrics")) + sb.OrderBy("metric_name ASC") + sb.Limit(params.Limit) + query, args = sb.BuildWithFlavor(sqlbuilder.ClickHouse) + } else { + rawSB.OrderBy("metric_name ASC") + rawSB.Limit(params.Limit) + query, args = rawSB.BuildWithFlavor(sqlbuilder.ClickHouse) } - sb.OrderBy("metric_name ASC") - sb.Limit(params.Limit) - - query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) - valueCtx := ctxtypes.SetClickhouseMaxThreads(ctx, m.config.TelemetryStore.Threads) db := m.telemetryStore.ClickhouseDB() rows, err := db.Query(valueCtx, query, args...) @@ -219,7 +249,7 @@ func (m *module) GetStats(ctx context.Context, orgID valuer.UUID, req *metricsex return nil, err } - metricStats, total, err := m.fetchMetricsStatsWithSamples(ctx, req, false) + metricStats, total, err := m.fetchMetricsStatsWithSamples(ctx, orgID, req) if err != nil { return nil, err } @@ -259,13 +289,13 @@ func (m *module) GetTreemap(ctx context.Context, orgID valuer.UUID, req *metrics resp := &metricsexplorertypes.TreemapResponse{} switch req.Mode { case metricsexplorertypes.TreemapModeSamples: - entries, err := m.computeSamplesTreemap(ctx, req) + entries, err := m.computeSamplesTreemap(ctx, orgID, req) if err != nil { return nil, err } resp.Samples = entries default: // TreemapModeTimeSeries - entries, err := m.computeTimeseriesTreemap(ctx, req) + entries, err := m.computeTimeseriesTreemap(ctx, orgID, req) if err != nil { return nil, err } @@ -388,14 +418,14 @@ func (m *module) GetMetricDashboardsV2(ctx context.Context, orgID valuer.UUID, m return metricsexplorertypes.NewMetricDashboardPanelsResponse(data[metricName]), nil } -func newMetricDashboardsResponse(dashboardList []map[string]string) *metricsexplorertypes.MetricDashboardsResponse { +func newMetricDashboardsResponse(dashboardList []dashboardtypes.DashboardPanelRef) *metricsexplorertypes.MetricDashboardsResponse { dashboards := make([]metricsexplorertypes.MetricDashboard, 0, len(dashboardList)) for _, item := range dashboardList { dashboards = append(dashboards, metricsexplorertypes.MetricDashboard{ - DashboardName: item["dashboard_name"], - DashboardID: item["dashboard_id"], - WidgetID: item["widget_id"], - WidgetName: item["widget_name"], + DashboardName: item.DashboardName, + DashboardID: item.DashboardID, + WidgetID: item.PanelID, + WidgetName: item.PanelName, }) } @@ -969,11 +999,13 @@ func (m *module) buildFilterClause(ctx context.Context, filter *qbtypes.Filter, func (m *module) fetchMetricsStatsWithSamples( ctx context.Context, + orgID valuer.UUID, req *metricsexplorertypes.StatsRequest, - normalized bool, ) ([]metricsexplorertypes.Stat, uint64, error) { ctx = m.withMetricsExplorerContext(ctx, "fetchMetricsStatsWithSamples") + reductionEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + hasFilter := req.Filter != nil && strings.TrimSpace(req.Filter.Expression) != "" var filterWhereClause *sqlbuilder.WhereClause if hasFilter { @@ -997,7 +1029,6 @@ func (m *module) fetchMetricsStatsWithSamples( tsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable)) tsSB.Where(tsSB.Between("unix_milli", start, end)) tsSB.Where("NOT startsWith(metric_name, 'signoz')") - tsSB.Where(tsSB.E("__normalized", normalized)) if filterWhereClause != nil { tsSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) } @@ -1025,7 +1056,6 @@ func (m *module) fetchMetricsStatsWithSamples( fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable)) fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end)) fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')") - fingerprintSB.Where(fingerprintSB.E("__normalized", normalized)) fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) fingerprintSB.GroupBy("fingerprint") @@ -1037,30 +1067,100 @@ func (m *module) fetchMetricsStatsWithSamples( metricNamesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable)) metricNamesSB.Where(metricNamesSB.Between("unix_milli", start, end)) metricNamesSB.Where("NOT startsWith(metric_name, 'signoz')") - metricNamesSB.Where(metricNamesSB.E("__normalized", normalized)) samplesSB.Where(fmt.Sprintf("metric_name IN (%s)", samplesSB.Var(metricNamesSB))) } samplesSB.GroupBy("metric_name") ctes = append(ctes, sqlbuilder.CTEQuery("__sample_counts").As(samplesSB)) - cteBuilder := sqlbuilder.With(ctes...) - - finalSB := cteBuilder.Select( - "COALESCE(ts.metric_name, s.metric_name) AS metric_name", - "COALESCE(ts.timeseries, 0) AS timeseries", - "COALESCE(s.samples, 0) AS samples", - "COUNT(*) OVER() AS total", - ) - finalSB.From("__time_series_counts ts") - finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__sample_counts s", "ts.metric_name = s.metric_name") - finalSB.Where("(COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0)") orderByColumn, orderDirection, err := getStatsOrderByColumn(req.OrderBy) if err != nil { return nil, 0, err } + var finalSB *sqlbuilder.SelectBuilder + if reductionEnabled { + reducedTsSB := sqlbuilder.NewSelectBuilder() + reducedTsSB.Select( + "metric_name", + "uniq(fingerprint) AS timeseries", + ) + + // the data lands either of the _reduced_last/_reduced_sum tables + reducedLastSB := sqlbuilder.NewSelectBuilder() + reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt") + reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName)) + reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End)) + reducedLastSB.Where("NOT startsWith(metric_name, 'signoz')") + + reducedSumSB := sqlbuilder.NewSelectBuilder() + reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt") + reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName)) + reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End)) + reducedSumSB.Where("NOT startsWith(metric_name, 'signoz')") + + if filterWhereClause == nil { + reducedTsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedTsSB.Where(reducedTsSB.Between("unix_milli", start, end)) + reducedTsSB.Where("NOT startsWith(metric_name, 'signoz')") + reducedTsSB.GroupBy("metric_name") + } else { + reducedFpSB := sqlbuilder.NewSelectBuilder() + reducedFpSB.Select("metric_name", "fingerprint") + reducedFpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedFpSB.Where(reducedFpSB.Between("unix_milli", start, end)) + reducedFpSB.Where("NOT startsWith(metric_name, 'signoz')") + reducedFpSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) + reducedFpSB.GroupBy("metric_name", "fingerprint") + ctes = append(ctes, sqlbuilder.CTEQuery("__reduced_filtered_fingerprints").As(reducedFpSB)) + + reducedTsSB.From("__reduced_filtered_fingerprints") + reducedTsSB.GroupBy("metric_name") + + reducedLastSB.Where("reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints)") + reducedSumSB.Where("reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints)") + } + reducedLastSB.GroupBy("metric_name") + reducedSumSB.GroupBy("metric_name") + + reducedSamplesSB := sqlbuilder.NewSelectBuilder() + reducedSamplesSB.Select("metric_name", "sum(cnt) AS samples") + reducedSamplesSB.From(reducedSamplesSB.BuilderAs(sqlbuilder.UnionAll(reducedLastSB, reducedSumSB), "reduced_samples")) + reducedSamplesSB.GroupBy("metric_name") + + ctes = append(ctes, + sqlbuilder.CTEQuery("__reduced_time_series_counts").As(reducedTsSB), + sqlbuilder.CTEQuery("__reduced_sample_counts").As(reducedSamplesSB), + ) + + cteBuilder := sqlbuilder.With(ctes...) + + finalSB = cteBuilder.Select( + "COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name", + "COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries", + "COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples", + "COUNT(*) OVER() AS total", + ) + finalSB.From("__time_series_counts ts") + finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__reduced_time_series_counts rts", "ts.metric_name = rts.metric_name") + finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__sample_counts s", "COALESCE(ts.metric_name, rts.metric_name) = s.metric_name") + finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__reduced_sample_counts rs", "COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name") + finalSB.Where("(COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0)") + } else { + cteBuilder := sqlbuilder.With(ctes...) + + finalSB = cteBuilder.Select( + "COALESCE(ts.metric_name, s.metric_name) AS metric_name", + "COALESCE(ts.timeseries, 0) AS timeseries", + "COALESCE(s.samples, 0) AS samples", + "COUNT(*) OVER() AS total", + ) + finalSB.From("__time_series_counts ts") + finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__sample_counts s", "ts.metric_name = s.metric_name") + finalSB.Where("(COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0)") + } + finalSB.OrderBy( fmt.Sprintf("%s %s", orderByColumn, strings.ToUpper(orderDirection)), "metric_name ASC", @@ -1069,6 +1169,10 @@ func (m *module) fetchMetricsStatsWithSamples( finalSB.Offset(req.Offset) query, args := finalSB.BuildWithFlavor(sqlbuilder.ClickHouse) + if reductionEnabled { + // unmatched FULL JOIN columns must be NULL (not '') for the COALESCE name/keys to resolve + query += " SETTINGS join_use_nulls = 1" + } valueCtx := ctxtypes.SetClickhouseMaxThreads(ctx, m.config.TelemetryStore.Threads) db := m.telemetryStore.ClickhouseDB() @@ -1100,9 +1204,11 @@ func (m *module) fetchMetricsStatsWithSamples( return metricStats, total, nil } -func (m *module) computeTimeseriesTreemap(ctx context.Context, req *metricsexplorertypes.TreemapRequest) ([]metricsexplorertypes.TreemapEntry, error) { +func (m *module) computeTimeseriesTreemap(ctx context.Context, orgID valuer.UUID, req *metricsexplorertypes.TreemapRequest) ([]metricsexplorertypes.TreemapEntry, error) { ctx = m.withMetricsExplorerContext(ctx, "computeTimeseriesTreemap") + reductionEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + hasFilter := req.Filter != nil && strings.TrimSpace(req.Filter.Expression) != "" var filterWhereClause *sqlbuilder.WhereClause if hasFilter { @@ -1119,7 +1225,6 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, req *metricsexplo totalTSBuilder.Select("uniq(fingerprint) AS total_time_series") totalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable)) totalTSBuilder.Where(totalTSBuilder.Between("unix_milli", start, end)) - totalTSBuilder.Where(totalTSBuilder.E("__normalized", false)) metricsSB := sqlbuilder.NewSelectBuilder() metricsSB.Select( @@ -1129,28 +1234,69 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, req *metricsexplo metricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable)) metricsSB.Where(metricsSB.Between("unix_milli", start, end)) metricsSB.Where("NOT startsWith(metric_name, 'signoz')") - metricsSB.Where(metricsSB.E("__normalized", false)) if filterWhereClause != nil { metricsSB.WhereClause.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) } metricsSB.GroupBy("metric_name") - cteBuilder := sqlbuilder.With( - sqlbuilder.CTEQuery("__total_time_series").As(totalTSBuilder), - sqlbuilder.CTEQuery("__metric_totals").As(metricsSB), - ) + var finalSB *sqlbuilder.SelectBuilder + if reductionEnabled { + reducedTotalTSBuilder := sqlbuilder.NewSelectBuilder() + reducedTotalTSBuilder.Select("uniq(fingerprint) AS total_time_series") + reducedTotalTSBuilder.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedTotalTSBuilder.Where(reducedTotalTSBuilder.Between("unix_milli", start, end)) - finalSB := cteBuilder.Select( - "mt.metric_name", - "mt.total_value", - "CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage", - ) - finalSB.From("__metric_totals mt") - finalSB.Join("__total_time_series tts", "1=1") + reducedMetricsSB := sqlbuilder.NewSelectBuilder() + reducedMetricsSB.Select( + "metric_name", + "uniq(fingerprint) AS total_value", + ) + reducedMetricsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedMetricsSB.Where(reducedMetricsSB.Between("unix_milli", start, end)) + reducedMetricsSB.Where("NOT startsWith(metric_name, 'signoz')") + if filterWhereClause != nil { + reducedMetricsSB.WhereClause.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) + } + reducedMetricsSB.GroupBy("metric_name") + + cteBuilder := sqlbuilder.With( + sqlbuilder.CTEQuery("__total_time_series").As(totalTSBuilder), + sqlbuilder.CTEQuery("__metric_totals").As(metricsSB), + sqlbuilder.CTEQuery("__reduced_total_time_series").As(reducedTotalTSBuilder), + sqlbuilder.CTEQuery("__reduced_metric_totals").As(reducedMetricsSB), + ) + + finalSB = cteBuilder.Select( + "COALESCE(mt.metric_name, rmt.metric_name) AS metric_name", + "COALESCE(mt.total_value, 0) + COALESCE(rmt.total_value, 0) AS total_value", + "CASE WHEN (tts.total_time_series + rtts.total_time_series) = 0 THEN 0 ELSE ((COALESCE(mt.total_value, 0) + COALESCE(rmt.total_value, 0)) * 100.0 / (tts.total_time_series + rtts.total_time_series)) END AS percentage", + ) + finalSB.From("__metric_totals mt") + finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__reduced_metric_totals rmt", "mt.metric_name = rmt.metric_name") + finalSB.Join("__total_time_series tts", "1=1") + finalSB.Join("__reduced_total_time_series rtts", "1=1") + } else { + cteBuilder := sqlbuilder.With( + sqlbuilder.CTEQuery("__total_time_series").As(totalTSBuilder), + sqlbuilder.CTEQuery("__metric_totals").As(metricsSB), + ) + + finalSB = cteBuilder.Select( + "mt.metric_name", + "mt.total_value", + "CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage", + ) + finalSB.From("__metric_totals mt") + finalSB.Join("__total_time_series tts", "1=1") + } finalSB.OrderByDesc("percentage") finalSB.Limit(req.Limit) query, args := finalSB.BuildWithFlavor(sqlbuilder.ClickHouse) + if reductionEnabled { + // unmatched FULL JOIN columns must be NULL (not '') so COALESCE on metric_name resolves + query += " SETTINGS join_use_nulls = 1" + } valueCtx := ctxtypes.SetClickhouseMaxThreads(ctx, m.config.TelemetryStore.Threads) db := m.telemetryStore.ClickhouseDB() @@ -1176,9 +1322,11 @@ func (m *module) computeTimeseriesTreemap(ctx context.Context, req *metricsexplo return entries, nil } -func (m *module) computeSamplesTreemap(ctx context.Context, req *metricsexplorertypes.TreemapRequest) ([]metricsexplorertypes.TreemapEntry, error) { +func (m *module) computeSamplesTreemap(ctx context.Context, orgID valuer.UUID, req *metricsexplorertypes.TreemapRequest) ([]metricsexplorertypes.TreemapEntry, error) { ctx = m.withMetricsExplorerContext(ctx, "computeSamplesTreemap") + reductionEnabled := m.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + hasFilter := req.Filter != nil && strings.TrimSpace(req.Filter.Expression) != "" var filterWhereClause *sqlbuilder.WhereClause if hasFilter { @@ -1199,7 +1347,6 @@ func (m *module) computeSamplesTreemap(ctx context.Context, req *metricsexplorer metricCandidatesSB.Select("metric_name") metricCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable)) metricCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')") - metricCandidatesSB.Where(metricCandidatesSB.E("__normalized", false)) metricCandidatesSB.Where(metricCandidatesSB.Between("unix_milli", start, end)) if filterWhereClause != nil { metricCandidatesSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) @@ -1212,6 +1359,23 @@ func (m *module) computeSamplesTreemap(ctx context.Context, req *metricsexplorer sqlbuilder.CTEQuery("__metric_candidates").As(metricCandidatesSB), } + var reducedCandidatesSB *sqlbuilder.SelectBuilder + if reductionEnabled { + reducedCandidatesSB = sqlbuilder.NewSelectBuilder() + reducedCandidatesSB.Select("metric_name") + reducedCandidatesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedCandidatesSB.Where("NOT startsWith(metric_name, 'signoz')") + reducedCandidatesSB.Where(reducedCandidatesSB.Between("unix_milli", start, end)) + if filterWhereClause != nil { + reducedCandidatesSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) + } + reducedCandidatesSB.GroupBy("metric_name") + reducedCandidatesSB.OrderBy("uniq(fingerprint) DESC") + reducedCandidatesSB.Limit(candidateLimit) + + cteQueries = append(cteQueries, sqlbuilder.CTEQuery("__reduced_metric_candidates").As(reducedCandidatesSB)) + } + totalSamplesSB := sqlbuilder.NewSelectBuilder() totalSamplesSB.Select(fmt.Sprintf("%s AS total_samples", countExp)) totalSamplesSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedSamplesTable)) @@ -1226,43 +1390,136 @@ func (m *module) computeSamplesTreemap(ctx context.Context, req *metricsexplorer sampleCountsSB.Where(sampleCountsSB.Between("unix_milli", req.Start, req.End)) sampleCountsSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates)") + // Reduced sample rows land in either the _reduced_last_60s or + // _reduced_sum_60s table + var reducedLastSB, reducedSumSB *sqlbuilder.SelectBuilder + if reductionEnabled { + reducedLastSB = sqlbuilder.NewSelectBuilder() + reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt") + reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName)) + reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End)) + reducedLastSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)") + + reducedSumSB = sqlbuilder.NewSelectBuilder() + reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt") + reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName)) + reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End)) + reducedSumSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)") + } + if filterWhereClause != nil { fingerprintSB := sqlbuilder.NewSelectBuilder() fingerprintSB.Select("fingerprint") fingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTsTable)) fingerprintSB.Where(fingerprintSB.Between("unix_milli", start, end)) fingerprintSB.Where("NOT startsWith(metric_name, 'signoz')") - fingerprintSB.Where(fingerprintSB.E("__normalized", false)) fingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) fingerprintSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates)") fingerprintSB.GroupBy("fingerprint") sampleCountsSB.Where("fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints)") - cteQueries = append(cteQueries, sqlbuilder.CTEQuery("__filtered_fingerprints").As(fingerprintSB)) + if reductionEnabled { + reducedFingerprintSB := sqlbuilder.NewSelectBuilder() + reducedFingerprintSB.Select("fingerprint") + reducedFingerprintSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName)) + reducedFingerprintSB.Where(reducedFingerprintSB.Between("unix_milli", start, end)) + reducedFingerprintSB.Where("NOT startsWith(metric_name, 'signoz')") + reducedFingerprintSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause)) + reducedFingerprintSB.Where("metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates)") + reducedFingerprintSB.GroupBy("fingerprint") + + reducedLastSB.Where("reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints)") + reducedSumSB.Where("reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints)") + + cteQueries = append(cteQueries, + sqlbuilder.CTEQuery("__filtered_fingerprints").As(fingerprintSB), + sqlbuilder.CTEQuery("__reduced_filtered_fingerprints").As(reducedFingerprintSB), + ) + } else { + cteQueries = append(cteQueries, sqlbuilder.CTEQuery("__filtered_fingerprints").As(fingerprintSB)) + } } sampleCountsSB.GroupBy("metric_name") - cteQueries = append(cteQueries, - sqlbuilder.CTEQuery("__sample_counts").As(sampleCountsSB), - sqlbuilder.CTEQuery("__total_samples").As(totalSamplesSB), - ) + var finalSB *sqlbuilder.SelectBuilder + if reductionEnabled { + reducedLastSB.GroupBy("metric_name") + reducedSumSB.GroupBy("metric_name") - cteBuilder := sqlbuilder.With(cteQueries...) + reducedSampleCountsSB := sqlbuilder.NewSelectBuilder() + reducedSampleCountsSB.Select("metric_name", "sum(cnt) AS samples") + reducedSampleCountsSB.From(reducedSampleCountsSB.BuilderAs(sqlbuilder.UnionAll(reducedLastSB, reducedSumSB), "reduced_samples")) + reducedSampleCountsSB.GroupBy("metric_name") - finalSB := cteBuilder.Select( - "mc.metric_name", - "COALESCE(sc.samples, 0) AS samples", - "CASE WHEN ts.total_samples = 0 THEN 0 ELSE (COALESCE(sc.samples, 0) * 100.0 / ts.total_samples) END AS percentage", - ) - finalSB.From("__metric_candidates mc") - finalSB.JoinWithOption(sqlbuilder.LeftJoin, "__sample_counts sc", "mc.metric_name = sc.metric_name") - finalSB.Join("__total_samples ts", "1=1") + // Grand total includes reduced sample volume so percentages stay consistent. + reducedTotalLastSB := sqlbuilder.NewSelectBuilder() + reducedTotalLastSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt") + reducedTotalLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName)) + reducedTotalLastSB.Where(reducedTotalLastSB.Between("unix_milli", req.Start, req.End)) + + reducedTotalSumSB := sqlbuilder.NewSelectBuilder() + reducedTotalSumSB.Select("uniq(reduced_fingerprint, unix_milli) AS cnt") + reducedTotalSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName)) + reducedTotalSumSB.Where(reducedTotalSumSB.Between("unix_milli", req.Start, req.End)) + + reducedTotalSamplesSB := sqlbuilder.NewSelectBuilder() + reducedTotalSamplesSB.Select("sum(cnt) AS total_samples") + reducedTotalSamplesSB.From(reducedTotalSamplesSB.BuilderAs(sqlbuilder.UnionAll(reducedTotalLastSB, reducedTotalSumSB), "reduced_total")) + + allCandidatesSB := sqlbuilder.NewSelectBuilder() + allCandidatesSB.Select("DISTINCT metric_name") + allCandidatesSB.From(allCandidatesSB.BuilderAs(sqlbuilder.UnionAll( + sqlbuilder.NewSelectBuilder().Select("metric_name").From("__metric_candidates"), + sqlbuilder.NewSelectBuilder().Select("metric_name").From("__reduced_metric_candidates"), + ), "candidates")) + + cteQueries = append(cteQueries, + sqlbuilder.CTEQuery("__sample_counts").As(sampleCountsSB), + sqlbuilder.CTEQuery("__reduced_sample_counts").As(reducedSampleCountsSB), + sqlbuilder.CTEQuery("__total_samples").As(totalSamplesSB), + sqlbuilder.CTEQuery("__reduced_total_samples").As(reducedTotalSamplesSB), + sqlbuilder.CTEQuery("__all_candidates").As(allCandidatesSB), + ) + + cteBuilder := sqlbuilder.With(cteQueries...) + + finalSB = cteBuilder.Select( + "ac.metric_name", + "COALESCE(sc.samples, 0) + COALESCE(rsc.samples, 0) AS samples", + "CASE WHEN (ts.total_samples + rts.total_samples) = 0 THEN 0 ELSE ((COALESCE(sc.samples, 0) + COALESCE(rsc.samples, 0)) * 100.0 / (ts.total_samples + rts.total_samples)) END AS percentage", + ) + finalSB.From("__all_candidates ac") + finalSB.JoinWithOption(sqlbuilder.LeftJoin, "__sample_counts sc", "ac.metric_name = sc.metric_name") + finalSB.JoinWithOption(sqlbuilder.LeftJoin, "__reduced_sample_counts rsc", "ac.metric_name = rsc.metric_name") + finalSB.Join("__total_samples ts", "1=1") + finalSB.Join("__reduced_total_samples rts", "1=1") + } else { + cteQueries = append(cteQueries, + sqlbuilder.CTEQuery("__sample_counts").As(sampleCountsSB), + sqlbuilder.CTEQuery("__total_samples").As(totalSamplesSB), + ) + + cteBuilder := sqlbuilder.With(cteQueries...) + + finalSB = cteBuilder.Select( + "mc.metric_name", + "COALESCE(sc.samples, 0) AS samples", + "CASE WHEN ts.total_samples = 0 THEN 0 ELSE (COALESCE(sc.samples, 0) * 100.0 / ts.total_samples) END AS percentage", + ) + finalSB.From("__metric_candidates mc") + finalSB.JoinWithOption(sqlbuilder.LeftJoin, "__sample_counts sc", "mc.metric_name = sc.metric_name") + finalSB.Join("__total_samples ts", "1=1") + } finalSB.OrderBy("percentage DESC") finalSB.Limit(req.Limit) query, args := finalSB.BuildWithFlavor(sqlbuilder.ClickHouse) + if reductionEnabled { + // unmatched LEFT JOIN sample columns must be NULL (not 0/'') so COALESCE applies + query += " SETTINGS join_use_nulls = 1" + } valueCtx := ctxtypes.SetClickhouseMaxThreads(ctx, m.config.TelemetryStore.Threads) db := m.telemetryStore.ClickhouseDB() diff --git a/pkg/modules/metricsexplorer/implmetricsexplorer/module_test.go b/pkg/modules/metricsexplorer/implmetricsexplorer/module_test.go index 31cd2d5883..e677ff4a0a 100644 --- a/pkg/modules/metricsexplorer/implmetricsexplorer/module_test.go +++ b/pkg/modules/metricsexplorer/implmetricsexplorer/module_test.go @@ -10,6 +10,8 @@ import ( "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/cache/cachetest" "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/flagger" + "github.com/SigNoz/signoz/pkg/flagger/configflagger" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/modules/metricsexplorer" "github.com/SigNoz/signoz/pkg/modules/metricsexplorer/implmetricsexplorer" @@ -28,13 +30,20 @@ const ( // to stable table names that the expected SQL strings can hard-code). testStartMillis int64 = 1700000000000 testEndMillis int64 = 1700003600000 // +1h - statsNoFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ?) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?" - statsOrderTimeseriesSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ?) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY timeseries ASC, metric_name ASC LIMIT ? OFFSET ?" - statsWithFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?" - treemapTimeseriesNoFilterSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND __normalized = ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name) SELECT mt.metric_name, mt.total_value, CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage FROM __metric_totals mt JOIN __total_time_series tts ON 1=1 ORDER BY percentage DESC LIMIT ?" - treemapTimeseriesWithFilterSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND __normalized = ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name) SELECT mt.metric_name, mt.total_value, CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage FROM __metric_totals mt JOIN __total_time_series tts ON 1=1 ORDER BY percentage DESC LIMIT ?" - treemapSamplesNoFilterSQL = "WITH __metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4 WHERE NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND unix_milli BETWEEN ? AND ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) GROUP BY metric_name), __total_samples AS (SELECT count(*) AS total_samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ?) SELECT mc.metric_name, COALESCE(sc.samples, 0) AS samples, CASE WHEN ts.total_samples = 0 THEN 0 ELSE (COALESCE(sc.samples, 0) * 100.0 / ts.total_samples) END AS percentage FROM __metric_candidates mc LEFT JOIN __sample_counts sc ON mc.metric_name = sc.metric_name JOIN __total_samples ts ON 1=1 ORDER BY percentage DESC LIMIT ?" - treemapSamplesWithFilterSQL = "WITH __metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4 WHERE NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND unix_milli BETWEEN ? AND ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name), __total_samples AS (SELECT count(*) AS total_samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ?) SELECT mc.metric_name, COALESCE(sc.samples, 0) AS samples, CASE WHEN ts.total_samples = 0 THEN 0 ELSE (COALESCE(sc.samples, 0) * 100.0 / ts.total_samples) END AS percentage FROM __metric_candidates mc LEFT JOIN __sample_counts sc ON mc.metric_name = sc.metric_name JOIN __total_samples ts ON 1=1 ORDER BY percentage DESC LIMIT ?" + statsNoFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz')) GROUP BY metric_name), __reduced_time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name) SELECT COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __reduced_time_series_counts rts ON ts.metric_name = rts.metric_name FULL OUTER JOIN __sample_counts s ON COALESCE(ts.metric_name, rts.metric_name) = s.metric_name FULL OUTER JOIN __reduced_sample_counts rs ON COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name WHERE (COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ? SETTINGS join_use_nulls = 1" + statsOrderTimeseriesSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz')) GROUP BY metric_name), __reduced_time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name) SELECT COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __reduced_time_series_counts rts ON ts.metric_name = rts.metric_name FULL OUTER JOIN __sample_counts s ON COALESCE(ts.metric_name, rts.metric_name) = s.metric_name FULL OUTER JOIN __reduced_sample_counts rs ON COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name WHERE (COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0) ORDER BY timeseries ASC, metric_name ASC LIMIT ? OFFSET ? SETTINGS join_use_nulls = 1" + statsWithFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name), __reduced_filtered_fingerprints AS (SELECT metric_name, fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name, fingerprint), __reduced_time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM __reduced_filtered_fingerprints GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints) GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints) GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name) SELECT COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __reduced_time_series_counts rts ON ts.metric_name = rts.metric_name FULL OUTER JOIN __sample_counts s ON COALESCE(ts.metric_name, rts.metric_name) = s.metric_name FULL OUTER JOIN __reduced_sample_counts rs ON COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name WHERE (COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ? SETTINGS join_use_nulls = 1" + treemapTimeseriesNoFilterSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name), __reduced_total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ?), __reduced_metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name) SELECT COALESCE(mt.metric_name, rmt.metric_name) AS metric_name, COALESCE(mt.total_value, 0) + COALESCE(rmt.total_value, 0) AS total_value, CASE WHEN (tts.total_time_series + rtts.total_time_series) = 0 THEN 0 ELSE ((COALESCE(mt.total_value, 0) + COALESCE(rmt.total_value, 0)) * 100.0 / (tts.total_time_series + rtts.total_time_series)) END AS percentage FROM __metric_totals mt FULL OUTER JOIN __reduced_metric_totals rmt ON mt.metric_name = rmt.metric_name JOIN __total_time_series tts ON 1=1 JOIN __reduced_total_time_series rtts ON 1=1 ORDER BY percentage DESC LIMIT ? SETTINGS join_use_nulls = 1" + treemapTimeseriesWithFilterSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name), __reduced_total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ?), __reduced_metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name) SELECT COALESCE(mt.metric_name, rmt.metric_name) AS metric_name, COALESCE(mt.total_value, 0) + COALESCE(rmt.total_value, 0) AS total_value, CASE WHEN (tts.total_time_series + rtts.total_time_series) = 0 THEN 0 ELSE ((COALESCE(mt.total_value, 0) + COALESCE(rmt.total_value, 0)) * 100.0 / (tts.total_time_series + rtts.total_time_series)) END AS percentage FROM __metric_totals mt FULL OUTER JOIN __reduced_metric_totals rmt ON mt.metric_name = rmt.metric_name JOIN __total_time_series tts ON 1=1 JOIN __reduced_total_time_series rtts ON 1=1 ORDER BY percentage DESC LIMIT ? SETTINGS join_use_nulls = 1" + treemapSamplesNoFilterSQL = "WITH __metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4 WHERE NOT startsWith(metric_name, 'signoz') AND unix_milli BETWEEN ? AND ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __reduced_metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4_reduced WHERE NOT startsWith(metric_name, 'signoz') AND unix_milli BETWEEN ? AND ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates) GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates) GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name), __total_samples AS (SELECT count(*) AS total_samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ?), __reduced_total_samples AS (SELECT sum(cnt) AS total_samples FROM ((SELECT uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ?) UNION ALL (SELECT uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ?)) AS reduced_total), __all_candidates AS (SELECT DISTINCT metric_name FROM ((SELECT metric_name FROM __metric_candidates) UNION ALL (SELECT metric_name FROM __reduced_metric_candidates)) AS candidates) SELECT ac.metric_name, COALESCE(sc.samples, 0) + COALESCE(rsc.samples, 0) AS samples, CASE WHEN (ts.total_samples + rts.total_samples) = 0 THEN 0 ELSE ((COALESCE(sc.samples, 0) + COALESCE(rsc.samples, 0)) * 100.0 / (ts.total_samples + rts.total_samples)) END AS percentage FROM __all_candidates ac LEFT JOIN __sample_counts sc ON ac.metric_name = sc.metric_name LEFT JOIN __reduced_sample_counts rsc ON ac.metric_name = rsc.metric_name JOIN __total_samples ts ON 1=1 JOIN __reduced_total_samples rts ON 1=1 ORDER BY percentage DESC LIMIT ? SETTINGS join_use_nulls = 1" + treemapSamplesWithFilterSQL = "WITH __metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4 WHERE NOT startsWith(metric_name, 'signoz') AND unix_milli BETWEEN ? AND ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __reduced_metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4_reduced WHERE NOT startsWith(metric_name, 'signoz') AND unix_milli BETWEEN ? AND ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) GROUP BY fingerprint), __reduced_filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? AND metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates) GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates) AND reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints) GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __reduced_metric_candidates) AND reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints) GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name), __total_samples AS (SELECT count(*) AS total_samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ?), __reduced_total_samples AS (SELECT sum(cnt) AS total_samples FROM ((SELECT uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ?) UNION ALL (SELECT uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ?)) AS reduced_total), __all_candidates AS (SELECT DISTINCT metric_name FROM ((SELECT metric_name FROM __metric_candidates) UNION ALL (SELECT metric_name FROM __reduced_metric_candidates)) AS candidates) SELECT ac.metric_name, COALESCE(sc.samples, 0) + COALESCE(rsc.samples, 0) AS samples, CASE WHEN (ts.total_samples + rts.total_samples) = 0 THEN 0 ELSE ((COALESCE(sc.samples, 0) + COALESCE(rsc.samples, 0)) * 100.0 / (ts.total_samples + rts.total_samples)) END AS percentage FROM __all_candidates ac LEFT JOIN __sample_counts sc ON ac.metric_name = sc.metric_name LEFT JOIN __reduced_sample_counts rsc ON ac.metric_name = rsc.metric_name JOIN __total_samples ts ON 1=1 JOIN __reduced_total_samples rts ON 1=1 ORDER BY percentage DESC LIMIT ? SETTINGS join_use_nulls = 1" + + // Raw-only SQL produced when the metrics-reduction feature flag is OFF. These + // must stay byte-for-byte identical to the pre-reduction queries. + statsNoFilterRawSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz')) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?" + statsWithFilterRawSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?" + treemapTimeseriesNoFilterRawSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name) SELECT mt.metric_name, mt.total_value, CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage FROM __metric_totals mt JOIN __total_time_series tts ON 1=1 ORDER BY percentage DESC LIMIT ?" + treemapSamplesNoFilterRawSQL = "WITH __metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4 WHERE NOT startsWith(metric_name, 'signoz') AND unix_milli BETWEEN ? AND ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) GROUP BY metric_name), __total_samples AS (SELECT count(*) AS total_samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ?) SELECT mc.metric_name, COALESCE(sc.samples, 0) AS samples, CASE WHEN ts.total_samples = 0 THEN 0 ELSE (COALESCE(sc.samples, 0) * 100.0 / ts.total_samples) END AS percentage FROM __metric_candidates mc LEFT JOIN __sample_counts sc ON mc.metric_name = sc.metric_name JOIN __total_samples ts ON 1=1 ORDER BY percentage DESC LIMIT ?" ) var testOrgID = valuer.GenerateUUID() @@ -43,9 +52,32 @@ type statsOpt func(*metricsexplorertypes.StatsRequest) type treemapOpt func(*metricsexplorertypes.TreemapRequest) +func newFlagger(t *testing.T, reductionEnabled bool) flagger.Flagger { + t.Helper() + registry := flagger.MustNewRegistry() + cfg := flagger.Config{} + if reductionEnabled { + cfg.Config.Boolean = map[string]bool{ + flagger.FeatureEnableMetricsReduction.String(): true, + } + } + fl, err := flagger.New( + context.Background(), + instrumentationtest.New().ToProviderSettings(), + cfg, + registry, + configflagger.NewFactory(registry), + ) + if err != nil { + t.Fatalf("flagger.New: %v", err) + } + return fl +} + // newTestModule builds the metricsexplorer module backed by a mocked clickhouse -// connection, a mock metadata store, and an in-memory cache. -func newTestModule(t *testing.T, matcher sqlmock.QueryMatcher) (metricsexplorer.Module, cmock.ClickConnMockCommon, *telemetrytypestest.MockMetadataStore) { +// connection, a mock metadata store, and an in-memory cache. reductionEnabled +// toggles the metrics-reduction feature flag. +func newTestModule(t *testing.T, matcher sqlmock.QueryMatcher, reductionEnabled bool) (metricsexplorer.Module, cmock.ClickConnMockCommon, *telemetrytypestest.MockMetadataStore) { t.Helper() ts := telemetrystoretest.New(telemetrystore.Config{}, matcher) @@ -56,7 +88,7 @@ func newTestModule(t *testing.T, matcher sqlmock.QueryMatcher) (metricsexplorer. } settings := instrumentationtest.New().ToProviderSettings() - mod := implmetricsexplorer.NewModule(ts, md, c, nil /*ruleStore*/, nil /*dashboardModule*/, settings, metricsexplorer.Config{}) + mod := implmetricsexplorer.NewModule(ts, md, c, nil /*ruleStore*/, nil /*dashboardModule*/, newFlagger(t, reductionEnabled), settings, metricsexplorer.Config{}) return mod, ts.Mock(), md } @@ -149,27 +181,31 @@ func treemapEntryRows() *cmock.Rows { func TestGetStats(t *testing.T) { tests := []struct { - name string - opts []statsOpt - seedKey string - queryErr error - expectSQL string - argCount int - noQuery bool // SQL never reaches clickhouse (validation/build error) - wantCode errors.Code + name string + opts []statsOpt + seedKey string + queryErr error + expectSQL string + argCount int + reductionEnabled bool + noQuery bool // SQL never reaches clickhouse (validation/build error) + wantCode errors.Code }{ - {name: "NoFilter_FastPathSQL", expectSQL: statsNoFilterSQL, argCount: 10}, - {name: "WhitespaceFilter_FastPathSQL", opts: []statsOpt{withStatsFilter(" ")}, expectSQL: statsNoFilterSQL, argCount: 10}, - {name: "WithFilter_FingerprintSQL", opts: []statsOpt{withStatsFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: statsWithFilterSQL, argCount: 12}, - {name: "OrderByTimeseriesAsc", opts: []statsOpt{withStatsOrderBy("timeseries", qbtypes.OrderDirectionAsc)}, expectSQL: statsOrderTimeseriesSQL, argCount: 10}, + {name: "NoFilter_FastPathSQL", expectSQL: statsNoFilterSQL, argCount: 14, reductionEnabled: true}, + {name: "WhitespaceFilter_FastPathSQL", opts: []statsOpt{withStatsFilter(" ")}, expectSQL: statsNoFilterSQL, argCount: 14, reductionEnabled: true}, + {name: "WithFilter_FingerprintSQL", opts: []statsOpt{withStatsFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: statsWithFilterSQL, argCount: 17, reductionEnabled: true}, + {name: "OrderByTimeseriesAsc", opts: []statsOpt{withStatsOrderBy("timeseries", qbtypes.OrderDirectionAsc)}, expectSQL: statsOrderTimeseriesSQL, argCount: 14, reductionEnabled: true}, {name: "OrderByInvalid", opts: []statsOpt{withStatsOrderBy("nonsense", qbtypes.OrderDirectionAsc)}, noQuery: true, wantCode: errors.CodeInvalidInput}, - {name: "QueryError", queryErr: assert.AnError, expectSQL: statsNoFilterSQL, argCount: 10, wantCode: errors.CodeInternal}, + {name: "QueryError", queryErr: assert.AnError, expectSQL: statsNoFilterSQL, argCount: 14, reductionEnabled: true, wantCode: errors.CodeInternal}, {name: "InvalidRequest_Limit", opts: []statsOpt{withStatsLimit(0)}, noQuery: true, wantCode: errors.CodeInvalidInput}, + // reduction off + {name: "ReductionOff_NoFilter_RawSQL", expectSQL: statsNoFilterRawSQL, argCount: 8}, + {name: "ReductionOff_WithFilter_RawSQL", opts: []statsOpt{withStatsFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: statsWithFilterRawSQL, argCount: 10}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - mod, mock, md := newTestModule(t, sqlmock.QueryMatcherRegexp) + mod, mock, md := newTestModule(t, sqlmock.QueryMatcherRegexp, tc.reductionEnabled) if tc.seedKey != "" { seedFilterKey(md, tc.seedKey) } @@ -202,34 +238,38 @@ func TestGetTreemap(t *testing.T) { } tests := []struct { - name string - mode metricsexplorertypes.TreemapMode - opts []treemapOpt - seedKey string - queryErr error - expectSQL string - argCount int - rows *cmock.Rows - wantSamples []metricsexplorertypes.TreemapEntry - wantTS []metricsexplorertypes.TreemapEntry - noQuery bool - wantCode errors.Code - wantErr bool + name string + mode metricsexplorertypes.TreemapMode + opts []treemapOpt + seedKey string + queryErr error + expectSQL string + argCount int + reductionEnabled bool + rows *cmock.Rows + wantSamples []metricsexplorertypes.TreemapEntry + wantTS []metricsexplorertypes.TreemapEntry + noQuery bool + wantCode errors.Code + wantErr bool }{ - {name: "TimeSeries_NoFilter_SQL", mode: metricsexplorertypes.TreemapModeTimeSeries, expectSQL: treemapTimeseriesNoFilterSQL, argCount: 7}, - {name: "TimeSeries_WithFilter_SQL", mode: metricsexplorertypes.TreemapModeTimeSeries, opts: []treemapOpt{withTreemapFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: treemapTimeseriesWithFilterSQL, argCount: 8}, - {name: "TimeSeries_ScansEntries", mode: metricsexplorertypes.TreemapModeTimeSeries, expectSQL: treemapTimeseriesNoFilterSQL, argCount: 7, rows: treemapEntryRows(), wantTS: wantEntries}, - {name: "Samples_NoFilter_SQL", mode: metricsexplorertypes.TreemapModeSamples, expectSQL: treemapSamplesNoFilterSQL, argCount: 9}, - {name: "Samples_WithFilter_SQL", mode: metricsexplorertypes.TreemapModeSamples, opts: []treemapOpt{withTreemapFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: treemapSamplesWithFilterSQL, argCount: 14}, - {name: "Samples_ScansEntries", mode: metricsexplorertypes.TreemapModeSamples, expectSQL: treemapSamplesNoFilterSQL, argCount: 9, rows: treemapEntryRows(), wantSamples: wantEntries}, + {name: "TimeSeries_NoFilter_SQL", mode: metricsexplorertypes.TreemapModeTimeSeries, expectSQL: treemapTimeseriesNoFilterSQL, argCount: 9, reductionEnabled: true}, + {name: "TimeSeries_WithFilter_SQL", mode: metricsexplorertypes.TreemapModeTimeSeries, opts: []treemapOpt{withTreemapFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: treemapTimeseriesWithFilterSQL, argCount: 11, reductionEnabled: true}, + {name: "TimeSeries_ScansEntries", mode: metricsexplorertypes.TreemapModeTimeSeries, expectSQL: treemapTimeseriesNoFilterSQL, argCount: 9, reductionEnabled: true, rows: treemapEntryRows(), wantTS: wantEntries}, + {name: "Samples_NoFilter_SQL", mode: metricsexplorertypes.TreemapModeSamples, expectSQL: treemapSamplesNoFilterSQL, argCount: 19, reductionEnabled: true}, + {name: "Samples_WithFilter_SQL", mode: metricsexplorertypes.TreemapModeSamples, opts: []treemapOpt{withTreemapFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: treemapSamplesWithFilterSQL, argCount: 27, reductionEnabled: true}, + {name: "Samples_ScansEntries", mode: metricsexplorertypes.TreemapModeSamples, expectSQL: treemapSamplesNoFilterSQL, argCount: 19, reductionEnabled: true, rows: treemapEntryRows(), wantSamples: wantEntries}, {name: "FilterBuildError", mode: metricsexplorertypes.TreemapModeTimeSeries, opts: []treemapOpt{withTreemapFilter("host.name =")}, noQuery: true, wantErr: true}, - {name: "QueryError", mode: metricsexplorertypes.TreemapModeTimeSeries, queryErr: assert.AnError, expectSQL: treemapTimeseriesNoFilterSQL, argCount: 7, wantCode: errors.CodeInternal}, + {name: "QueryError", mode: metricsexplorertypes.TreemapModeTimeSeries, queryErr: assert.AnError, expectSQL: treemapTimeseriesNoFilterSQL, argCount: 7, reductionEnabled: true, wantCode: errors.CodeInternal}, {name: "InvalidMode", mode: metricsexplorertypes.TreemapMode{}, noQuery: true, wantCode: errors.CodeInvalidInput}, + // reduction off + {name: "ReductionOff_TimeSeries_NoFilter_RawSQL", mode: metricsexplorertypes.TreemapModeTimeSeries, expectSQL: treemapTimeseriesNoFilterRawSQL, argCount: 5}, + {name: "ReductionOff_Samples_NoFilter_RawSQL", mode: metricsexplorertypes.TreemapModeSamples, expectSQL: treemapSamplesNoFilterRawSQL, argCount: 8}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - mod, mock, md := newTestModule(t, sqlmock.QueryMatcherRegexp) + mod, mock, md := newTestModule(t, sqlmock.QueryMatcherRegexp, tc.reductionEnabled) if tc.seedKey != "" { seedFilterKey(md, tc.seedKey) } diff --git a/pkg/prometheus/clickhouseprometheus/capture.go b/pkg/prometheus/clickhouseprometheus/capture.go new file mode 100644 index 0000000000..840f5bcd89 --- /dev/null +++ b/pkg/prometheus/clickhouseprometheus/capture.go @@ -0,0 +1,88 @@ +package clickhouseprometheus + +import ( + "context" + "sync" + + "github.com/SigNoz/signoz/pkg/prometheus" + "github.com/prometheus/prometheus/prompb" + "github.com/prometheus/prometheus/storage" +) + +// statementRecorder collects the statements a PromQL evaluation would run. +// Safe for concurrent use: the engine may Select selectors concurrently. +type statementRecorder struct { + mu sync.Mutex + statements []prometheus.CapturedStatement +} + +func (r *statementRecorder) record(query string, args []any) { + r.mu.Lock() + defer r.mu.Unlock() + r.statements = append(r.statements, prometheus.CapturedStatement{Query: query, Args: args}) +} + +func (r *statementRecorder) Statements() []prometheus.CapturedStatement { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]prometheus.CapturedStatement, len(r.statements)) + copy(out, r.statements) + return out +} + +// captureClient builds the same SQL as the real client but records it and +// returns an empty result instead of executing. +type captureClient struct { + *client + recorder *statementRecorder +} + +func (c *captureClient) Read(ctx context.Context, query *prompb.Query, _ bool) (storage.SeriesSet, error) { + // Raw-SQL passthrough ({job="rawsql", query="..."}): record the raw query. + if len(query.Matchers) == 2 { + var hasJob bool + var queryString string + for _, m := range query.Matchers { + if m.Type == prompb.LabelMatcher_EQ && m.Name == "job" && m.Value == "rawsql" { + hasJob = true + } + if m.Type == prompb.LabelMatcher_EQ && m.Name == "query" { + queryString = m.Value + } + } + if hasJob && queryString != "" { + c.recorder.record(queryString, nil) + return storage.EmptySeriesSet(), nil + } + } + + var metricName string + for _, matcher := range query.Matchers { + if matcher.Name == "__name__" { + metricName = matcher.Value + } + } + + // Build the executing path's queries, but only record them. + subQuery, args, err := c.queryToClickhouseQuery(ctx, query, metricName, true) + if err != nil { + return nil, err + } + samplesQuery, samplesArgs := buildSamplesQuery(int64(query.StartTimestampMs), int64(query.EndTimestampMs), metricName, subQuery, args) + c.recorder.record(samplesQuery, samplesArgs) + + return storage.EmptySeriesSet(), nil +} + +// captureQueryable adapts the capturing read client to storage.Queryable. +type captureQueryable struct { + inner storage.SampleAndChunkQueryable +} + +func (c captureQueryable) Querier(mint, maxt int64) (storage.Querier, error) { + querier, err := c.inner.Querier(mint, maxt) + if err != nil { + return nil, err + } + return storage.NewMergeQuerier(nil, []storage.Querier{querier}, storage.ChainedSeriesMerge), nil +} diff --git a/pkg/prometheus/clickhouseprometheus/client.go b/pkg/prometheus/clickhouseprometheus/client.go index 2446505b9a..14439cff55 100644 --- a/pkg/prometheus/clickhouseprometheus/client.go +++ b/pkg/prometheus/clickhouseprometheus/client.go @@ -204,8 +204,9 @@ func (client *client) getFingerprintsFromClickhouseQuery(ctx context.Context, qu return fingerprints, nil } -func (client *client) querySamples(ctx context.Context, start int64, end int64, fingerprints map[uint64][]prompb.Label, metricName string, subQuery string, args []any) ([]*prompb.TimeSeries, error) { - ctx = client.withClickhousePrometheusContext(ctx, "querySamples") +// buildSamplesQuery renders the samples SQL (and args) that fetches data +// points for the series selected by subQuery. +func buildSamplesQuery(start int64, end int64, metricName string, subQuery string, args []any) (string, []any) { argCount := len(args) query := fmt.Sprintf(` @@ -217,6 +218,13 @@ func (client *client) querySamples(ctx context.Context, start int64, end int64, allArgs := append([]any{metricName}, args...) allArgs = append(allArgs, start, end) + return query, allArgs +} + +func (client *client) querySamples(ctx context.Context, start int64, end int64, fingerprints map[uint64][]prompb.Label, metricName string, subQuery string, args []any) ([]*prompb.TimeSeries, error) { + ctx = client.withClickhousePrometheusContext(ctx, "querySamples") + + query, allArgs := buildSamplesQuery(start, end, metricName, subQuery, args) rows, err := client.telemetryStore.ClickhouseDB().Query(ctx, query, allArgs...) if err != nil { diff --git a/pkg/prometheus/clickhouseprometheus/client_query_test.go b/pkg/prometheus/clickhouseprometheus/client_query_test.go index 0a1e75d21e..38e96b6f37 100644 --- a/pkg/prometheus/clickhouseprometheus/client_query_test.go +++ b/pkg/prometheus/clickhouseprometheus/client_query_test.go @@ -5,8 +5,8 @@ import ( "sort" "testing" - "github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest" cmock "github.com/SigNoz/clickhouse-go-mock" + "github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest" "github.com/stretchr/testify/require" "github.com/DATA-DOG/go-sqlmock" diff --git a/pkg/prometheus/clickhouseprometheus/provider.go b/pkg/prometheus/clickhouseprometheus/provider.go index ee0dc5a896..f42b0634da 100644 --- a/pkg/prometheus/clickhouseprometheus/provider.go +++ b/pkg/prometheus/clickhouseprometheus/provider.go @@ -64,3 +64,15 @@ func (provider *provider) Querier(mint, maxt int64) (storage.Querier, error) { return storage.NewMergeQuerier(nil, []storage.Querier{querier}, storage.ChainedSeriesMerge), nil } + +// CapturingStorage implements prometheus.StatementCapturer. Uses a fresh +// recorder per call so concurrent dry-runs don't share state. +func (provider *provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) { + recorder := &statementRecorder{} + capture := &captureClient{ + client: &client{settings: provider.settings, telemetryStore: provider.telemetryStore}, + recorder: recorder, + } + queryable := remote.NewSampleAndChunkQueryableClient(capture, labels.EmptyLabels(), []*labels.Matcher{}, false, stCallback) + return captureQueryable{inner: queryable}, recorder +} diff --git a/pkg/prometheus/prometheus.go b/pkg/prometheus/prometheus.go index c3ce3f50f7..bbe6452140 100644 --- a/pkg/prometheus/prometheus.go +++ b/pkg/prometheus/prometheus.go @@ -15,3 +15,23 @@ type Prometheus interface { Storage() storage.Queryable Parser() Parser } + +// CapturedStatement is one datastore statement a PromQL query would run, +// captured without executing. +type CapturedStatement struct { + Query string + Args []any +} + +// StatementRecorder reads back the statements captured against a capturing +// Storage (see StatementCapturer). +type StatementRecorder interface { + Statements() []CapturedStatement +} + +// StatementCapturer is an optional Prometheus-provider capability, discovered +// via type assertion: it returns a Storage that records each Select's statement +// without executing it, plus a recorder to read them back. +type StatementCapturer interface { + CapturingStorage() (storage.Queryable, StatementRecorder) +} diff --git a/pkg/querier/api.go b/pkg/querier/api.go index aff1611a48..20f67ae590 100644 --- a/pkg/querier/api.go +++ b/pkg/querier/api.go @@ -73,6 +73,53 @@ func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) { render.Success(rw, http.StatusOK, queryRangeResponse) } + +// QueryRangePreview is the dry-run counterpart of QueryRange: it validates and +// renders each query without executing it. +func (handler *handler) QueryRangePreview(rw http.ResponseWriter, req *http.Request) { + ctx := req.Context() + ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ + instrumentationtypes.CodeNamespace: "querier", + instrumentationtypes.CodeFunctionName: "QueryRangePreview", + }) + + claims, err := authtypes.ClaimsFromContext(ctx) + if err != nil { + render.Error(rw, err) + return + } + + var queryRangeRequest qbtypes.QueryRangeRequest + if err := json.NewDecoder(req.Body).Decode(&queryRangeRequest); err != nil { + render.Error(rw, err) + return + } + + // Validation is deferred to QueryRangePreview, which reports per-query + // errors instead of failing fast. + + orgID, err := valuer.NewUUID(claims.OrgID) + if err != nil { + render.Error(rw, err) + return + } + + previewParams := qbtypes.QueryRangePreviewParams{Verbose: req.URL.Query().Get("verbose")} + previewOpts, err := previewParams.Validate() + if err != nil { + render.Error(rw, err) + return + } + + preview, err := handler.querier.QueryRangePreview(ctx, orgID, &queryRangeRequest, previewOpts) + if err != nil { + render.Error(rw, err) + return + } + + render.Success(rw, http.StatusOK, preview) +} + func (handler *handler) QueryRawStream(rw http.ResponseWriter, req *http.Request) { ctx := req.Context() diff --git a/pkg/querier/builder_query.go b/pkg/querier/builder_query.go index 3cdd377c97..d4adb2beae 100644 --- a/pkg/querier/builder_query.go +++ b/pkg/querier/builder_query.go @@ -36,6 +36,7 @@ type builderQuery[T any] struct { } var _ qbtypes.Query = (*builderQuery[any])(nil) +var _ qbtypes.StatementProvider = (*builderQuery[any])(nil) type builderConfig struct { logTraceIDWindowPaddingMS uint64 @@ -211,6 +212,11 @@ func (q *builderQuery[T]) isWindowList() bool { return true } +// Statement renders the SQL without executing it, for the preview path. +func (q *builderQuery[T]) Statement(ctx context.Context) (*qbtypes.Statement, error) { + return q.stmtBuilder.Build(ctx, q.fromMS, q.toMS, q.kind, q.spec, q.variables) +} + func (q *builderQuery[T]) Execute(ctx context.Context) (*qbtypes.Result, error) { // can we do window based pagination? diff --git a/pkg/querier/clickhouse_query.go b/pkg/querier/clickhouse_query.go index bbce735120..52f035f435 100644 --- a/pkg/querier/clickhouse_query.go +++ b/pkg/querier/clickhouse_query.go @@ -32,6 +32,7 @@ type chSQLQuery struct { } var _ qbtypes.Query = (*chSQLQuery)(nil) +var _ qbtypes.StatementProvider = (*chSQLQuery)(nil) func newchSQLQuery( logger *slog.Logger, @@ -99,6 +100,15 @@ func (q *chSQLQuery) renderVars(query string, vars map[string]qbtypes.VariableIt return newQuery.String(), nil } +// Statement renders the SQL without executing it, for the preview path. +func (q *chSQLQuery) Statement(_ context.Context) (*qbtypes.Statement, error) { + rendered, err := q.renderVars(q.query.Query, q.vars, q.fromMS, q.toMS) + if err != nil { + return nil, err + } + return &qbtypes.Statement{Query: rendered, Args: q.args}, nil +} + func (q *chSQLQuery) Execute(ctx context.Context) (*qbtypes.Result, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.QueryDuration: instrumentationtypes.DurationBucket(q.fromMS, q.toMS), diff --git a/pkg/querier/interfaces.go b/pkg/querier/interfaces.go index 06a2a1befa..4bfb9f0306 100644 --- a/pkg/querier/interfaces.go +++ b/pkg/querier/interfaces.go @@ -14,6 +14,8 @@ type Querier interface { QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream) statsreporter.StatsCollector + // QueryRangePreview validates and renders the queries without executing them. + QueryRangePreview(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, opts qbtypes.QueryRangePreviewOptions) (*qbtypes.QueryRangePreviewResponse, error) } // BucketCache is the interface for bucket-based caching. @@ -26,6 +28,8 @@ type BucketCache interface { type Handler interface { QueryRange(rw http.ResponseWriter, req *http.Request) + // QueryRangePreview is the dry-run endpoint: validate and render without executing. + QueryRangePreview(rw http.ResponseWriter, req *http.Request) QueryRawStream(rw http.ResponseWriter, req *http.Request) ReplaceVariables(rw http.ResponseWriter, req *http.Request) } diff --git a/pkg/querier/preview.go b/pkg/querier/preview.go new file mode 100644 index 0000000000..f2fabc19d1 --- /dev/null +++ b/pkg/querier/preview.go @@ -0,0 +1,338 @@ +package querier + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + + "github.com/SigNoz/signoz/pkg/errors" + qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" + "github.com/SigNoz/signoz/pkg/valuer" +) + +// QueryRangePreview validates and renders each query without executing it. +// When opts.Verbose, it also attaches each statement's EXPLAIN ESTIMATE and +// granule analysis. +func (q *querier) QueryRangePreview( + ctx context.Context, + orgID valuer.UUID, + req *qbtypes.QueryRangeRequest, + opts qbtypes.QueryRangePreviewOptions, +) (*qbtypes.QueryRangePreviewResponse, error) { + + validationOpts, err := req.ValidateRequestScope() + if err != nil { + return nil, err + } + + dependencyQueries, err := q.constructTraceOperatorDependencyMap(req.CompositeQuery.Queries) + if err != nil { + return nil, err + } + + results := make(map[string]qbtypes.QueryPreview, len(req.CompositeQuery.Queries)) + + prepared := make(map[string]qbtypes.QueryPreview, len(req.CompositeQuery.Queries)) + missingMetricQuerySet := make(map[string]bool) + for idx := range req.CompositeQuery.Queries { + name := req.CompositeQuery.Queries[idx].GetQueryName() + ps := qbtypes.QueryPreview{Warnings: []string{}, Statements: []qbtypes.PreviewStatement{}} + + if vErr := req.CompositeQuery.Queries[idx].Validate(validationOpts...); vErr != nil { + ps.Error = vErr + prepared[name] = ps + continue + } + + env := []qbtypes.QueryEnvelope{req.CompositeQuery.Queries[idx]} + ps.Warnings = append(ps.Warnings, q.adjustStepInterval(env, req.Start, req.End)...) + + missingMetricQueries, metricWarnings, mErr := q.resolveMetricMetadata(ctx, orgID, env, req.Start, req.End) + if mErr != nil { + // Report this query's error but keep previewing the rest. + ps.Error = mErr + } else { + ps.Warnings = append(ps.Warnings, metricWarnings...) + if len(missingMetricQueries) > 0 { + missingMetricQuerySet[name] = true + if len(metricWarnings) == 0 { + if metricNames := missingMetricNames(env[0]); len(metricNames) > 0 { + ps.Warnings = append(ps.Warnings, fmt.Sprintf( + "query %q references metric(s) %s with no data available; it will return an empty result", + name, strings.Join(metricNames, ", "))) + } + } + } + } + + req.CompositeQuery.Queries[idx] = env[0] + prepared[name] = ps + } + + skip := make(map[string]bool, len(prepared)) + for name, ps := range prepared { + if ps.Error != nil || missingMetricQuerySet[name] { + skip[name] = true + } + } + providers, buildErrs := q.buildPreviewProviders(req, dependencyQueries, missingMetricQuerySet, skip) + + // Render each executing query's statement and collect the ClickHouse-bound + // analysis work to run concurrently. + var previewTasks []qbtypes.PreviewTask + for _, query := range req.CompositeQuery.Queries { + name := query.GetQueryName() + ps := prepared[name] + + if ps.Error != nil { + results[name] = ps + continue + } + if missingMetricQuerySet[name] { + results[name] = ps + continue + } + if bErr := buildErrs[name]; bErr != nil { + ps.Error = bErr + results[name] = ps + continue + } + + provider, ok := providers[name] + if !ok { + if !rendersStandaloneStatement(query.Type) { + ps.Warnings = append(ps.Warnings, fmt.Sprintf( + "query type %q has no standalone statement to preview; it is evaluated from the queries it references", query.Type.StringValue())) + results[name] = ps + continue + } + ps.Error = errors.NewInternalf(errors.CodeInternal, "query produced no provider") + results[name] = ps + continue + } + + stmtProvider, ok := provider.(qbtypes.StatementProvider) + if !ok { + ps.Error = errors.NewInternalf(errors.CodeInternal, "query does not support preview") + results[name] = ps + continue + } + + stmt, sErr := stmtProvider.Statement(ctx) + if sErr != nil { + ps.Error = sErr + results[name] = ps + continue + } + + ps.Warnings = append(ps.Warnings, stmt.Warnings...) + + if query.Type == qbtypes.QueryTypeClickHouseSQL { + if bindErr := q.telemetryStore.Plan(ctx, stmt.Query, stmt.Args...); bindErr != nil { + if errors.Ast(bindErr, errors.TypeInvalidInput) || errors.Ast(bindErr, errors.TypeNotFound) { + ps.Error = bindErr + results[name] = ps + continue + } + ps.Warnings = append(ps.Warnings, "could not validate ClickHouse SQL: "+bindErr.Error()) + } + } + + if !opts.Verbose { + results[name] = ps + continue + } + + if query.Type == qbtypes.QueryTypePromQL { + if pq, ok := provider.(*promqlQuery); ok { + sqlStmts, pErr := pq.PreviewStatements(ctx) + if pErr != nil { + ps.Warnings = append(ps.Warnings, "could not render underlying ClickHouse SQL: "+pErr.Error()) + } else { + for _, s := range sqlStmts { + ps.Statements = append(ps.Statements, qbtypes.PreviewStatement{Query: s.Query, Args: orEmpty(s.Args), Estimate: []telemetrystoretypes.EstimateEntry{}}) + } + } + } + } else { + ps.Statements = []qbtypes.PreviewStatement{{Query: stmt.Query, Args: orEmpty(stmt.Args), Estimate: []telemetrystoretypes.EstimateEntry{}}} + } + + results[name] = ps + + for j := range ps.Statements { + previewTasks = append(previewTasks, qbtypes.PreviewTask{Name: name, StmtIdx: j, Query: ps.Statements[j].Query, Args: ps.Statements[j].Args}) + } + } + + q.runPreviewTasks(ctx, previewTasks, results) + + return &qbtypes.QueryRangePreviewResponse{ + CompositeQuery: results, + }, nil +} + +// missingMetricNames returns the distinct metric names referenced by a metric +// builder query, or nil for a non-metric query. +func missingMetricNames(env qbtypes.QueryEnvelope) []string { + spec, ok := env.Spec.(qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]) + if !ok { + return nil + } + names := make([]string, 0, len(spec.Aggregations)) + for _, agg := range spec.Aggregations { + if agg.MetricName != "" && !slices.Contains(names, agg.MetricName) { + names = append(names, agg.MetricName) + } + } + return names +} + +func (q *querier) buildPreviewProviders( + req *qbtypes.QueryRangeRequest, + dependencyQueries map[string]bool, + missingMetricQuerySet map[string]bool, + skip map[string]bool, +) (providers map[string]qbtypes.Query, errs map[string]error) { + providers = make(map[string]qbtypes.Query) + errs = make(map[string]error) + + event := &qbtypes.QBEvent{} // preview emits no analytics + + for _, query := range req.CompositeQuery.Queries { + name := query.GetQueryName() + if skip[name] { + continue + } + + sub := *req // shallow copy: only CompositeQuery and RequestType are swapped + + // deps is the set buildQueries skips: empty for a standalone query, the + // operator's referenced siblings for a trace operator. + var deps map[string]bool + + switch { + case query.GetType() == qbtypes.QueryTypeTraceOperator: + refs, rErr := q.traceOperatorPreviewComposite(req, query) + if rErr != nil { + errs[name] = rErr + continue + } + sub.CompositeQuery = qbtypes.CompositeQuery{Queries: refs} + deps = dependencyQueries + case dependencyQueries[name]: + sub.RequestType = qbtypes.RequestTypeRaw + sub.CompositeQuery = qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{query}} + default: + sub.CompositeQuery = qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{query}} + } + + built, _, bErr := q.buildQueries(&sub, deps, missingMetricQuerySet, event) + if bErr != nil { + errs[name] = bErr + continue + } + + if provider, ok := built[name]; ok { + providers[name] = provider + } + } + return providers, errs +} + +// rendersStandaloneStatement reports whether a query type renders its own +// statement. Formula/join/sub-query don't — they reference other queries. +func rendersStandaloneStatement(t qbtypes.QueryType) bool { + switch t { + case qbtypes.QueryTypeBuilder, + qbtypes.QueryTypePromQL, + qbtypes.QueryTypeClickHouseSQL, + qbtypes.QueryTypeTraceOperator: + return true + default: + return false + } +} + +func (q *querier) traceOperatorPreviewComposite(req *qbtypes.QueryRangeRequest, operator qbtypes.QueryEnvelope) ([]qbtypes.QueryEnvelope, error) { + spec, ok := operator.Spec.(qbtypes.QueryBuilderTraceOperator) + if !ok { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid trace operator query spec %T", operator.Spec) + } + if err := spec.ParseExpression(); err != nil { + return nil, err + } + + referenced := make(map[string]bool) + for _, name := range spec.CollectReferencedQueries(spec.ParsedExpression) { + referenced[name] = true + } + + queries := make([]qbtypes.QueryEnvelope, 0, len(referenced)+1) + for _, qe := range req.CompositeQuery.Queries { + if referenced[qe.GetQueryName()] { + queries = append(queries, qe) + } + } + return append(queries, operator), nil +} + +func (q *querier) runPreviewTasks(ctx context.Context, tasks []qbtypes.PreviewTask, previews map[string]qbtypes.QueryPreview) { + if len(tasks) == 0 { + return + } + + type outcome struct { + granules *telemetrystoretypes.Granules + estimate []telemetrystoretypes.EstimateEntry + warnings []string + } + outcomes := make([]outcome, len(tasks)) + + var wg sync.WaitGroup + for i := range tasks { + wg.Add(1) + go func(i int) { + defer wg.Done() + t := tasks[i] + var out outcome + if granules, ok, scErr := q.telemetryStore.Indexes(ctx, t.Query, t.Args...); scErr != nil { + out.warnings = append(out.warnings, "could not compute granule stats: "+scErr.Error()) + } else if ok { + out.granules = &granules + } + if estimate, eErr := q.telemetryStore.Estimate(ctx, t.Query, t.Args...); eErr != nil { + out.warnings = append(out.warnings, "could not run EXPLAIN ESTIMATE: "+eErr.Error()) + } else { + out.estimate = estimate + } + outcomes[i] = out + }(i) + } + wg.Wait() + + for i := range tasks { + ps := previews[tasks[i].Name] + if idx := tasks[i].StmtIdx; idx >= 0 && idx < len(ps.Statements) { + if outcomes[i].granules != nil { + ps.Statements[idx].Granules = outcomes[i].granules + } + if len(outcomes[i].estimate) > 0 { + ps.Statements[idx].Estimate = outcomes[i].estimate + } + } + ps.Warnings = append(ps.Warnings, outcomes[i].warnings...) + previews[tasks[i].Name] = ps + } +} + +// orEmpty returns s, or a non-nil empty slice when s is nil. +func orEmpty[T any](s []T) []T { + if s == nil { + return []T{} + } + return s +} diff --git a/pkg/querier/promql_query.go b/pkg/querier/promql_query.go index 22ce69bccd..4fc9bc51bd 100644 --- a/pkg/querier/promql_query.go +++ b/pkg/querier/promql_query.go @@ -101,6 +101,7 @@ type promqlQuery struct { } var _ qbv5.Query = (*promqlQuery)(nil) +var _ qbv5.StatementProvider = (*promqlQuery)(nil) func newPromqlQuery( logger *slog.Logger, @@ -220,6 +221,62 @@ func (q *promqlQuery) renderVars(query string, vars map[string]qbv5.VariableItem return newQuery.String(), nil } +// Statement renders the PromQL string (no SQL args) without executing it, for +// the preview path. +func (q *promqlQuery) Statement(_ context.Context) (*qbv5.Statement, error) { + rendered, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To) + if err != nil { + return nil, err + } + return &qbv5.Statement{Query: rendered}, nil +} + +// PreviewStatements returns the ClickHouse statement(s) this PromQL query would +// run, captured by driving the engine with a Storage that records each selector's +// SQL and returns no data. Returns nil if capture is unsupported. +func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.CapturedStatement, error) { + storer, ok := q.promEngine.(prometheus.StatementCapturer) + if !ok { + return nil, nil + } + + rendered, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To) + if err != nil { + return nil, err + } + + start := int64(querybuilder.ToNanoSecs(q.tr.From)) + end := int64(querybuilder.ToNanoSecs(q.tr.To)) + + capStorage, recorder := storer.CapturingStorage() + qry, err := q.promEngine.Engine().NewRangeQuery( + ctx, + capStorage, + nil, + rendered, + time.Unix(0, start), + time.Unix(0, end), + q.query.Step.Duration, + ) + if err != nil { + if e := tryEnhancePromQLExecError(err); e != nil { + return nil, e + } + return nil, enhancePromQLError(rendered, err) + } + defer qry.Close() + + // Exec drives a Select per selector (recording SQL) but reads no data. + if res := qry.Exec(ctx); res.Err != nil { + if e := tryEnhancePromQLExecError(res.Err); e != nil { + return nil, e + } + return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err) + } + + return recorder.Statements(), nil +} + func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 4c594bde70..fe73a60182 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -413,7 +413,7 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID, // Classify each missing metric: never-seen -> warning with empty result; // seen-but-no-data-in-window -> dormant warning. - lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, externalMissingMetrics...) + lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, orgID, externalMissingMetrics...) var nonExistentMetrics []string var dormantMetrics []string for _, name := range externalMissingMetrics { diff --git a/pkg/querier/trace_operator_query.go b/pkg/querier/trace_operator_query.go index f91a127a00..2b1c89424b 100644 --- a/pkg/querier/trace_operator_query.go +++ b/pkg/querier/trace_operator_query.go @@ -23,6 +23,7 @@ type traceOperatorQuery struct { } var _ qbtypes.Query = (*traceOperatorQuery)(nil) +var _ qbtypes.StatementProvider = (*traceOperatorQuery)(nil) func (q *traceOperatorQuery) Fingerprint() string { return "" @@ -32,6 +33,11 @@ func (q *traceOperatorQuery) Window() (uint64, uint64) { return q.fromMS, q.toMS } +// Statement renders the SQL without executing it, for the preview path. +func (q *traceOperatorQuery) Statement(ctx context.Context) (*qbtypes.Statement, error) { + return q.stmtBuilder.Build(ctx, q.fromMS, q.toMS, q.kind, q.spec, q.compositeQuery) +} + func (q *traceOperatorQuery) Execute(ctx context.Context) (*qbtypes.Result, error) { stmt, err := q.stmtBuilder.Build( ctx, diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index a6d5e881fe..aceed4323f 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -17,11 +17,13 @@ import ( "github.com/uptrace/bun" + "github.com/SigNoz/signoz/pkg/flagger" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/sqlstore" "github.com/SigNoz/signoz/pkg/telemetrystore" "github.com/SigNoz/signoz/pkg/types" "github.com/SigNoz/signoz/pkg/types/ctxtypes" + "github.com/SigNoz/signoz/pkg/types/featuretypes" "github.com/SigNoz/signoz/pkg/types/instrumentationtypes" "github.com/SigNoz/signoz/pkg/types/retentiontypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" @@ -97,6 +99,8 @@ const ( signozTSLocalTableNameV41Week = "time_series_v4_1week" signozTSTableNameV41Week = "distributed_time_series_v4_1week" + signozTSTableNameV4Reduced = "distributed_time_series_v4_reduced" + signozTableAttributesMetadata = "distributed_attributes_metadata" signozLocalTableAttributesMetadata = "attributes_metadata" @@ -162,6 +166,8 @@ type ClickHouseReader struct { cache cache.Cache metadataDB string metadataTable string + + fl flagger.Flagger } // NewTraceReader returns a TraceReader for the database @@ -172,6 +178,7 @@ func NewReader( prometheus prometheus.Prometheus, cluster string, cache cache.Cache, + fl flagger.Flagger, options *Options, ) *ClickHouseReader { if options == nil { @@ -221,6 +228,7 @@ func NewReader( cache: cache, metadataDB: options.primary.MetadataDB, metadataTable: options.primary.MetadataTable, + fl: fl, } } @@ -2953,23 +2961,33 @@ func (r *ClickHouseReader) GetMetricAggregateAttributes(ctx context.Context, org instrumentationtypes.CodeFunctionName: "GetMetricAggregateAttributes", }) var response v3.AggregateAttributeResponse - normalized := true - if constants.IsDotMetricsEnabled { - normalized = false - } - // Query all relevant metric names from time_series_v4, but leave metadata retrieval to cache/db - query := fmt.Sprintf( - `SELECT DISTINCT metric_name - FROM %s.%s - WHERE metric_name ILIKE $1 AND __normalized = $2`, - signozMetricDBName, signozTSTableNameV41Day) + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + + // Query all relevant metric names from time_series_v4, but leave metadata retrieval to cache/db. + var query string + if reductionEnabled { + query = fmt.Sprintf( + `SELECT DISTINCT metric_name FROM ( + SELECT metric_name FROM %s.%s WHERE metric_name ILIKE $1 + UNION ALL + SELECT metric_name FROM %s.%s WHERE metric_name ILIKE $1 + )`, + signozMetricDBName, signozTSTableNameV41Day, + signozMetricDBName, signozTSTableNameV4Reduced) + } else { + query = fmt.Sprintf( + `SELECT DISTINCT metric_name + FROM %s.%s + WHERE metric_name ILIKE $1`, + signozMetricDBName, signozTSTableNameV41Day) + } if req.Limit != 0 { query = query + fmt.Sprintf(" LIMIT %d;", req.Limit) } - rows, err := r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText), normalized) + rows, err := r.db.Query(ctx, query, fmt.Sprintf("%%%s%%", req.SearchText)) if err != nil { r.logger.Error("Error while querying metric names", errorsV2.Attr(err)) return nil, fmt.Errorf("error while executing metric name query: %s", err.Error()) @@ -3082,7 +3100,7 @@ func (r *ClickHouseReader) GetMeterAggregateAttributes(ctx context.Context, orgI return &response, nil } -func (r *ClickHouseReader) GetMetricAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (r *ClickHouseReader) GetMetricAttributeKeys(ctx context.Context, orgID valuer.UUID, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(), instrumentationtypes.CodeNamespace: "clickhouse-reader", @@ -3093,17 +3111,18 @@ func (r *ClickHouseReader) GetMetricAttributeKeys(ctx context.Context, req *v3.F var rows driver.Rows var response v3.FilterAttributeKeyResponse - normalized := true - if constants.IsDotMetricsEnabled { - normalized = false - } + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) // skips the internal attributes i.e attributes starting with __ - query = fmt.Sprintf("SELECT arrayJoin(tagKeys) AS distinctTagKey FROM (SELECT JSONExtractKeys(labels) AS tagKeys FROM %s.%s WHERE metric_name=$1 AND unix_milli >= $2 AND __normalized = $3 GROUP BY tagKeys) WHERE distinctTagKey ILIKE $4 AND distinctTagKey NOT LIKE '\\_\\_%%' GROUP BY distinctTagKey", signozMetricDBName, signozTSTableNameV41Day) + if reductionEnabled { + query = fmt.Sprintf("SELECT arrayJoin(tagKeys) AS distinctTagKey FROM (SELECT JSONExtractKeys(labels) AS tagKeys FROM %s.%s WHERE metric_name=$1 AND unix_milli >= $2 GROUP BY tagKeys UNION ALL SELECT JSONExtractKeys(labels) AS tagKeys FROM %s.%s WHERE metric_name=$1 AND unix_milli >= $2 GROUP BY tagKeys) WHERE distinctTagKey ILIKE $3 AND distinctTagKey NOT LIKE '\\_\\_%%' GROUP BY distinctTagKey", signozMetricDBName, signozTSTableNameV41Day, signozMetricDBName, signozTSTableNameV4Reduced) + } else { + query = fmt.Sprintf("SELECT arrayJoin(tagKeys) AS distinctTagKey FROM (SELECT JSONExtractKeys(labels) AS tagKeys FROM %s.%s WHERE metric_name=$1 AND unix_milli >= $2 GROUP BY tagKeys) WHERE distinctTagKey ILIKE $3 AND distinctTagKey NOT LIKE '\\_\\_%%' GROUP BY distinctTagKey", signozMetricDBName, signozTSTableNameV41Day) + } if req.Limit != 0 { query = query + fmt.Sprintf(" LIMIT %d;", req.Limit) } - rows, err = r.db.Query(ctx, query, req.AggregateAttribute, common.PastDayRoundOff(), normalized, fmt.Sprintf("%%%s%%", req.SearchText)) + rows, err = r.db.Query(ctx, query, req.AggregateAttribute, common.PastDayRoundOff(), fmt.Sprintf("%%%s%%", req.SearchText)) if err != nil { r.logger.Error("Error while executing query", errorsV2.Attr(err)) return nil, fmt.Errorf("error while executing query: %s", err.Error()) @@ -3167,7 +3186,7 @@ func (r *ClickHouseReader) GetMeterAttributeKeys(ctx context.Context, req *v3.Fi return &response, nil } -func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID valuer.UUID, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(), @@ -3184,14 +3203,20 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, req *v3 normalized = false } - query = fmt.Sprintf("SELECT JSONExtractString(labels, $1) AS tagValue FROM %s.%s WHERE metric_name IN $2 AND JSONExtractString(labels, $3) ILIKE $4 AND unix_milli >= $5 AND __normalized=$6 GROUP BY tagValue", signozMetricDBName, signozTSTableNameV41Day) + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + + if reductionEnabled { + query = fmt.Sprintf("SELECT tagValue FROM (SELECT JSONExtractString(labels, $1) AS tagValue FROM %s.%s WHERE metric_name IN $2 AND JSONExtractString(labels, $3) ILIKE $4 AND unix_milli >= $5 UNION ALL SELECT JSONExtractString(labels, $1) AS tagValue FROM %s.%s WHERE metric_name IN $2 AND JSONExtractString(labels, $3) ILIKE $4 AND unix_milli >= $5) GROUP BY tagValue", signozMetricDBName, signozTSTableNameV41Day, signozMetricDBName, signozTSTableNameV4Reduced) + } else { + query = fmt.Sprintf("SELECT JSONExtractString(labels, $1) AS tagValue FROM %s.%s WHERE metric_name IN $2 AND JSONExtractString(labels, $3) ILIKE $4 AND unix_milli >= $5 GROUP BY tagValue", signozMetricDBName, signozTSTableNameV41Day) + } if req.Limit != 0 { query = query + fmt.Sprintf(" LIMIT %d;", req.Limit) } names := []string{req.AggregateAttribute} names = append(names, metrics.GetTransitionedMetric(req.AggregateAttribute, normalized)) - rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, names, req.FilterAttributeKey, fmt.Sprintf("%%%s%%", req.SearchText), common.PastDayRoundOff(), normalized) + rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, names, req.FilterAttributeKey, fmt.Sprintf("%%%s%%", req.SearchText), common.PastDayRoundOff()) if err != nil { r.logger.Error("Error while executing query", errorsV2.Attr(err)) @@ -3255,7 +3280,31 @@ func (r *ClickHouseReader) GetMetricMetadata(ctx context.Context, orgID valuer.U // 2. Only for Histograms, get `le` buckets var leFloat64 []float64 if metricType == string(v3.MetricTypeHistogram) { - query := fmt.Sprintf(` + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + + var query string + if reductionEnabled { + // `le` is a protected label that survives reduction + query = fmt.Sprintf(` + SELECT le FROM ( + SELECT JSONExtractString(labels, 'le') AS le + FROM %s.%s + WHERE metric_name = $1 + AND unix_milli >= $2 + AND type = 'Histogram' + AND (JSONExtractString(labels, 'service_name') = $3 OR JSONExtractString(labels, 'service.name') = $4) + UNION ALL + SELECT JSONExtractString(labels, 'le') AS le + FROM %s.%s + WHERE metric_name = $1 + AND unix_milli >= $2 + AND type = 'Histogram' + AND (JSONExtractString(labels, 'service_name') = $3 OR JSONExtractString(labels, 'service.name') = $4) + ) + GROUP BY le + ORDER BY le`, signozMetricDBName, signozTSTableNameV41Day, signozMetricDBName, signozTSTableNameV4Reduced) + } else { + query = fmt.Sprintf(` SELECT JSONExtractString(labels, 'le') AS le FROM %s.%s WHERE metric_name = $1 @@ -3264,6 +3313,7 @@ func (r *ClickHouseReader) GetMetricMetadata(ctx context.Context, orgID valuer.U AND (JSONExtractString(labels, 'service_name') = $3 OR JSONExtractString(labels, 'service.name') = $4) GROUP BY le ORDER BY le`, signozMetricDBName, signozTSTableNameV41Day) + } rows, err := r.db.Query(ctx, query, metricName, unixMilli, serviceName, serviceName) if err != nil { @@ -3352,7 +3402,7 @@ func (r *ClickHouseReader) GetActiveHostsFromMetricMetadata(ctx context.Context, } func (r *ClickHouseReader) GetLatestReceivedMetric( - ctx context.Context, metricNames []string, labelValues map[string]string, + ctx context.Context, orgID valuer.UUID, metricNames []string, labelValues map[string]string, ) (*model.MetricStatus, *model.ApiError) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(), @@ -3375,13 +3425,11 @@ func (r *ClickHouseReader) GetLatestReceivedMetric( fmt.Sprintf(`metric_name in (%s)`, commaSeparatedMetricNames), } - if labelValues != nil { - for label, val := range labelValues { - whereClauseParts = append( - whereClauseParts, - fmt.Sprintf(`JSONExtractString(labels, '%s') = '%s'`, label, val), - ) - } + for label, val := range labelValues { + whereClauseParts = append( + whereClauseParts, + fmt.Sprintf(`JSONExtractString(labels, '%s') = '%s'`, label, val), + ) } if len(whereClauseParts) < 1 { @@ -3390,14 +3438,32 @@ func (r *ClickHouseReader) GetLatestReceivedMetric( whereClause := strings.Join(whereClauseParts, " AND ") - query := fmt.Sprintf(` + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + + var query string + if reductionEnabled { + query = fmt.Sprintf(` + SELECT metric_name, anyLast(labels), max(unix_milli) + from ( + SELECT metric_name, labels, unix_milli FROM %s.%s WHERE %s + UNION ALL + SELECT metric_name, labels, unix_milli FROM %s.%s WHERE %s + ) + group by metric_name + limit 1 + `, signozMetricDBName, signozTSTableNameV4, whereClause, + signozMetricDBName, signozTSTableNameV4Reduced, whereClause, + ) + } else { + query = fmt.Sprintf(` SELECT metric_name, anyLast(labels), max(unix_milli) from %s.%s where %s group by metric_name limit 1 `, signozMetricDBName, signozTSTableNameV4, whereClause, - ) + ) + } rows, err := r.db.Query(ctx, query) if err != nil { @@ -5052,7 +5118,7 @@ func (r *ClickHouseReader) UpdateMetricsMetadata(ctx context.Context, orgID valu }) if req.MetricType == v3.MetricTypeHistogram { labels := []string{"le"} - hasLabels, apiError := r.CheckForLabelsInMetric(ctx, req.MetricName, labels) + hasLabels, apiError := r.CheckForLabelsInMetric(ctx, orgID, req.MetricName, labels) if apiError != nil { return apiError } @@ -5066,7 +5132,7 @@ func (r *ClickHouseReader) UpdateMetricsMetadata(ctx context.Context, orgID valu if req.MetricType == v3.MetricTypeSummary { labels := []string{"quantile"} - hasLabels, apiError := r.CheckForLabelsInMetric(ctx, req.MetricName, labels) + hasLabels, apiError := r.CheckForLabelsInMetric(ctx, orgID, req.MetricName, labels) if apiError != nil { return apiError } @@ -5093,7 +5159,7 @@ VALUES ( ?, ?, ?, ?, ?, ?, ?);`, signozMetricDBName, signozUpdatedMetricsMetadat return nil } -func (r *ClickHouseReader) CheckForLabelsInMetric(ctx context.Context, metricName string, labels []string) (bool, *model.ApiError) { +func (r *ClickHouseReader) CheckForLabelsInMetric(ctx context.Context, orgID valuer.UUID, metricName string, labels []string) (bool, *model.ApiError) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(), instrumentationtypes.CodeNamespace: "clickhouse-reader", @@ -5108,16 +5174,39 @@ func (r *ClickHouseReader) CheckForLabelsInMetric(ctx context.Context, metricNam conditions += " AND JSONHas(labels, ?) = 1" } - query := fmt.Sprintf(` + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + + var query string + var args []interface{} + if reductionEnabled { + query = fmt.Sprintf(` + SELECT count(*) > 0 as has_le + FROM ( + SELECT 1 FROM %s.%s WHERE %s + UNION ALL + SELECT 1 FROM %s.%s WHERE %s + ) + LIMIT 1`, signozMetricDBName, signozTSTableNameV41Day, conditions, signozMetricDBName, signozTSTableNameV4Reduced, conditions) + + args = make([]interface{}, 0, (len(labels)+1)*2) + for i := 0; i < 2; i++ { + args = append(args, metricName) + for _, label := range labels { + args = append(args, label) + } + } + } else { + query = fmt.Sprintf(` SELECT count(*) > 0 as has_le FROM %s.%s WHERE %s LIMIT 1`, signozMetricDBName, signozTSTableNameV41Day, conditions) - args := make([]interface{}, 0, len(labels)+1) - args = append(args, metricName) - for _, label := range labels { - args = append(args, label) + args = make([]interface{}, 0, len(labels)+1) + args = append(args, metricName) + for _, label := range labels { + args = append(args, label) + } } var hasLE bool @@ -5211,9 +5300,20 @@ func (r *ClickHouseReader) GetUpdatedMetricsMetadata(ctx context.Context, orgID // 3. Fallback: Try time_series_v4_1week table if len(stillMissing) > 0 { metricList := "'" + strings.Join(stillMissing, "', '") + "'" - query := fmt.Sprintf(`SELECT DISTINCT metric_name, type, description, temporality, is_monotonic, unit + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + var query string + if reductionEnabled { + query = fmt.Sprintf(`SELECT DISTINCT metric_name, type, description, temporality, is_monotonic, unit + FROM ( + SELECT metric_name, type, description, temporality, is_monotonic, unit FROM %s.%s WHERE metric_name IN (%s) + UNION ALL + SELECT metric_name, type, description, temporality, is_monotonic, unit FROM %s.%s WHERE metric_name IN (%s) + )`, signozMetricDBName, signozTSTableNameV4, metricList, signozMetricDBName, signozTSTableNameV4Reduced, metricList) + } else { + query = fmt.Sprintf(`SELECT DISTINCT metric_name, type, description, temporality, is_monotonic, unit FROM %s.%s WHERE metric_name IN (%s)`, signozMetricDBName, signozTSTableNameV4, metricList) + } valueCtx := context.WithValue(ctx, "clickhouse_max_threads", constants.MetricsExplorerClickhouseThreads) rows, err := r.db.Query(valueCtx, query) if err != nil { @@ -5392,13 +5492,30 @@ func (r *ClickHouseReader) GetNormalizedStatus( placeholders := "'" + strings.Join(uncached, "', '") + "'" - q := fmt.Sprintf( - `SELECT metric_name, toUInt8(__normalized) + reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + + var q string + if reductionEnabled { + q = fmt.Sprintf( + `SELECT metric_name, toUInt8(__normalized) + FROM ( + SELECT metric_name, __normalized FROM %s.%s WHERE metric_name IN (%s) + UNION ALL + SELECT metric_name, __normalized FROM %s.%s WHERE metric_name IN (%s) + ) + GROUP BY metric_name, __normalized`, + signozMetricDBName, signozTSTableNameV41Day, placeholders, + signozMetricDBName, signozTSTableNameV4Reduced, placeholders, + ) + } else { + q = fmt.Sprintf( + `SELECT metric_name, toUInt8(__normalized) FROM %s.%s WHERE metric_name IN (%s) GROUP BY metric_name, __normalized`, - signozMetricDBName, signozTSTableNameV41Day, placeholders, - ) + signozMetricDBName, signozTSTableNameV41Day, placeholders, + ) + } rows, err := r.db.Query(ctx, q) if err != nil { diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 6a6a3355b7..c6b94fef18 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -3045,7 +3045,7 @@ func (aH *APIHandler) calculateConnectionStatus( } statusForLastReceivedMetric, apiErr := aH.reader.GetLatestReceivedMetric( - ctx, connectionTests.Metrics, nil, + ctx, orgID, connectionTests.Metrics, nil, ) resultLock.Lock() @@ -3514,9 +3514,16 @@ func (aH *APIHandler) autoCompleteAttributeKeys(w http.ResponseWriter, r *http.R return } + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + aH.HandleError(w, err, http.StatusInternalServerError) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + switch req.DataSource { case v3.DataSourceMetrics: - response, err = aH.reader.GetMetricAttributeKeys(r.Context(), req) + response, err = aH.reader.GetMetricAttributeKeys(r.Context(), orgID, req) case v3.DataSourceMeter: response, err = aH.reader.GetMeterAttributeKeys(r.Context(), req) case v3.DataSourceLogs: @@ -3545,9 +3552,16 @@ func (aH *APIHandler) autoCompleteAttributeValues(w http.ResponseWriter, r *http return } + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + aH.HandleError(w, err, http.StatusInternalServerError) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + switch req.DataSource { case v3.DataSourceMetrics: - response, err = aH.reader.GetMetricAttributeValues(r.Context(), req) + response, err = aH.reader.GetMetricAttributeValues(r.Context(), orgID, req) case v3.DataSourceLogs: response, err = aH.reader.GetLogAttributeValues(r.Context(), req) case v3.DataSourceTraces: @@ -3574,9 +3588,16 @@ func (aH *APIHandler) autoCompleteAttributeValuesPost(w http.ResponseWriter, r * return } + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + aH.HandleError(w, err, http.StatusInternalServerError) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + switch req.DataSource { case v3.DataSourceMetrics: - response, err = aH.reader.GetMetricAttributeValues(r.Context(), req) + response, err = aH.reader.GetMetricAttributeValues(r.Context(), orgID, req) case v3.DataSourceLogs: response, err = aH.reader.GetLogAttributeValues(r.Context(), req) case v3.DataSourceTraces: diff --git a/pkg/query-service/app/infra.go b/pkg/query-service/app/infra.go index 671519122c..cdb0490ef4 100644 --- a/pkg/query-service/app/infra.go +++ b/pkg/query-service/app/infra.go @@ -20,7 +20,14 @@ func (aH *APIHandler) getHostAttributeKeys(w http.ResponseWriter, r *http.Reques } // get attribute keys - keys, err := aH.hostsRepo.GetHostAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.hostsRepo.GetHostAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -41,7 +48,14 @@ func (aH *APIHandler) getHostAttributeValues(w http.ResponseWriter, r *http.Requ } // get attribute values - values, err := aH.hostsRepo.GetHostAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.hostsRepo.GetHostAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -91,7 +105,14 @@ func (aH *APIHandler) getProcessAttributeKeys(w http.ResponseWriter, r *http.Req return } - keys, err := aH.processesRepo.GetProcessAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.processesRepo.GetProcessAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -108,7 +129,14 @@ func (aH *APIHandler) getProcessAttributeValues(w http.ResponseWriter, r *http.R return } - values, err := aH.processesRepo.GetProcessAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.processesRepo.GetProcessAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -154,7 +182,14 @@ func (aH *APIHandler) getPodAttributeKeys(w http.ResponseWriter, r *http.Request return } - keys, err := aH.podsRepo.GetPodAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.podsRepo.GetPodAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -171,7 +206,14 @@ func (aH *APIHandler) getPodAttributeValues(w http.ResponseWriter, r *http.Reque return } - values, err := aH.podsRepo.GetPodAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.podsRepo.GetPodAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -217,7 +259,14 @@ func (aH *APIHandler) getNodeAttributeKeys(w http.ResponseWriter, r *http.Reques return } - keys, err := aH.nodesRepo.GetNodeAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.nodesRepo.GetNodeAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -234,7 +283,14 @@ func (aH *APIHandler) getNodeAttributeValues(w http.ResponseWriter, r *http.Requ return } - values, err := aH.nodesRepo.GetNodeAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.nodesRepo.GetNodeAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -297,7 +353,14 @@ func (aH *APIHandler) getNamespaceAttributeValues(w http.ResponseWriter, r *http return } - values, err := aH.namespacesRepo.GetNamespaceAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.namespacesRepo.GetNamespaceAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -343,7 +406,14 @@ func (aH *APIHandler) getClusterAttributeKeys(w http.ResponseWriter, r *http.Req return } - keys, err := aH.clustersRepo.GetClusterAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.clustersRepo.GetClusterAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -360,7 +430,14 @@ func (aH *APIHandler) getClusterAttributeValues(w http.ResponseWriter, r *http.R return } - values, err := aH.clustersRepo.GetClusterAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.clustersRepo.GetClusterAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -406,7 +483,14 @@ func (aH *APIHandler) getDeploymentAttributeKeys(w http.ResponseWriter, r *http. return } - keys, err := aH.deploymentsRepo.GetDeploymentAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.deploymentsRepo.GetDeploymentAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -423,7 +507,14 @@ func (aH *APIHandler) getDeploymentAttributeValues(w http.ResponseWriter, r *htt return } - values, err := aH.deploymentsRepo.GetDeploymentAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.deploymentsRepo.GetDeploymentAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -469,7 +560,14 @@ func (aH *APIHandler) getDaemonSetAttributeKeys(w http.ResponseWriter, r *http.R return } - keys, err := aH.daemonsetsRepo.GetDaemonSetAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.daemonsetsRepo.GetDaemonSetAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -486,7 +584,14 @@ func (aH *APIHandler) getDaemonSetAttributeValues(w http.ResponseWriter, r *http return } - values, err := aH.daemonsetsRepo.GetDaemonSetAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.daemonsetsRepo.GetDaemonSetAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -532,7 +637,14 @@ func (aH *APIHandler) getStatefulSetAttributeKeys(w http.ResponseWriter, r *http return } - keys, err := aH.statefulsetsRepo.GetStatefulSetAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.statefulsetsRepo.GetStatefulSetAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -549,7 +661,14 @@ func (aH *APIHandler) getStatefulSetAttributeValues(w http.ResponseWriter, r *ht return } - values, err := aH.statefulsetsRepo.GetStatefulSetAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.statefulsetsRepo.GetStatefulSetAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -595,7 +714,14 @@ func (aH *APIHandler) getJobAttributeKeys(w http.ResponseWriter, r *http.Request return } - keys, err := aH.jobsRepo.GetJobAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.jobsRepo.GetJobAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -611,7 +737,14 @@ func (aH *APIHandler) getJobAttributeValues(w http.ResponseWriter, r *http.Reque return } - values, err := aH.jobsRepo.GetJobAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.jobsRepo.GetJobAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -685,7 +818,14 @@ func (aH *APIHandler) getPvcAttributeKeys(w http.ResponseWriter, r *http.Request return } - keys, err := aH.pvcsRepo.GetPvcAttributeKeys(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + keys, err := aH.pvcsRepo.GetPvcAttributeKeys(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return @@ -702,7 +842,14 @@ func (aH *APIHandler) getPvcAttributeValues(w http.ResponseWriter, r *http.Reque return } - values, err := aH.pvcsRepo.GetPvcAttributeValues(ctx, *req) + claims, err := authtypes.ClaimsFromContext(r.Context()) + if err != nil { + render.Error(w, err) + return + } + orgID := valuer.MustNewUUID(claims.OrgID) + + values, err := aH.pvcsRepo.GetPvcAttributeValues(ctx, orgID, *req) if err != nil { RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil) return diff --git a/pkg/query-service/app/inframetrics/clusters.go b/pkg/query-service/app/inframetrics/clusters.go index bb1f2e7bc7..cdf09686a5 100644 --- a/pkg/query-service/app/inframetrics/clusters.go +++ b/pkg/query-service/app/inframetrics/clusters.go @@ -43,14 +43,14 @@ func NewClustersRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *Cl return &ClustersRepo{reader: reader, querierV2: querierV2} } -func (n *ClustersRepo) GetClusterAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (n *ClustersRepo) GetClusterAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForClusters if req.Limit == 0 { req.Limit = 50 } - attributeKeysResponse, err := n.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := n.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -58,14 +58,14 @@ func (n *ClustersRepo) GetClusterAttributeKeys(ctx context.Context, req v3.Filte return attributeKeysResponse, nil } -func (n *ClustersRepo) GetClusterAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (n *ClustersRepo) GetClusterAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForClusters if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := n.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := n.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/daemonsets.go b/pkg/query-service/app/inframetrics/daemonsets.go index 8a7b75cbd2..390b1556b9 100644 --- a/pkg/query-service/app/inframetrics/daemonsets.go +++ b/pkg/query-service/app/inframetrics/daemonsets.go @@ -99,7 +99,7 @@ func NewDaemonSetsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) * return &DaemonSetsRepo{reader: reader, querierV2: querierV2} } -func (d *DaemonSetsRepo) GetDaemonSetAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (d *DaemonSetsRepo) GetDaemonSetAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any pod metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForDaemonSets @@ -107,7 +107,7 @@ func (d *DaemonSetsRepo) GetDaemonSetAttributeKeys(ctx context.Context, req v3.F req.Limit = 50 } - attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -125,14 +125,14 @@ func (d *DaemonSetsRepo) GetDaemonSetAttributeKeys(ctx context.Context, req v3.F return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (d *DaemonSetsRepo) GetDaemonSetAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (d *DaemonSetsRepo) GetDaemonSetAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForDaemonSets if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/deployments.go b/pkg/query-service/app/inframetrics/deployments.go index 139c546b47..b898a09f71 100644 --- a/pkg/query-service/app/inframetrics/deployments.go +++ b/pkg/query-service/app/inframetrics/deployments.go @@ -99,7 +99,7 @@ func NewDeploymentsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) return &DeploymentsRepo{reader: reader, querierV2: querierV2} } -func (d *DeploymentsRepo) GetDeploymentAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (d *DeploymentsRepo) GetDeploymentAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any pod metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForDeployments @@ -107,7 +107,7 @@ func (d *DeploymentsRepo) GetDeploymentAttributeKeys(ctx context.Context, req v3 req.Limit = 50 } - attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -125,14 +125,14 @@ func (d *DeploymentsRepo) GetDeploymentAttributeKeys(ctx context.Context, req v3 return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (d *DeploymentsRepo) GetDeploymentAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (d *DeploymentsRepo) GetDeploymentAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForDeployments if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/hosts.go b/pkg/query-service/app/inframetrics/hosts.go index 8bc4047d2a..70ffe091dc 100644 --- a/pkg/query-service/app/inframetrics/hosts.go +++ b/pkg/query-service/app/inframetrics/hosts.go @@ -130,7 +130,7 @@ func NewHostsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *Hosts return &HostsRepo{reader: reader, querierV2: querierV2} } -func (h *HostsRepo) GetHostAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (h *HostsRepo) GetHostAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any system metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForHostAttributes @@ -138,7 +138,7 @@ func (h *HostsRepo) GetHostAttributeKeys(ctx context.Context, req v3.FilterAttri req.Limit = 50 } - attributeKeysResponse, err := h.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := h.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -156,14 +156,14 @@ func (h *HostsRepo) GetHostAttributeKeys(ctx context.Context, req v3.FilterAttri return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (h *HostsRepo) GetHostAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (h *HostsRepo) GetHostAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForHostAttributes if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := h.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := h.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/jobs.go b/pkg/query-service/app/inframetrics/jobs.go index 8b528f420c..32a1f9245b 100644 --- a/pkg/query-service/app/inframetrics/jobs.go +++ b/pkg/query-service/app/inframetrics/jobs.go @@ -143,7 +143,7 @@ func NewJobsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *JobsRe return &JobsRepo{reader: reader, querierV2: querierV2} } -func (d *JobsRepo) GetJobAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (d *JobsRepo) GetJobAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any pod metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForJobs @@ -151,7 +151,7 @@ func (d *JobsRepo) GetJobAttributeKeys(ctx context.Context, req v3.FilterAttribu req.Limit = 50 } - attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -169,14 +169,14 @@ func (d *JobsRepo) GetJobAttributeKeys(ctx context.Context, req v3.FilterAttribu return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (d *JobsRepo) GetJobAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (d *JobsRepo) GetJobAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForJobs if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/namespaces.go b/pkg/query-service/app/inframetrics/namespaces.go index 322baedd12..cff819e2a5 100644 --- a/pkg/query-service/app/inframetrics/namespaces.go +++ b/pkg/query-service/app/inframetrics/namespaces.go @@ -53,14 +53,14 @@ func (p *NamespacesRepo) GetNamespaceAttributeKeys(ctx context.Context, req v3.F return &v3.FilterAttributeKeyResponse{AttributeKeys: attributesKeysForNamespaces}, nil } -func (p *NamespacesRepo) GetNamespaceAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (p *NamespacesRepo) GetNamespaceAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForNamespaces if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/nodes.go b/pkg/query-service/app/inframetrics/nodes.go index eab13d378e..3132b48302 100644 --- a/pkg/query-service/app/inframetrics/nodes.go +++ b/pkg/query-service/app/inframetrics/nodes.go @@ -53,14 +53,14 @@ func NewNodesRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *Nodes return &NodesRepo{reader: reader, querierV2: querierV2} } -func (n *NodesRepo) GetNodeAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (n *NodesRepo) GetNodeAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForNodes if req.Limit == 0 { req.Limit = 50 } - attributeKeysResponse, err := n.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := n.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -82,14 +82,14 @@ func (n *NodesRepo) DidSendNodeMetrics(ctx context.Context) (bool, error) { return count > 0, nil } -func (n *NodesRepo) GetNodeAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (n *NodesRepo) GetNodeAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForNodes if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := n.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := n.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/pods.go b/pkg/query-service/app/inframetrics/pods.go index 232056f578..48347a3692 100644 --- a/pkg/query-service/app/inframetrics/pods.go +++ b/pkg/query-service/app/inframetrics/pods.go @@ -71,7 +71,7 @@ func NewPodsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *PodsRe return &PodsRepo{reader: reader, querierV2: querierV2} } -func (p *PodsRepo) GetPodAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (p *PodsRepo) GetPodAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any pod metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForPods @@ -79,7 +79,7 @@ func (p *PodsRepo) GetPodAttributeKeys(ctx context.Context, req v3.FilterAttribu req.Limit = 50 } - attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -97,14 +97,14 @@ func (p *PodsRepo) GetPodAttributeKeys(ctx context.Context, req v3.FilterAttribu return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (p *PodsRepo) GetPodAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (p *PodsRepo) GetPodAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForPods if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/processes.go b/pkg/query-service/app/inframetrics/processes.go index e87f9a1f3a..3bbf136d9b 100644 --- a/pkg/query-service/app/inframetrics/processes.go +++ b/pkg/query-service/app/inframetrics/processes.go @@ -43,7 +43,7 @@ func NewProcessesRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *P return &ProcessesRepo{reader: reader, querierV2: querierV2} } -func (p *ProcessesRepo) GetProcessAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (p *ProcessesRepo) GetProcessAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any system metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = GetDotMetrics("process_memory_usage") @@ -51,7 +51,7 @@ func (p *ProcessesRepo) GetProcessAttributeKeys(ctx context.Context, req v3.Filt req.Limit = 50 } - attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -69,14 +69,14 @@ func (p *ProcessesRepo) GetProcessAttributeKeys(ctx context.Context, req v3.Filt return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (p *ProcessesRepo) GetProcessAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (p *ProcessesRepo) GetProcessAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = GetDotMetrics("process_memory_usage") if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/pvcs.go b/pkg/query-service/app/inframetrics/pvcs.go index 16ce82b24f..545d8c064a 100644 --- a/pkg/query-service/app/inframetrics/pvcs.go +++ b/pkg/query-service/app/inframetrics/pvcs.go @@ -61,14 +61,14 @@ func NewPvcsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *PvcsRe return &PvcsRepo{reader: reader, querierV2: querierV2} } -func (p *PvcsRepo) GetPvcAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (p *PvcsRepo) GetPvcAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForVolumes if req.Limit == 0 { req.Limit = 50 } - attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -86,14 +86,14 @@ func (p *PvcsRepo) GetPvcAttributeKeys(ctx context.Context, req v3.FilterAttribu return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (p *PvcsRepo) GetPvcAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (p *PvcsRepo) GetPvcAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForVolumes if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/inframetrics/statefulsets.go b/pkg/query-service/app/inframetrics/statefulsets.go index 6b6a1325e7..35c02b3c2e 100644 --- a/pkg/query-service/app/inframetrics/statefulsets.go +++ b/pkg/query-service/app/inframetrics/statefulsets.go @@ -99,7 +99,7 @@ func NewStatefulSetsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) return &StatefulSetsRepo{reader: reader, querierV2: querierV2} } -func (d *StatefulSetsRepo) GetStatefulSetAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { +func (d *StatefulSetsRepo) GetStatefulSetAttributeKeys(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) { // TODO(srikanthccv): remove hardcoded metric name and support keys from any pod metric req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForStatefulSets @@ -107,7 +107,7 @@ func (d *StatefulSetsRepo) GetStatefulSetAttributeKeys(ctx context.Context, req req.Limit = 50 } - attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, &req) + attributeKeysResponse, err := d.reader.GetMetricAttributeKeys(ctx, orgID, &req) if err != nil { return nil, err } @@ -125,14 +125,14 @@ func (d *StatefulSetsRepo) GetStatefulSetAttributeKeys(ctx context.Context, req return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil } -func (d *StatefulSetsRepo) GetStatefulSetAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { +func (d *StatefulSetsRepo) GetStatefulSetAttributeValues(ctx context.Context, orgID valuer.UUID, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) { req.DataSource = v3.DataSourceMetrics req.AggregateAttribute = metricToUseForStatefulSets if req.Limit == 0 { req.Limit = 50 } - attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, &req) + attributeValuesResponse, err := d.reader.GetMetricAttributeValues(ctx, orgID, &req) if err != nil { return nil, err } diff --git a/pkg/query-service/app/querier/querier_test.go b/pkg/query-service/app/querier/querier_test.go index 842d6f2887..f77de7c3e9 100644 --- a/pkg/query-service/app/querier/querier_test.go +++ b/pkg/query-service/app/querier/querier_test.go @@ -13,6 +13,7 @@ import ( cmock "github.com/SigNoz/clickhouse-go-mock" "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/cache/cachetest" + "github.com/SigNoz/signoz/pkg/flagger/flaggertest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/prometheus/prometheustest" @@ -1410,6 +1411,7 @@ func Test_querier_Traces_runWindowBasedListQueryDesc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) @@ -1634,6 +1636,7 @@ func Test_querier_Traces_runWindowBasedListQueryAsc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) @@ -1933,6 +1936,7 @@ func Test_querier_Logs_runWindowBasedListQueryDesc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) @@ -2159,6 +2163,7 @@ func Test_querier_Logs_runWindowBasedListQueryAsc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) diff --git a/pkg/query-service/app/querier/v2/querier_test.go b/pkg/query-service/app/querier/v2/querier_test.go index fc00a48b02..18787bee39 100644 --- a/pkg/query-service/app/querier/v2/querier_test.go +++ b/pkg/query-service/app/querier/v2/querier_test.go @@ -12,6 +12,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/SigNoz/signoz/pkg/cache" "github.com/SigNoz/signoz/pkg/cache/cachetest" + "github.com/SigNoz/signoz/pkg/flagger/flaggertest" "github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest" "github.com/SigNoz/signoz/pkg/prometheus" "github.com/SigNoz/signoz/pkg/prometheus/prometheustest" @@ -1462,6 +1463,7 @@ func Test_querier_Traces_runWindowBasedListQueryDesc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) @@ -1686,6 +1688,7 @@ func Test_querier_Traces_runWindowBasedListQueryAsc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) @@ -1984,6 +1987,7 @@ func Test_querier_Logs_runWindowBasedListQueryDesc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) @@ -2210,6 +2214,7 @@ func Test_querier_Logs_runWindowBasedListQueryAsc(t *testing.T) { prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", nil, + flaggertest.New(t), options, ) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 0a22827f43..ca420fa623 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -65,6 +65,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) { signoz.Prometheus, signoz.TelemetryStore.Cluster(), signoz.Cache, + signoz.Flagger, nil, ) diff --git a/pkg/query-service/interfaces/interface.go b/pkg/query-service/interfaces/interface.go index be82d940b2..ca91aa528d 100644 --- a/pkg/query-service/interfaces/interface.go +++ b/pkg/query-service/interfaces/interface.go @@ -50,13 +50,13 @@ type Reader interface { FetchTemporality(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string]map[v3.Temporality]bool, error) GetMetricAggregateAttributes(ctx context.Context, orgID valuer.UUID, req *v3.AggregateAttributeRequest, skipSignozMetrics bool) (*v3.AggregateAttributeResponse, error) GetMeterAggregateAttributes(ctx context.Context, orgID valuer.UUID, req *v3.AggregateAttributeRequest) (*v3.AggregateAttributeResponse, error) - GetMetricAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) + GetMetricAttributeKeys(ctx context.Context, orgID valuer.UUID, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) GetMeterAttributeKeys(ctx context.Context, req *v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) - GetMetricAttributeValues(ctx context.Context, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) + GetMetricAttributeValues(ctx context.Context, orgID valuer.UUID, req *v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) // Returns `MetricStatus` for latest received metric among `metricNames`. Useful for status calculations GetLatestReceivedMetric( - ctx context.Context, metricNames []string, labelValues map[string]string, + ctx context.Context, orgID valuer.UUID, metricNames []string, labelValues map[string]string, ) (*model.MetricStatus, *model.ApiError) // QB V3 metrics/traces/logs @@ -107,7 +107,7 @@ type Reader interface { UpdateMetricsMetadata(ctx context.Context, orgID valuer.UUID, req *model.UpdateMetricsMetadata) *model.ApiError GetUpdatedMetricsMetadata(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]*model.UpdateMetricsMetadata, *model.ApiError) - CheckForLabelsInMetric(ctx context.Context, metricName string, labels []string) (bool, *model.ApiError) + CheckForLabelsInMetric(ctx context.Context, orgID valuer.UUID, metricName string, labels []string) (bool, *model.ApiError) GetNormalizedStatus(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string]bool, error) } diff --git a/pkg/querybuilder/where_clause_visitor.go b/pkg/querybuilder/where_clause_visitor.go index ba2ca41c70..67702d3e85 100644 --- a/pkg/querybuilder/where_clause_visitor.go +++ b/pkg/querybuilder/where_clause_visitor.go @@ -145,7 +145,7 @@ func PrepareWhereClause(query string, opts FilterExprVisitorOpts) (PreparedWhere "Found %d syntax errors while parsing the search expression.", len(parserErrorListener.SyntaxErrors), ) - additionals := make([]string, len(parserErrorListener.SyntaxErrors)) + additionals := make([]string, 0, len(parserErrorListener.SyntaxErrors)) for _, err := range parserErrorListener.SyntaxErrors { if err.Error() != "" { additionals = append(additionals, err.Error()) diff --git a/pkg/signoz/module.go b/pkg/signoz/module.go index c79b0fa0d4..ade78358fc 100644 --- a/pkg/signoz/module.go +++ b/pkg/signoz/module.go @@ -150,9 +150,9 @@ func NewModules( Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, authDomainModule, tokenizer, orgGetter, authz), SpanPercentile: implspanpercentile.NewModule(querier, providerSettings), Services: implservices.NewModule(querier, telemetryStore), - MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, providerSettings, config.MetricsExplorer), + MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, fl, providerSettings, config.MetricsExplorer), MetricReductionRule: metricReductionRule, - InfraMonitoring: implinframonitoring.NewModule(telemetryStore, telemetryMetadataStore, querier, providerSettings, config.InfraMonitoring), + InfraMonitoring: implinframonitoring.NewModule(telemetryStore, telemetryMetadataStore, querier, fl, providerSettings, config.InfraMonitoring), Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore), ServiceAccount: serviceAccount, LogsPipeline: impllogspipeline.NewModule(sqlstore), diff --git a/pkg/signoz/provider.go b/pkg/signoz/provider.go index 1e72c8fc24..e94748393c 100644 --- a/pkg/signoz/provider.go +++ b/pkg/signoz/provider.go @@ -217,6 +217,7 @@ func NewSQLMigrationProviderFactories( sqlmigration.NewAddDashboardViewFactory(sqlstore, sqlschema), sqlmigration.NewMigrateSSORoleMappingNamesFactory(sqlstore), sqlmigration.NewAddMetricReductionRulesFactory(sqlstore, sqlschema), + sqlmigration.NewRemoveOrganizationTuplesFactory(sqlstore), ) } diff --git a/pkg/sqlmigration/098_remove_organization_tuples.go b/pkg/sqlmigration/098_remove_organization_tuples.go new file mode 100644 index 0000000000..eb268446b7 --- /dev/null +++ b/pkg/sqlmigration/098_remove_organization_tuples.go @@ -0,0 +1,52 @@ +package sqlmigration + +import ( + "context" + + "github.com/SigNoz/signoz/pkg/factory" + "github.com/SigNoz/signoz/pkg/sqlstore" + "github.com/uptrace/bun" + "github.com/uptrace/bun/migrate" +) + +type removeOrganizationTuples struct { + sqlstore sqlstore.SQLStore +} + +func NewRemoveOrganizationTuplesFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] { + return factory.NewProviderFactory(factory.MustNewName("remove_organization_tuples"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) { + return &removeOrganizationTuples{sqlstore: sqlstore}, nil + }) +} + +func (migration *removeOrganizationTuples) Register(migrations *migrate.Migrations) error { + return migrations.Register(migration.Up, migration.Down) +} + +func (migration *removeOrganizationTuples) Up(ctx context.Context, db *bun.DB) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + var storeID string + err = tx.QueryRowContext(ctx, `SELECT id FROM store WHERE name = ? LIMIT 1`, "signoz").Scan(&storeID) + if err != nil { + return err + } + + if _, err := tx.ExecContext(ctx, `DELETE FROM tuple WHERE store = ? AND object_type = ?`, storeID, "organization"); err != nil { + return err + } + + if _, err := tx.ExecContext(ctx, `DELETE FROM changelog WHERE store = ? AND object_type = ?`, storeID, "organization"); err != nil { + return err + } + + return tx.Commit() +} + +func (migration *removeOrganizationTuples) Down(context.Context, *bun.DB) error { + return nil +} diff --git a/pkg/telemetrymetadata/metadata.go b/pkg/telemetrymetadata/metadata.go index 1901516327..51c4e8c143 100644 --- a/pkg/telemetrymetadata/metadata.go +++ b/pkg/telemetrymetadata/metadata.go @@ -1791,7 +1791,7 @@ func (t *telemetryMetaStore) getAuditFieldValues(ctx context.Context, fieldValue } // getMetricFieldValues returns field values and whether the result is complete. -func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) { +func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(), instrumentationtypes.CodeNamespace: "metadata", @@ -1802,7 +1802,7 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValu limit = 50 } - values, err := t.getIntrinsicMetricFieldValues(ctx, fieldValueSelector, limit) + values, err := t.getIntrinsicMetricFieldValues(ctx, orgID, fieldValueSelector, limit) if err != nil { return nil, false, err } @@ -1887,7 +1887,7 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValu } // getIntrinsicMetricFieldValues returns values, isSearchComplete, error. -func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector, limit int) (*telemetrytypes.TelemetryFieldValues, error) { +func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *telemetrytypes.FieldValueSelector, limit int) (*telemetrytypes.TelemetryFieldValues, error) { ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{ instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(), instrumentationtypes.CodeNamespace: "metadata", @@ -1916,8 +1916,58 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, return &telemetrytypes.TelemetryFieldValues{}, nil } + // Reduced metrics live in time_series_v4_reduced + the buffer (is_reduced=false), not the raw + // tables; query those only when reduction is enabled (the tables may not exist otherwise). + reductionEnabled := t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) + sources := []struct { + tableName string + extraConds []string + }{ + {telemetrymetrics.TimeseriesV41weekTableName, nil}, + } + if reductionEnabled { + sources = append(sources, + struct { + tableName string + extraConds []string + }{telemetrymetrics.TimeseriesV4BufferTableName, []string{"is_reduced = false"}}, + struct { + tableName string + extraConds []string + }{telemetrymetrics.TimeseriesV4ReducedTableName, nil}, + ) + } + + values := &telemetrytypes.TelemetryFieldValues{} + seen := make(map[string]struct{}) + for _, src := range sources { + if len(values.StringValues) >= limit { + break + } + tableValues, err := t.getIntrinsicMetricFieldValuesForTable(ctx, fieldValueSelector, key, src.tableName, limit, src.extraConds...) + if err != nil { + return nil, err + } + for _, str := range tableValues { + if _, ok := seen[str]; ok { + continue + } + seen[str] = struct{}{} + values.StringValues = append(values.StringValues, str) + if len(values.StringValues) >= limit { + break + } + } + } + return values, nil +} + +// getIntrinsicMetricFieldValuesForTable returns the distinct intrinsic field values for the +// given key from a single time series table. extraConds are appended verbatim (e.g. the +// buffer's "is_reduced = false" filter). +func (t *telemetryMetaStore) getIntrinsicMetricFieldValuesForTable(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector, key telemetrytypes.TelemetryFieldKey, tableName string, limit int, extraConds ...string) ([]string, error) { sb := sqlbuilder.Select(sqlbuilder.Escape(key.Name)). - From(t.metricsDBName + "." + telemetrymetrics.TimeseriesV41weekTableName) + From(t.metricsDBName + "." + tableName) if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" { sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName)) @@ -1941,6 +1991,11 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, sb.Where(sb.ILike(key.Name, "%"+escapeForLike(fieldValueSelector.Value)+"%")) } } + + for _, cond := range extraConds { + sb.Where(cond) + } + sb.GroupBy(sqlbuilder.Escape(key.Name)) sb.Limit(limit + 1) @@ -1951,7 +2006,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, } defer rows.Close() - values := &telemetrytypes.TelemetryFieldValues{} + var values []string rowCount := 0 for rows.Next() { rowCount++ @@ -1963,7 +2018,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, if err := rows.Scan(&str); err != nil { return nil, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error()) } - values.StringValues = append(values.StringValues, str) + values = append(values, str) } return values, nil } @@ -2079,7 +2134,7 @@ func populateAllUnspecifiedValues(allUnspecifiedValues *telemetrytypes.Telemetry } // GetAllValues returns all values and whether the result is complete. -func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) { +func (t *telemetryMetaStore) GetAllValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) { values := &telemetrytypes.TelemetryFieldValues{} var complete = true var err error @@ -2102,7 +2157,7 @@ func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelecto if fieldValueSelector.Source == telemetrytypes.SourceMeter { values, complete, err = t.getMeterSourceMetricFieldValues(ctx, fieldValueSelector) } else { - values, complete, err = t.getMetricFieldValues(ctx, fieldValueSelector) + values, complete, err = t.getMetricFieldValues(ctx, orgID, fieldValueSelector) } case telemetrytypes.SignalUnspecified: mapOfValues := make(map[any]bool) @@ -2121,7 +2176,7 @@ func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelecto complete = complete && logsComplete && populateComplete } - metricsValues, metricsComplete, err := t.getMetricFieldValues(ctx, fieldValueSelector) + metricsValues, metricsComplete, err := t.getMetricFieldValues(ctx, orgID, fieldValueSelector) if err == nil { populateComplete := populateAllUnspecifiedValues(allUnspecifiedValues, mapOfValues, mapOfRelatedValues, metricsValues, limit) complete = complete && metricsComplete && populateComplete @@ -2208,45 +2263,89 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context, instrumentationtypes.CodeNamespace: "metadata", instrumentationtypes.CodeFunctionName: "fetchMetricsTemporalityAndType", }) - temporalities := make(map[string][]metrictypes.Temporality) - types := make(map[string]metrictypes.Type) - reduced := make(map[string]bool) - adjustedStartTs, adjustedEndTs, tsTableName, _ := telemetrymetrics.WhichTSTableToUse(queryTimeRangeStartTs, queryTimeRangeEndTs, false, nil) - cols := []string{"metric_name", "temporality", "any(type) AS type", "any(is_monotonic) as is_monotonic"} - - // When reduction is enabled, fold the reduced-catalog presence check into the - // same query so a metric's reduced status comes back in one round trip. - var reducedArgs []any - if reductionEnabled { - rs := sqlbuilder.NewSelectBuilder() - rs.Select("metric_name") - rs.From(t.metricsDBName + "." + telemetrymetrics.TimeseriesV4ReducedTableName) - rs.Where(rs.In("metric_name", metricNames), rs.GTE("unix_milli", adjustedStartTs), rs.LT("unix_milli", adjustedEndTs)) - rs.GroupBy("metric_name") - rsQuery, rsArgs := rs.BuildWithFlavor(sqlbuilder.ClickHouse) - cols = append(cols, fmt.Sprintf("metric_name GLOBAL IN (%s) AS reduced", rsQuery)) - reducedArgs = rsArgs + temporalities, types, err := t.fetchTemporalityTypeForTable(ctx, tsTableName, adjustedStartTs, adjustedEndTs, metricNames) + if err != nil { + return nil, nil, nil, err } + reduced := make(map[string]bool) + if reductionEnabled { + bufferTemporalities, bufferTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4BufferTableName, adjustedStartTs, adjustedEndTs, metricNames, "is_reduced = false") + if err != nil { + return nil, nil, nil, err + } + for metricName, bufferTemps := range bufferTemporalities { + temporalities[metricName] = unionTemporalities(temporalities[metricName], bufferTemps) + } + for metricName, metricType := range bufferTypes { + if _, ok := types[metricName]; !ok { + types[metricName] = metricType + } + } + + reducedTemporalities, reducedTypes, err := t.fetchTemporalityTypeForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, adjustedStartTs, adjustedEndTs, metricNames) + if err != nil { + return nil, nil, nil, err + } + for metricName := range reducedTypes { + reduced[metricName] = true + } + for metricName, reducedTemps := range reducedTemporalities { + if len(temporalities[metricName]) == 0 { + temporalities[metricName] = reducedTemps + } + } + for metricName, metricType := range reducedTypes { + if _, ok := types[metricName]; !ok { + types[metricName] = metricType + } + } + } + + return temporalities, types, reduced, nil +} + +// unionTemporalities appends the distinct temporalities from additional to existing. +func unionTemporalities(existing, additional []metrictypes.Temporality) []metrictypes.Temporality { + for _, t := range additional { + found := false + for _, e := range existing { + if e == t { + found = true + break + } + } + if !found { + existing = append(existing, t) + } + } + return existing +} + +func (t *telemetryMetaStore) fetchTemporalityTypeForTable(ctx context.Context, tableName string, adjustedStartTs, adjustedEndTs uint64, metricNames []string, extraConds ...string) (map[string][]metrictypes.Temporality, map[string]metrictypes.Type, error) { + temporalities := make(map[string][]metrictypes.Temporality) + types := make(map[string]metrictypes.Type) + sb := sqlbuilder.NewSelectBuilder() - sb.Select(cols...) - sb.From(t.metricsDBName + "." + tsTableName) - sb.Where( + sb.Select("metric_name", "temporality", "any(type) AS type", "any(is_monotonic) as is_monotonic") + sb.From(t.metricsDBName + "." + tableName) + conds := []string{ sb.In("metric_name", metricNames), sb.GTE("unix_milli", adjustedStartTs), sb.LT("unix_milli", adjustedEndTs), - ) + } + conds = append(conds, extraConds...) + sb.Where(conds...) sb.GroupBy("metric_name", "temporality") - query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, reducedArgs...) - + query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse) t.logger.DebugContext(ctx, "fetching metric temporality", slog.String("query", query), slog.Any("args", args)) rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...) if err != nil { - return nil, nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to fetch metric temporality") + return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to fetch metric temporality") } defer rows.Close() @@ -2255,13 +2354,8 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context, var temporality metrictypes.Temporality var metricType metrictypes.Type var isMonotonic bool - var isReduced uint8 - dest := []any{&metricName, &temporality, &metricType, &isMonotonic} - if reductionEnabled { - dest = append(dest, &isReduced) - } - if err := rows.Scan(dest...); err != nil { - return nil, nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to scan temporality result") + if err := rows.Scan(&metricName, &temporality, &metricType, &isMonotonic); err != nil { + return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to scan temporality result") } if temporality != metrictypes.Unknown { temporalities[metricName] = append(temporalities[metricName], temporality) @@ -2270,15 +2364,12 @@ func (t *telemetryMetaStore) fetchMetricsTemporalityAndType(ctx context.Context, metricType = metrictypes.GaugeType } types[metricName] = metricType - if isReduced != 0 { - reduced[metricName] = true - } } if err := rows.Err(); err != nil { - return nil, nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error iterating over metrics temporality rows") + return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "error iterating over metrics temporality rows") } - return temporalities, types, reduced, nil + return temporalities, types, nil } func (t *telemetryMetaStore) fetchMeterSourceMetricsTemporalityAndType(ctx context.Context, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error) { @@ -2525,12 +2616,33 @@ func (t *telemetryMetaStore) GetFirstSeenFromMetricMetadata(ctx context.Context, return result, nil } -func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, metricNames ...string) (map[string]int64, error) { +func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) { + lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4TableName, metricNames) + if err != nil { + return nil, err + } + + if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) { + reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, metricNames) + if err != nil { + return nil, err + } + for metricName, ts := range reducedLastSeen { + if ts > lastSeenInfo[metricName] { + lastSeenInfo[metricName] = ts + } + } + } + + return lastSeenInfo, nil +} + +func (t *telemetryMetaStore) fetchLastSeenInfoForTable(ctx context.Context, tableName string, metricNames []string) (map[string]int64, error) { sb := sqlbuilder.Select( "metric_name", "max(unix_milli)", ). - From(t.metricsDBName + "." + telemetrymetrics.TimeseriesV4TableName) + From(t.metricsDBName + "." + tableName) sb.Where(sb.In("metric_name", metricNames)) sb.GroupBy("metric_name") diff --git a/pkg/telemetrystore/clickhousetelemetrystore/explain.go b/pkg/telemetrystore/clickhousetelemetrystore/explain.go new file mode 100644 index 0000000000..6f96360cf0 --- /dev/null +++ b/pkg/telemetrystore/clickhousetelemetrystore/explain.go @@ -0,0 +1,218 @@ +package clickhousetelemetrystore + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" +) + +// ExplainPlanNode is a node in ClickHouse's `EXPLAIN json = 1, indexes = 1` +// output, parsed to derive the granule-skip breakdown. +type ExplainPlanNode struct { + NodeType string `json:"Node Type"` + Description string `json:"Description"` + Indexes []ExplainPlanIndex `json:"Indexes"` + Plans []ExplainPlanNode `json:"Plans"` +} + +// ExplainPlanIndex is one index entry under a ReadFromMergeTree node, reporting +// the parts/granules entering and surviving the index. +type ExplainPlanIndex struct { + Type string `json:"Type"` + Name string `json:"Name"` + Keys []string `json:"Keys"` + Condition string `json:"Condition"` + InitialParts *int64 `json:"Initial Parts"` + SelectedParts *int64 `json:"Selected Parts"` + InitialGranules *int64 `json:"Initial Granules"` + SelectedGranules *int64 `json:"Selected Granules"` +} + +// RunExplainEstimate backs TelemetryStore.Estimate. +func RunExplainEstimate(ctx context.Context, conn clickhouse.Conn, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) { + if err := ValidateExplainStatement(stmt); err != nil { + return nil, err + } + + rows, err := conn.Query(ctx, "EXPLAIN ESTIMATE "+stmt, args...) + if err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to run EXPLAIN ESTIMATE") + } + defer rows.Close() + + colTypes := rows.ColumnTypes() + var entries []telemetrystoretypes.EstimateEntry + for rows.Next() { + dest := make([]any, len(colTypes)) + for i, ct := range colTypes { + dest[i] = reflect.New(ct.ScanType()).Interface() + } + if err := rows.Scan(dest...); err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan EXPLAIN ESTIMATE row") + } + var entry telemetrystoretypes.EstimateEntry + for i, ct := range colTypes { + val := reflect.ValueOf(dest[i]).Elem().Interface() + switch strings.ToLower(ct.Name()) { + case "database": + entry.Database = fmt.Sprintf("%v", val) + case "table": + entry.Table = fmt.Sprintf("%v", val) + case "parts": + entry.Parts = toInt64(val) + case "rows": + entry.Rows = toInt64(val) + case "marks": + entry.Marks = toInt64(val) + } + } + entries = append(entries, entry) + } + if err := rows.Err(); err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "EXPLAIN ESTIMATE row iteration failed") + } + return entries, nil +} + +// RunExplainPlan backs TelemetryStore.Plan, returning the driver error when stmt +// does not parse or bind. +func RunExplainPlan(ctx context.Context, conn clickhouse.Conn, stmt string, args ...any) error { + if err := ValidateExplainStatement(stmt); err != nil { + return err + } + + rows, err := conn.Query(ctx, "EXPLAIN PLAN "+stmt, args...) + if err != nil { + return err + } + rows.Close() + return nil +} + +// RunExplainIndexes backs TelemetryStore.Indexes, summing the breakdown +// across every ReadFromMergeTree node. +func RunExplainIndexes(ctx context.Context, conn clickhouse.Conn, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) { + if err := ValidateExplainStatement(stmt); err != nil { + return telemetrystoretypes.Granules{}, false, err + } + + rows, err := conn.Query(ctx, "EXPLAIN json = 1, indexes = 1 "+stmt, args...) + if err != nil { + return telemetrystoretypes.Granules{}, false, errors.WrapInternalf(err, errors.CodeInternal, "failed to run EXPLAIN for granule stats") + } + defer rows.Close() + + // json=1 emits one JSON document; join rows in case the driver splits it. + var sb strings.Builder + for rows.Next() { + var line string + if err := rows.Scan(&line); err != nil { + return telemetrystoretypes.Granules{}, false, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan EXPLAIN json row") + } + sb.WriteString(line) + sb.WriteByte('\n') + } + if err := rows.Err(); err != nil { + return telemetrystoretypes.Granules{}, false, errors.WrapInternalf(err, errors.CodeInternal, "EXPLAIN json row iteration failed") + } + + var plans []struct { + Plan ExplainPlanNode `json:"Plan"` + } + if err := json.Unmarshal([]byte(sb.String()), &plans); err != nil { + return telemetrystoretypes.Granules{}, false, errors.WrapInternalf(err, errors.CodeInternal, "failed to parse EXPLAIN json") + } + + var totalInitial, totalSelected int64 + reads := []telemetrystoretypes.MergeTreeRead{} + for i := range plans { + collectMergeTreeReads(&plans[i].Plan, &reads, &totalInitial, &totalSelected) + } + if totalInitial <= 0 { + // No MergeTree index analysis — nothing to report. + return telemetrystoretypes.Granules{}, false, nil + } + if totalSelected < 0 { + totalSelected = 0 + } + skippedGranules := totalInitial - totalSelected + if skippedGranules < 0 { + skippedGranules = 0 + } + return telemetrystoretypes.Granules{ + Initial: totalInitial, + Selected: totalSelected, + Skipped: skippedGranules, + Reads: reads, + }, true, nil +} + +func collectMergeTreeReads(node *ExplainPlanNode, reads *[]telemetrystoretypes.MergeTreeRead, totalInitial, totalSelected *int64) { + if node.NodeType == "ReadFromMergeTree" && len(node.Indexes) > 0 { + steps := make([]telemetrystoretypes.IndexStep, 0, len(node.Indexes)) + var initial, selected *int64 + for i := range node.Indexes { + idx := node.Indexes[i] + if idx.InitialGranules != nil && initial == nil { + initial = idx.InitialGranules + } + if idx.SelectedGranules != nil { + selected = idx.SelectedGranules + } + steps = append(steps, telemetrystoretypes.IndexStep{ + Type: idx.Type, + Name: idx.Name, + Keys: orEmpty(idx.Keys), + Condition: idx.Condition, + InitialParts: derefInt64(idx.InitialParts), + SelectedParts: derefInt64(idx.SelectedParts), + InitialGranules: derefInt64(idx.InitialGranules), + SelectedGranules: derefInt64(idx.SelectedGranules), + }) + } + if initial != nil && selected != nil { + *totalInitial += *initial + *totalSelected += *selected + } + *reads = append(*reads, telemetrystoretypes.MergeTreeRead{Table: node.Description, Steps: steps}) + } + for i := range node.Plans { + collectMergeTreeReads(&node.Plans[i], reads, totalInitial, totalSelected) + } +} + +// toInt64 coerces a driver-scanned numeric value to int64 (0 if non-numeric). +func toInt64(v any) int64 { + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return rv.Int() + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return int64(rv.Uint()) + case reflect.Float32, reflect.Float64: + return int64(rv.Float()) + default: + return 0 + } +} + +func derefInt64(p *int64) int64 { + if p == nil { + return 0 + } + return *p +} + +// orEmpty returns s, or a non-nil empty slice when s is nil. +func orEmpty[T any](s []T) []T { + if s == nil { + return []T{} + } + return s +} diff --git a/pkg/telemetrystore/clickhousetelemetrystore/explain_statement.go b/pkg/telemetrystore/clickhousetelemetrystore/explain_statement.go new file mode 100644 index 0000000000..7948d63a5f --- /dev/null +++ b/pkg/telemetrystore/clickhousetelemetrystore/explain_statement.go @@ -0,0 +1,122 @@ +package clickhousetelemetrystore + +import ( + "strings" + + "github.com/SigNoz/signoz/pkg/errors" +) + +// ErrCodeUnsafeStatement is returned when a statement is not a single statement +// safe to wrap in an EXPLAIN prefix. +var ErrCodeUnsafeStatement = errors.MustNewCode("unsafe_statement") + +// scanState is the lexer state while scanning a statement. +type scanState int + +const ( + scanNormal scanState = iota + scanSingle + scanDouble + scanBacktick + scanLineComment + scanBlockComment +) + +// ValidateExplainStatement rejects stacked statements (e.g. `SELECT 1; DROP TABLE t`), +// the only injection vector left once values are bound and the EXPLAIN prefix is fixed. +// It scans stmt as ClickHouse SQL — ignoring ';' inside string literals, quoted +// identifiers, and comments — and rejects any content after a top-level ';'. +func ValidateExplainStatement(stmt string) error { + if strings.TrimSpace(stmt) == "" { + return errors.NewInvalidInputf(ErrCodeUnsafeStatement, "statement is empty") + } + + state := scanNormal + // terminated is set at a top-level ';'; after it only whitespace, ';', and + // comments may appear — anything else is a second statement. + terminated := false + + for i := 0; i < len(stmt); i++ { + c := stmt[i] + switch state { + case scanNormal: + if terminated { + switch { + case isSQLSpace(c) || c == ';': + // harmless trailing whitespace / empty statements + case c == '-' && i+1 < len(stmt) && stmt[i+1] == '-': + state = scanLineComment + i++ + case c == '/' && i+1 < len(stmt) && stmt[i+1] == '*': + state = scanBlockComment + i++ + default: + return errors.NewInvalidInputf(ErrCodeUnsafeStatement, "statement must be a single statement; content found after ';'") + } + continue + } + switch { + case c == '\'': + state = scanSingle + case c == '"': + state = scanDouble + case c == '`': + state = scanBacktick + case c == '-' && i+1 < len(stmt) && stmt[i+1] == '-': + state = scanLineComment + i++ + case c == '/' && i+1 < len(stmt) && stmt[i+1] == '*': + state = scanBlockComment + i++ + case c == ';': + terminated = true + } + case scanSingle: + i = skipQuoted(stmt, i, '\'', &state) + case scanDouble: + i = skipQuoted(stmt, i, '"', &state) + case scanBacktick: + i = skipQuoted(stmt, i, '`', &state) + case scanLineComment: + if c == '\n' { + state = scanNormal + } + case scanBlockComment: + if c == '*' && i+1 < len(stmt) && stmt[i+1] == '/' { + state = scanNormal + i++ + } + } + } + + return nil +} + +// skipQuoted advances one character within a quoted literal/identifier delimited +// by quote. A backslash or doubled quote escapes; an unescaped quote ends the +// literal (resetting *state). It returns the index to resume from (caller adds one). +func skipQuoted(s string, i int, quote byte, state *scanState) int { + c := s[i] + switch c { + case '\\': + // skip the escaped character + return i + 1 + case quote: + if i+1 < len(s) && s[i+1] == quote { + // doubled quote: stay inside the literal + return i + 1 + } + *state = scanNormal + } + return i +} + +// isSQLSpace reports whether c is SQL statement whitespace. +func isSQLSpace(c byte) bool { + switch c { + case ' ', '\t', '\n', '\r', '\v', '\f': + return true + default: + return false + } +} diff --git a/pkg/telemetrystore/clickhousetelemetrystore/explain_statement_test.go b/pkg/telemetrystore/clickhousetelemetrystore/explain_statement_test.go new file mode 100644 index 0000000000..dbf98deb7e --- /dev/null +++ b/pkg/telemetrystore/clickhousetelemetrystore/explain_statement_test.go @@ -0,0 +1,46 @@ +package clickhousetelemetrystore + +import "testing" + +func TestValidateExplainStatement(t *testing.T) { + cases := []struct { + name string + stmt string + ok bool + }{ + {"simple select", "SELECT 1", true}, + {"with cte", "WITH x AS (SELECT 1) SELECT * FROM x", true}, + {"trailing semicolon", "SELECT 1;", true}, + {"trailing semicolon and space", "SELECT 1; \n", true}, + {"double trailing semicolon", "SELECT 1;;", true}, + {"trailing line comment", "SELECT 1; -- done", true}, + {"trailing block comment", "SELECT 1; /* done */", true}, + {"semicolon inside string", "SELECT 'a; b' AS x", true}, + {"semicolon inside backtick ident", "SELECT 1 AS `a;b`", true}, + {"semicolon inside double-quoted ident", "SELECT 1 AS \"a;b\"", true}, + {"semicolon inside line comment", "SELECT 1 -- a; b", true}, + {"semicolon inside block comment", "SELECT /* a; b */ 1", true}, + {"escaped quote then semicolon in string", "SELECT 'a\\'; DROP' AS x", true}, + {"doubled quote then semicolon in string", "SELECT 'a''; DROP' AS x", true}, + + {"empty", "", false}, + {"whitespace only", " \n\t", false}, + {"stacked statement", "SELECT 1; DROP TABLE t", false}, + {"stacked statement no space", "SELECT 1;DROP TABLE t", false}, + {"stacked after string close", "SELECT 'a'; DROP TABLE t", false}, + {"stacked string statement", "SELECT 1; 'x'", false}, + {"stacked after comment", "SELECT 1; /* c */ SELECT 2", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateExplainStatement(tc.stmt) + if tc.ok && err != nil { + t.Fatalf("expected valid, got error: %v", err) + } + if !tc.ok && err == nil { + t.Fatalf("expected error, got nil") + } + }) + } +} diff --git a/pkg/telemetrystore/clickhousetelemetrystore/provider.go b/pkg/telemetrystore/clickhousetelemetrystore/provider.go index cfd51455a9..9eb2e1f0d9 100644 --- a/pkg/telemetrystore/clickhousetelemetrystore/provider.go +++ b/pkg/telemetrystore/clickhousetelemetrystore/provider.go @@ -9,6 +9,7 @@ import ( "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/telemetrystore" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" "go.opentelemetry.io/otel/metric" ) @@ -130,18 +131,31 @@ func New(ctx context.Context, providerSettings factory.ProviderSettings, config return nil, err } - return &provider{ + p := &provider{ settings: settings, clickHouseConn: chConn, cluster: config.Clickhouse.Cluster, hooks: hooks, - }, nil + } + return p, nil } func (p *provider) ClickhouseDB() clickhouse.Conn { return p } +func (p *provider) Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) { + return RunExplainEstimate(ctx, p, stmt, args...) +} + +func (p *provider) Plan(ctx context.Context, stmt string, args ...any) error { + return RunExplainPlan(ctx, p, stmt, args...) +} + +func (p *provider) Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) { + return RunExplainIndexes(ctx, p, stmt, args...) +} + func (p *provider) Cluster() string { return p.cluster } diff --git a/pkg/telemetrystore/telemetrystore.go b/pkg/telemetrystore/telemetrystore.go index b33c00b857..7ae97b70e1 100644 --- a/pkg/telemetrystore/telemetrystore.go +++ b/pkg/telemetrystore/telemetrystore.go @@ -4,14 +4,24 @@ import ( "context" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" ) type TelemetryStore interface { - // ClickhouseDB returns the clickhouse database connection. + // ClickhouseDB returns the clickhouse connection, which can also EXPLAIN. ClickhouseDB() clickhouse.Conn // Cluster returns the cluster name. Cluster() string + + // Estimate returns the per-table scan estimate from EXPLAIN ESTIMATE. + Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) + + // Plan runs EXPLAIN PLAN to check stmt parses and binds. + Plan(ctx context.Context, stmt string, args ...any) error + + // Indexes returns the granule-skip breakdown from EXPLAIN json = 1, indexes = 1. + Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) } type TelemetryStoreHook interface { diff --git a/pkg/telemetrystore/telemetrystoretest/provider.go b/pkg/telemetrystore/telemetrystoretest/provider.go index 050349edc5..db73cee285 100644 --- a/pkg/telemetrystore/telemetrystoretest/provider.go +++ b/pkg/telemetrystore/telemetrystoretest/provider.go @@ -1,10 +1,14 @@ package telemetrystoretest import ( + "context" + "github.com/ClickHouse/clickhouse-go/v2" "github.com/DATA-DOG/go-sqlmock" - "github.com/SigNoz/signoz/pkg/telemetrystore" cmock "github.com/SigNoz/clickhouse-go-mock" + "github.com/SigNoz/signoz/pkg/telemetrystore" + "github.com/SigNoz/signoz/pkg/telemetrystore/clickhousetelemetrystore" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" ) var _ telemetrystore.TelemetryStore = (*Provider)(nil) @@ -36,6 +40,21 @@ func (p *Provider) Cluster() string { return "cluster" } +// Estimate runs EXPLAIN ESTIMATE against the mock connection. +func (p *Provider) Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) { + return clickhousetelemetrystore.RunExplainEstimate(ctx, p.clickhouseDB.(clickhouse.Conn), stmt, args...) +} + +// Plan runs EXPLAIN PLAN against the mock connection. +func (p *Provider) Plan(ctx context.Context, stmt string, args ...any) error { + return clickhousetelemetrystore.RunExplainPlan(ctx, p.clickhouseDB.(clickhouse.Conn), stmt, args...) +} + +// Indexes runs EXPLAIN indexes against the mock connection. +func (p *Provider) Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) { + return clickhousetelemetrystore.RunExplainIndexes(ctx, p.clickhouseDB.(clickhouse.Conn), stmt, args...) +} + // Mock returns the underlying Clickhouse mock instance for setting expectations. func (p *Provider) Mock() cmock.ClickConnMockCommon { return p.clickhouseDB diff --git a/pkg/types/cloudintegrationtypes/account.go b/pkg/types/cloudintegrationtypes/account.go index 16630ffd06..95502f601d 100644 --- a/pkg/types/cloudintegrationtypes/account.go +++ b/pkg/types/cloudintegrationtypes/account.go @@ -31,11 +31,13 @@ type AgentReport struct { type AccountConfig struct { AWS *AWSAccountConfig `json:"aws,omitempty" required:"false" nullable:"false"` Azure *AzureAccountConfig `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *GCPAccountConfig `json:"gcp,omitempty" required:"false" nullable:"false"` } type UpdatableAccountConfig struct { AWS *UpdatableAWSAccountConfig `json:"aws,omitempty" required:"false" nullable:"false"` Azure *UpdatableAzureAccountConfig `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *UpdatableGCPAccountConfig `json:"gcp,omitempty" required:"false" nullable:"false"` } type PostableAccount struct { @@ -48,6 +50,7 @@ type PostableAccountConfig struct { AgentVersion string AWS *AWSPostableAccountConfig `json:"aws,omitempty" required:"false" nullable:"false"` Azure *AzurePostableAccountConfig `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *GCPPostableAccountConfig `json:"gcp,omitempty" required:"false" nullable:"false"` } type Credentials struct { @@ -66,6 +69,7 @@ type ConnectionArtifact struct { // required till new providers are added AWS *AWSConnectionArtifact `json:"aws,omitempty" required:"false" nullable:"false"` Azure *AzureConnectionArtifact `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *GCPConnectionArtifact `json:"gcp,omitempty" required:"false" nullable:"false"` } type GetConnectionArtifactRequest = PostableAccount @@ -211,6 +215,30 @@ func NewAccountConfigFromPostable(provider CloudProviderType, config *PostableAc } return &AccountConfig{Azure: &AzureAccountConfig{DeploymentRegion: config.Azure.DeploymentRegion, ResourceGroups: config.Azure.ResourceGroups}}, nil + case CloudProviderTypeGCP: + if config.GCP == nil { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "GCP config can not be nil for GCP provider") + } + + if config.GCP.DeploymentProjectID == "" { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "deployment project ID is required for GCP provider") + } + + if err := validateGCPRegion(config.GCP.DeploymentRegion); err != nil { + return nil, err + } + + if len(config.GCP.ProjectIDs) == 0 { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one project id is required for GCP provider") + } + + return &AccountConfig{ + GCP: &GCPAccountConfig{ + DeploymentProjectID: config.GCP.DeploymentProjectID, + ProjectIDs: config.GCP.ProjectIDs, + DeploymentRegion: config.GCP.DeploymentRegion, + }, + }, nil default: return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) } @@ -244,6 +272,30 @@ func NewAccountConfigFromUpdatable(provider CloudProviderType, config *Updatable } return &AccountConfig{Azure: &AzureAccountConfig{ResourceGroups: config.Config.Azure.ResourceGroups}}, nil + case CloudProviderTypeGCP: + if config.Config.GCP == nil { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "GCP config can not be nil for GCP provider") + } + + if err := validateGCPRegion(config.Config.GCP.DeploymentRegion); err != nil { + return nil, err + } + + if len(config.Config.GCP.ProjectIDs) == 0 { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one project id is required for GCP provider") + } + + if config.Config.GCP.DeploymentProjectID == "" { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "deployment project ID is required for GCP provider") + } + + return &AccountConfig{ + GCP: &GCPAccountConfig{ + DeploymentProjectID: config.Config.GCP.DeploymentProjectID, + ProjectIDs: config.Config.GCP.ProjectIDs, + DeploymentRegion: config.Config.GCP.DeploymentRegion, + }, + }, nil default: return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) } @@ -332,15 +384,16 @@ func (config *PostableAccountConfig) SetAgentVersion(agentVersion string) { // thats why not naming it MarshalJSON(), as it will interfere with default JSON marshalling of AccountConfig struct. // NOTE: this entertains first non-null provider's config. func (config *AccountConfig) ToJSON() ([]byte, error) { - if config.AWS != nil { + switch { + case config.AWS != nil: return json.Marshal(config.AWS) - } - - if config.Azure != nil { + case config.Azure != nil: return json.Marshal(config.Azure) + case config.GCP != nil: + return json.Marshal(config.GCP) + default: + return nil, errors.NewInternalf(errors.CodeInternal, "no provider account config found") } - - return nil, errors.NewInternalf(errors.CodeInternal, "no provider account config found") } func NewIngestionKeyName(provider CloudProviderType) string { diff --git a/pkg/types/cloudintegrationtypes/checkin.go b/pkg/types/cloudintegrationtypes/checkin.go index 50768e91e3..83def5c7e0 100644 --- a/pkg/types/cloudintegrationtypes/checkin.go +++ b/pkg/types/cloudintegrationtypes/checkin.go @@ -50,6 +50,7 @@ type IntegrationConfig struct { type ProviderIntegrationConfig struct { AWS *AWSIntegrationConfig `json:"aws,omitempty" required:"false" nullable:"false"` Azure *AzureIntegrationConfig `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *GCPIntegrationConfig `json:"gcp,omitempty" required:"false" nullable:"false"` } // NewGettableAgentCheckIn constructs a backward-compatible response from an AgentCheckInResponse. diff --git a/pkg/types/cloudintegrationtypes/cloudintegration.go b/pkg/types/cloudintegrationtypes/cloudintegration.go index 2cfb5509c4..414e465526 100644 --- a/pkg/types/cloudintegrationtypes/cloudintegration.go +++ b/pkg/types/cloudintegrationtypes/cloudintegration.go @@ -63,6 +63,7 @@ type StorableCloudIntegrationService struct { type StorableServiceConfig struct { AWS *StorableAWSServiceConfig Azure *StorableAzureServiceConfig + GCP *StorableGCPServiceConfig } type StorableAWSServiceConfig struct { @@ -92,6 +93,15 @@ type StorableAzureMetricsServiceConfig struct { Enabled bool `json:"enabled"` } +type StorableGCPServiceConfig struct { + Logs *StorableGCPServiceLogsConfig `json:"logs,omitempty"` + Metrics *StorableGCPServiceMetricsConfig `json:"metrics,omitempty"` +} + +type StorableGCPServiceLogsConfig = GCPServiceLogsConfig + +type StorableGCPServiceMetricsConfig = GCPServiceMetricsConfig + // Scan scans value from DB. func (r *StorableAgentReport) Scan(src any) error { var data []byte @@ -225,6 +235,30 @@ func newStorableServiceConfig(provider CloudProviderType, serviceID ServiceID, s } return &StorableServiceConfig{Azure: storableAzureServiceConfig}, nil + case CloudProviderTypeGCP: + storableGCPServiceConfig := new(StorableGCPServiceConfig) + + if supportedSignals.Logs { + if serviceConfig.GCP.Logs == nil { + return nil, errors.NewInvalidInputf(ErrCodeCloudIntegrationInvalidConfig, "logs config is required for GCP service: %s", serviceID.StringValue()) + } + + storableGCPServiceConfig.Logs = &StorableGCPServiceLogsConfig{ + Enabled: serviceConfig.GCP.Logs.Enabled, + } + } + + if supportedSignals.Metrics { + if serviceConfig.GCP.Metrics == nil { + return nil, errors.NewInvalidInputf(ErrCodeCloudIntegrationInvalidConfig, "metrics config is required for GCP service: %s", serviceID.StringValue()) + } + + storableGCPServiceConfig.Metrics = &StorableGCPServiceMetricsConfig{ + Enabled: serviceConfig.GCP.Metrics.Enabled, + } + } + + return &StorableServiceConfig{GCP: storableGCPServiceConfig}, nil default: return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) } @@ -246,6 +280,13 @@ func newStorableServiceConfigFromJSON(provider CloudProviderType, jsonStr string return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't parse Azure service config JSON") } return &StorableServiceConfig{Azure: azureConfig}, nil + case CloudProviderTypeGCP: + gcpConfig := new(StorableGCPServiceConfig) + err := json.Unmarshal([]byte(jsonStr), gcpConfig) + if err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't parse GCP service config JSON") + } + return &StorableServiceConfig{GCP: gcpConfig}, nil default: return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) } @@ -266,6 +307,13 @@ func (config *StorableServiceConfig) toJSON(provider CloudProviderType) ([]byte, return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't serialize Azure service config to JSON") } + return jsonBytes, nil + case CloudProviderTypeGCP: + jsonBytes, err := json.Marshal(config.GCP) + if err != nil { + return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't serialize GCP service config to JSON") + } + return jsonBytes, nil default: return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) diff --git a/pkg/types/cloudintegrationtypes/cloudprovider.go b/pkg/types/cloudintegrationtypes/cloudprovider.go index dd5568fa5b..23d16fcfab 100644 --- a/pkg/types/cloudintegrationtypes/cloudprovider.go +++ b/pkg/types/cloudintegrationtypes/cloudprovider.go @@ -11,6 +11,7 @@ var ( // cloud providers. CloudProviderTypeAWS = CloudProviderType{valuer.NewString("aws")} CloudProviderTypeAzure = CloudProviderType{valuer.NewString("azure")} + CloudProviderTypeGCP = CloudProviderType{valuer.NewString("gcp")} ErrCodeCloudProviderInvalidInput = errors.MustNewCode("cloud_integration_invalid_cloud_provider") ) @@ -21,6 +22,8 @@ func NewCloudProvider(provider string) (CloudProviderType, error) { return CloudProviderTypeAWS, nil case CloudProviderTypeAzure.StringValue(): return CloudProviderTypeAzure, nil + case CloudProviderTypeGCP.StringValue(): + return CloudProviderTypeGCP, nil default: return CloudProviderType{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider) } diff --git a/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go b/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go new file mode 100644 index 0000000000..0095db65ed --- /dev/null +++ b/pkg/types/cloudintegrationtypes/cloudprovider_gcp.go @@ -0,0 +1,40 @@ +package cloudintegrationtypes + +type GCPAccountConfig struct { + // Project ID where central pub/sub for logs exist + DeploymentProjectID string `json:"deploymentProjectId" required:"true"` + // Project ID where otel collector will be deployed + DeploymentRegion string `json:"deploymentRegion" required:"true"` + // List of project IDs to monitor + ProjectIDs []string `json:"projectIds" required:"true" nullable:"false"` +} + +type GCPPostableAccountConfig = GCPAccountConfig + +type UpdatableGCPAccountConfig struct { + // Project ID where central pub/sub for logs exist + DeploymentProjectID string `json:"deploymentProjectId" required:"true"` + // Compute service region where otel collector will be deployed + DeploymentRegion string `json:"deploymentRegion" required:"true"` + // List of project IDs to monitor + ProjectIDs []string `json:"projectIds" required:"true"` +} + +type GCPConnectionArtifact struct{} + +type GCPIntegrationConfig struct{} + +type GCPTelemetryCollectionStrategy struct{} + +type GCPServiceConfig struct { + Logs *GCPServiceLogsConfig `json:"logs,omitempty" required:"false"` + Metrics *GCPServiceMetricsConfig `json:"metrics,omitempty" required:"false"` +} + +type GCPServiceLogsConfig struct { + Enabled bool `json:"enabled" required:"true"` +} + +type GCPServiceMetricsConfig struct { + Enabled bool `json:"enabled" required:"true"` +} diff --git a/pkg/types/cloudintegrationtypes/regions.go b/pkg/types/cloudintegrationtypes/regions.go index a2d2af9511..8feafeaf92 100644 --- a/pkg/types/cloudintegrationtypes/regions.go +++ b/pkg/types/cloudintegrationtypes/regions.go @@ -102,6 +102,51 @@ var ( AzureRegionWestUS = CloudProviderRegion{valuer.NewString("westus")} // West US. AzureRegionWestUS2 = CloudProviderRegion{valuer.NewString("westus2")} // West US 2. AzureRegionWestUS3 = CloudProviderRegion{valuer.NewString("westus3")} // West US 3. + + // GCP regions. + GCPRegionAfricaSouth1 = CloudProviderRegion{valuer.NewString("africa-south1")} // Johannesburg, South Africa. Africa. + GCPRegionAsiaEast1 = CloudProviderRegion{valuer.NewString("asia-east1")} // Changhua County, Taiwan. APAC. + GCPRegionAsiaEast2 = CloudProviderRegion{valuer.NewString("asia-east2")} // Hong Kong. APAC. + GCPRegionAsiaNortheast1 = CloudProviderRegion{valuer.NewString("asia-northeast1")} // Tokyo, Japan. APAC. + GCPRegionAsiaNortheast2 = CloudProviderRegion{valuer.NewString("asia-northeast2")} // Osaka, Japan. APAC. + GCPRegionAsiaNortheast3 = CloudProviderRegion{valuer.NewString("asia-northeast3")} // Seoul, South Korea. APAC. + GCPRegionAsiaSouth1 = CloudProviderRegion{valuer.NewString("asia-south1")} // Mumbai, India. APAC. + GCPRegionAsiaSouth2 = CloudProviderRegion{valuer.NewString("asia-south2")} // Delhi, India. APAC. + GCPRegionAsiaSoutheast1 = CloudProviderRegion{valuer.NewString("asia-southeast1")} // Jurong West, Singapore. APAC. + GCPRegionAsiaSoutheast2 = CloudProviderRegion{valuer.NewString("asia-southeast2")} // Jakarta, Indonesia. APAC. + GCPRegionAsiaSoutheast3 = CloudProviderRegion{valuer.NewString("asia-southeast3")} // Bangkok, Thailand. APAC. + GCPRegionAustraliaSoutheast1 = CloudProviderRegion{valuer.NewString("australia-southeast1")} // Sydney, Australia. APAC. + GCPRegionAustraliaSoutheast2 = CloudProviderRegion{valuer.NewString("australia-southeast2")} // Melbourne, Australia. APAC. + GCPRegionEuropeCentral2 = CloudProviderRegion{valuer.NewString("europe-central2")} // Warsaw, Poland. Europe. + GCPRegionEuropeNorth1 = CloudProviderRegion{valuer.NewString("europe-north1")} // Hamina, Finland. Europe. + GCPRegionEuropeNorth2 = CloudProviderRegion{valuer.NewString("europe-north2")} // Stockholm, Sweden. Europe. + GCPRegionEuropeSouthwest1 = CloudProviderRegion{valuer.NewString("europe-southwest1")} // Madrid, Spain. Europe. + GCPRegionEuropeWest1 = CloudProviderRegion{valuer.NewString("europe-west1")} // St. Ghislain, Belgium. Europe. + GCPRegionEuropeWest2 = CloudProviderRegion{valuer.NewString("europe-west2")} // London, England. Europe. + GCPRegionEuropeWest3 = CloudProviderRegion{valuer.NewString("europe-west3")} // Frankfurt, Germany. Europe. + GCPRegionEuropeWest4 = CloudProviderRegion{valuer.NewString("europe-west4")} // Eemshaven, Netherlands. Europe. + GCPRegionEuropeWest6 = CloudProviderRegion{valuer.NewString("europe-west6")} // Zurich, Switzerland. Europe. + GCPRegionEuropeWest8 = CloudProviderRegion{valuer.NewString("europe-west8")} // Milan, Italy. Europe. + GCPRegionEuropeWest9 = CloudProviderRegion{valuer.NewString("europe-west9")} // Paris, France. Europe. + GCPRegionEuropeWest10 = CloudProviderRegion{valuer.NewString("europe-west10")} // Berlin, Germany. Europe. + GCPRegionEuropeWest12 = CloudProviderRegion{valuer.NewString("europe-west12")} // Turin, Italy. Europe. + GCPRegionMECentral1 = CloudProviderRegion{valuer.NewString("me-central1")} // Doha, Qatar. Middle East. + GCPRegionMECentral2 = CloudProviderRegion{valuer.NewString("me-central2")} // Dammam, Saudi Arabia. Middle East. + GCPRegionMEWest1 = CloudProviderRegion{valuer.NewString("me-west1")} // Tel Aviv, Israel. Middle East. + GCPRegionNorthamericaNortheast1 = CloudProviderRegion{valuer.NewString("northamerica-northeast1")} // Montréal, Québec, Canada. North America. + GCPRegionNorthamericaNortheast2 = CloudProviderRegion{valuer.NewString("northamerica-northeast2")} // Toronto, Ontario, Canada. North America. + GCPRegionNorthamericaSouth1 = CloudProviderRegion{valuer.NewString("northamerica-south1")} // Querétaro, Mexico. North America. + GCPRegionSouthamericaEast1 = CloudProviderRegion{valuer.NewString("southamerica-east1")} // Osasco, São Paulo, Brazil. South America. + GCPRegionSouthamericaWest1 = CloudProviderRegion{valuer.NewString("southamerica-west1")} // Santiago, Chile. South America. + GCPRegionUSCentral1 = CloudProviderRegion{valuer.NewString("us-central1")} // Council Bluffs, Iowa. North America. + GCPRegionUSEast1 = CloudProviderRegion{valuer.NewString("us-east1")} // Moncks Corner, South Carolina. North America. + GCPRegionUSEast4 = CloudProviderRegion{valuer.NewString("us-east4")} // Ashburn, Virginia. North America. + GCPRegionUSEast5 = CloudProviderRegion{valuer.NewString("us-east5")} // Columbus, Ohio. North America. + GCPRegionUSSouth1 = CloudProviderRegion{valuer.NewString("us-south1")} // Dallas, Texas. North America. + GCPRegionUSWest1 = CloudProviderRegion{valuer.NewString("us-west1")} // The Dalles, Oregon. North America. + GCPRegionUSWest2 = CloudProviderRegion{valuer.NewString("us-west2")} // Los Angeles, California. North America. + GCPRegionUSWest3 = CloudProviderRegion{valuer.NewString("us-west3")} // Salt Lake City, Utah. North America. + GCPRegionUSWest4 = CloudProviderRegion{valuer.NewString("us-west4")} // Las Vegas, Nevada. North America. ) func Enum() []any { @@ -127,6 +172,18 @@ func Enum() []any { AzureRegionSwedenCentral, AzureRegionSwitzerlandNorth, AzureRegionSwitzerlandWest, AzureRegionUAECentral, AzureRegionUAENorth, AzureRegionUKSouth, AzureRegionUKWest, AzureRegionWestCentralUS, AzureRegionWestEurope, AzureRegionWestIndia, AzureRegionWestUS, AzureRegionWestUS2, AzureRegionWestUS3, + // GCP regions. + GCPRegionAfricaSouth1, GCPRegionAsiaEast1, GCPRegionAsiaEast2, GCPRegionAsiaNortheast1, GCPRegionAsiaNortheast2, GCPRegionAsiaNortheast3, + GCPRegionAsiaSouth1, GCPRegionAsiaSouth2, GCPRegionAsiaSoutheast1, GCPRegionAsiaSoutheast2, GCPRegionAsiaSoutheast3, + GCPRegionAustraliaSoutheast1, GCPRegionAustraliaSoutheast2, + GCPRegionEuropeCentral2, GCPRegionEuropeNorth1, GCPRegionEuropeNorth2, GCPRegionEuropeSouthwest1, + GCPRegionEuropeWest1, GCPRegionEuropeWest2, GCPRegionEuropeWest3, GCPRegionEuropeWest4, GCPRegionEuropeWest6, + GCPRegionEuropeWest8, GCPRegionEuropeWest9, GCPRegionEuropeWest10, GCPRegionEuropeWest12, + GCPRegionMECentral1, GCPRegionMECentral2, GCPRegionMEWest1, + GCPRegionNorthamericaNortheast1, GCPRegionNorthamericaNortheast2, GCPRegionNorthamericaSouth1, + GCPRegionSouthamericaEast1, GCPRegionSouthamericaWest1, + GCPRegionUSCentral1, GCPRegionUSEast1, GCPRegionUSEast4, GCPRegionUSEast5, GCPRegionUSSouth1, + GCPRegionUSWest1, GCPRegionUSWest2, GCPRegionUSWest3, GCPRegionUSWest4, } } @@ -154,6 +211,19 @@ var SupportedRegions = map[CloudProviderType][]CloudProviderRegion{ AzureRegionUAECentral, AzureRegionUAENorth, AzureRegionUKSouth, AzureRegionUKWest, AzureRegionWestCentralUS, AzureRegionWestEurope, AzureRegionWestIndia, AzureRegionWestUS, AzureRegionWestUS2, AzureRegionWestUS3, }, + CloudProviderTypeGCP: { + GCPRegionAfricaSouth1, GCPRegionAsiaEast1, GCPRegionAsiaEast2, GCPRegionAsiaNortheast1, GCPRegionAsiaNortheast2, GCPRegionAsiaNortheast3, + GCPRegionAsiaSouth1, GCPRegionAsiaSouth2, GCPRegionAsiaSoutheast1, GCPRegionAsiaSoutheast2, GCPRegionAsiaSoutheast3, + GCPRegionAustraliaSoutheast1, GCPRegionAustraliaSoutheast2, + GCPRegionEuropeCentral2, GCPRegionEuropeNorth1, GCPRegionEuropeNorth2, GCPRegionEuropeSouthwest1, + GCPRegionEuropeWest1, GCPRegionEuropeWest2, GCPRegionEuropeWest3, GCPRegionEuropeWest4, GCPRegionEuropeWest6, + GCPRegionEuropeWest8, GCPRegionEuropeWest9, GCPRegionEuropeWest10, GCPRegionEuropeWest12, + GCPRegionMECentral1, GCPRegionMECentral2, GCPRegionMEWest1, + GCPRegionNorthamericaNortheast1, GCPRegionNorthamericaNortheast2, GCPRegionNorthamericaSouth1, + GCPRegionSouthamericaEast1, GCPRegionSouthamericaWest1, + GCPRegionUSCentral1, GCPRegionUSEast1, GCPRegionUSEast4, GCPRegionUSEast5, GCPRegionUSSouth1, + GCPRegionUSWest1, GCPRegionUSWest2, GCPRegionUSWest3, GCPRegionUSWest4, + }, } func validateAWSRegion(region string) error { @@ -175,3 +245,13 @@ func validateAzureRegion(region string) error { return errors.NewInvalidInputf(ErrCodeInvalidCloudRegion, "invalid Azure region: %s", region) } + +func validateGCPRegion(region string) error { + for _, r := range SupportedRegions[CloudProviderTypeGCP] { + if r.StringValue() == region { + return nil + } + } + + return errors.NewInvalidInputf(ErrCodeInvalidCloudRegion, "invalid GCP region: %s", region) +} diff --git a/pkg/types/cloudintegrationtypes/service.go b/pkg/types/cloudintegrationtypes/service.go index f947a5fdd4..73467d5d0d 100644 --- a/pkg/types/cloudintegrationtypes/service.go +++ b/pkg/types/cloudintegrationtypes/service.go @@ -21,6 +21,7 @@ type CloudIntegrationService struct { type ServiceConfig struct { AWS *AWSServiceConfig `json:"aws,omitempty" required:"false" nullable:"false"` Azure *AzureServiceConfig `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *GCPServiceConfig `json:"gcp,omitempty" required:"false" nullable:"false"` } // ServiceMetadata helps to quickly list available services and whether it is enabled or not. @@ -96,6 +97,7 @@ type DataCollected struct { type TelemetryCollectionStrategy struct { AWS *AWSTelemetryCollectionStrategy `json:"aws,omitempty" required:"false" nullable:"false"` Azure *AzureTelemetryCollectionStrategy `json:"azure,omitempty" required:"false" nullable:"false"` + GCP *GCPTelemetryCollectionStrategy `json:"gcp,omitempty" required:"false" nullable:"false"` } // Assets represents the collection of dashboards. @@ -145,6 +147,10 @@ func NewCloudIntegrationService(serviceID ServiceID, cloudIntegrationID valuer.U if config.Azure == nil { return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "Azure config is required for Azure service") } + case CloudProviderTypeGCP: + if config.GCP == nil { + return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "GCP config is required for GCP service") + } } return &CloudIntegrationService{ @@ -261,6 +267,22 @@ func NewServiceConfigFromJSON(provider CloudProviderType, jsonString string) (*S } return &ServiceConfig{Azure: azureServiceConfig}, nil + case CloudProviderTypeGCP: + gcpServiceConfig := new(GCPServiceConfig) + + if storableServiceConfig.GCP.Logs != nil { + gcpServiceConfig.Logs = &GCPServiceLogsConfig{ + Enabled: storableServiceConfig.GCP.Logs.Enabled, + } + } + + if storableServiceConfig.GCP.Metrics != nil { + gcpServiceConfig.Metrics = &GCPServiceMetricsConfig{ + Enabled: storableServiceConfig.GCP.Metrics.Enabled, + } + } + + return &ServiceConfig{GCP: gcpServiceConfig}, nil default: return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) } @@ -285,6 +307,10 @@ func (service *CloudIntegrationService) Update(provider CloudProviderType, servi if config.Azure == nil { return errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "Azure config is required for Azure service") } + case CloudProviderTypeGCP: + if config.GCP == nil { + return errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "GCP config is required for GCP service") + } default: return errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue()) } @@ -306,6 +332,10 @@ func (config *ServiceConfig) IsServiceEnabled(provider CloudProviderType) bool { logsEnabled := config.Azure.Logs != nil && config.Azure.Logs.Enabled metricsEnabled := config.Azure.Metrics != nil && config.Azure.Metrics.Enabled return logsEnabled || metricsEnabled + case CloudProviderTypeGCP: + logsEnabled := config.GCP.Logs != nil && config.GCP.Logs.Enabled + metricsEnabled := config.GCP.Metrics != nil && config.GCP.Metrics.Enabled + return logsEnabled || metricsEnabled default: return false } @@ -319,6 +349,8 @@ func (config *ServiceConfig) IsMetricsEnabled(provider CloudProviderType) bool { return config.AWS.Metrics != nil && config.AWS.Metrics.Enabled case CloudProviderTypeAzure: return config.Azure.Metrics != nil && config.Azure.Metrics.Enabled + case CloudProviderTypeGCP: + return config.GCP.Metrics != nil && config.GCP.Metrics.Enabled default: return false } @@ -331,6 +363,8 @@ func (config *ServiceConfig) IsLogsEnabled(provider CloudProviderType) bool { return config.AWS.Logs != nil && config.AWS.Logs.Enabled case CloudProviderTypeAzure: return config.Azure.Logs != nil && config.Azure.Logs.Enabled + case CloudProviderTypeGCP: + return config.GCP.Logs != nil && config.GCP.Logs.Enabled default: return false } diff --git a/pkg/types/cloudintegrationtypes/serviceid.go b/pkg/types/cloudintegrationtypes/serviceid.go index d9fbc94bf7..d790bc7f76 100644 --- a/pkg/types/cloudintegrationtypes/serviceid.go +++ b/pkg/types/cloudintegrationtypes/serviceid.go @@ -39,6 +39,9 @@ var ( AzureServiceCosmosDB = ServiceID{valuer.NewString("cosmosdb")} AzureServiceCassandraDB = ServiceID{valuer.NewString("cassandradb")} AzureServiceRedis = ServiceID{valuer.NewString("redis")} + + // GCP services. + GCPServiceCloudSQL = ServiceID{valuer.NewString("cloudsql")} ) func (ServiceID) Enum() []any { @@ -70,6 +73,7 @@ func (ServiceID) Enum() []any { AzureServiceCosmosDB, AzureServiceCassandraDB, AzureServiceRedis, + GCPServiceCloudSQL, } } @@ -106,6 +110,9 @@ var SupportedServices = map[CloudProviderType][]ServiceID{ AzureServiceCassandraDB, AzureServiceRedis, }, + CloudProviderTypeGCP: { + GCPServiceCloudSQL, + }, } func NewServiceID(provider CloudProviderType, service string) (ServiceID, error) { diff --git a/pkg/types/coretypes/object.go b/pkg/types/coretypes/object.go index d7cb07599f..638f421fdc 100644 --- a/pkg/types/coretypes/object.go +++ b/pkg/types/coretypes/object.go @@ -46,18 +46,33 @@ func MustNewObject(resource ResourceRef, inputSelector string) *Object { } func MustNewObjectFromString(input string) *Object { + typeParts := strings.SplitN(input, ":", 2) + if len(typeParts) != 2 { + panic(errors.Newf(errors.TypeInternal, errors.CodeInternal, "invalid type format: %s", input)) + } + + typed := MustNewType(typeParts[0]) + + // The organization resource is the root entity and encodes its object as + // "organization:organization/" — without the orgID and kind + // segments used by every other resource (":organization///"). + if typed.Equals(TypeOrganization) { + orgParts := strings.Split(typeParts[1], "/") + if len(orgParts) != 2 { + panic(errors.Newf(errors.TypeInternal, errors.CodeInternal, "invalid input format: %s", input)) + } + + resource := ResourceRef{Type: typed, Kind: MustNewKind(orgParts[0])} + return &Object{Resource: resource, Selector: typed.MustSelector(orgParts[1])} + } + parts := strings.Split(input, "/") if len(parts) != 4 { panic(errors.Newf(errors.TypeInternal, errors.CodeInternal, "invalid input format: %s", input)) } - typeParts := strings.Split(parts[0], ":") - if len(typeParts) != 2 { - panic(errors.Newf(errors.TypeInternal, errors.CodeInternal, "invalid type format: %s", parts[0])) - } - resource := ResourceRef{ - Type: MustNewType(typeParts[0]), + Type: typed, Kind: MustNewKind(parts[2]), } diff --git a/pkg/types/coretypes/registry_managed_role.go b/pkg/types/coretypes/registry_managed_role.go index 49e0f29255..c2a2eecb93 100644 --- a/pkg/types/coretypes/registry_managed_role.go +++ b/pkg/types/coretypes/registry_managed_role.go @@ -83,9 +83,6 @@ var ManagedRoleToTransactions = map[string][]Transaction{ {Verb: VerbDelete, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindSubscription}, WildCardSelectorString)}, {Verb: VerbCreate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindSubscription}, WildCardSelectorString)}, {Verb: VerbList, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindSubscription}, WildCardSelectorString)}, - // organization — admin only - {Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeOrganization, Kind: KindOrganization}, WildCardSelectorString)}, - {Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeOrganization, Kind: KindOrganization}, WildCardSelectorString)}, // org-preference — admin only {Verb: VerbRead, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindOrgPreference}, WildCardSelectorString)}, {Verb: VerbUpdate, Object: *MustNewObject(ResourceRef{Type: TypeMetaResource, Kind: KindOrgPreference}, WildCardSelectorString)}, diff --git a/pkg/types/dashboardtypes/dashboards_metric_usage.go b/pkg/types/dashboardtypes/dashboards_metric_usage.go index a4d898e3d6..5743095550 100644 --- a/pkg/types/dashboardtypes/dashboards_metric_usage.go +++ b/pkg/types/dashboardtypes/dashboards_metric_usage.go @@ -1,11 +1,13 @@ package dashboardtypes -// DashboardPanelRef identifies a single panel within a dashboard. The -// "dashboards by metric name" lookup returns these to report each panel that -// references a given metric. +// DashboardPanelRef identifies a single panel/widget within a dashboard that +// references a given metric, along with the group-by and filter labels used for +// that metric so callers can reason about the impact of dropping its labels. type DashboardPanelRef struct { - DashboardID string `json:"dashboardId" required:"true"` - DashboardName string `json:"dashboardName" required:"true"` - PanelID string `json:"panelId" required:"true"` - PanelName string `json:"panelName" required:"true"` + DashboardID string `json:"dashboardId" required:"true"` + DashboardName string `json:"dashboardName" required:"true"` + PanelID string `json:"panelId" required:"true"` + PanelName string `json:"panelName" required:"true"` + GroupBy []string `json:"groupBy,omitempty"` + FilterBy []string `json:"filterBy,omitempty"` } diff --git a/pkg/types/metricreductionruletypes/list.go b/pkg/types/metricreductionruletypes/list.go index a1e9f4efc5..1ef1e1548c 100644 --- a/pkg/types/metricreductionruletypes/list.go +++ b/pkg/types/metricreductionruletypes/list.go @@ -26,16 +26,15 @@ var ( OrderByMetricName = ReductionRuleOrderBy{valuer.NewString("metric")} OrderByIngestedVolume = ReductionRuleOrderBy{valuer.NewString("ingested_volume")} OrderByReducedVolume = ReductionRuleOrderBy{valuer.NewString("reduced_volume")} - OrderByReduction = ReductionRuleOrderBy{valuer.NewString("reduction")} OrderByLastUpdated = ReductionRuleOrderBy{valuer.NewString("last_updated")} ) func (ReductionRuleOrderBy) Enum() []any { - return []any{OrderByMetricName, OrderByIngestedVolume, OrderByReducedVolume, OrderByReduction, OrderByLastUpdated} + return []any{OrderByMetricName, OrderByIngestedVolume, OrderByReducedVolume, OrderByLastUpdated} } type ListReductionRulesParams struct { - OrderBy ReductionRuleOrderBy `query:"orderBy,default=reduction" json:"orderBy"` + OrderBy ReductionRuleOrderBy `query:"orderBy,default=ingested_volume" json:"orderBy"` Order Order `query:"order,default=desc" json:"order"` Search string `query:"search" json:"search"` MetricName string `query:"metricName" json:"metricName,omitempty"` diff --git a/pkg/types/metricreductionruletypes/list_test.go b/pkg/types/metricreductionruletypes/list_test.go index 036a07f3e3..f82741feeb 100644 --- a/pkg/types/metricreductionruletypes/list_test.go +++ b/pkg/types/metricreductionruletypes/list_test.go @@ -13,7 +13,7 @@ func TestListReductionRulesParamsSortDefaults(t *testing.T) { var params metricreductionruletypes.ListReductionRulesParams require.NoError(t, binding.Query.BindQuery(map[string][]string{"limit": {"10"}}, ¶ms)) - assert.Equal(t, metricreductionruletypes.OrderByReduction, params.OrderBy, "orderBy defaults to reduction") + assert.Equal(t, metricreductionruletypes.OrderByIngestedVolume, params.OrderBy, "orderBy defaults to ingested volume") assert.Equal(t, metricreductionruletypes.OrderDesc, params.Order, "order defaults to desc") } diff --git a/pkg/types/metricreductionruletypes/reduction_rule.go b/pkg/types/metricreductionruletypes/reduction_rule.go index 1a7f14948d..6194889816 100644 --- a/pkg/types/metricreductionruletypes/reduction_rule.go +++ b/pkg/types/metricreductionruletypes/reduction_rule.go @@ -102,7 +102,8 @@ type GettableReductionRule struct { Active bool `json:"active" required:"true"` IngestedSeries uint64 `json:"ingestedSeries" required:"true"` RetainedSeries uint64 `json:"retainedSeries" required:"true"` - ReductionPercent float64 `json:"reductionPercent" required:"true"` + IngestedSamples uint64 `json:"ingestedSamples" required:"true"` + RetainedSamples uint64 `json:"retainedSamples" required:"true"` } type GettableReductionRules struct { diff --git a/pkg/types/metricreductionruletypes/stats.go b/pkg/types/metricreductionruletypes/stats.go index d5f15c40fc..1c6152cfe6 100644 --- a/pkg/types/metricreductionruletypes/stats.go +++ b/pkg/types/metricreductionruletypes/stats.go @@ -4,5 +4,7 @@ package metricreductionruletypes type GettableReductionRuleStats struct { IngestedSeries uint64 `json:"ingestedSeries" required:"true"` RetainedSeries uint64 `json:"retainedSeries" required:"true"` + IngestedSamples uint64 `json:"ingestedSamples" required:"true"` + RetainedSamples uint64 `json:"retainedSamples" required:"true"` EstimatedMonthlySavingsUsd float64 `json:"estimatedMonthlySavingsUsd" required:"true"` } diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/preview.go b/pkg/types/querybuildertypes/querybuildertypesv5/preview.go new file mode 100644 index 0000000000..d34e89bc72 --- /dev/null +++ b/pkg/types/querybuildertypes/querybuildertypesv5/preview.go @@ -0,0 +1,10 @@ +package querybuildertypesv5 + +// PreviewTask is one rendered statement queued for granule/estimate analysis. +// StmtIdx is where its results merge back into the query's Statements. +type PreviewTask struct { + Name string + StmtIdx int + Query string + Args []any +} diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go index b6899cbded..abc93ef267 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/qb.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/qb.go @@ -58,3 +58,8 @@ type TraceOperatorStatementBuilder interface { // Build builds the trace operator query. Build(ctx context.Context, start, end uint64, requestType RequestType, query QueryBuilderTraceOperator, compositeQuery *CompositeQuery) (*Statement, error) } + +// StatementProvider renders a query's underlying statement without executing it. +type StatementProvider interface { + Statement(ctx context.Context) (*Statement, error) +} diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/resp.go b/pkg/types/querybuildertypes/querybuildertypesv5/resp.go index 85a381be27..c0aa6c260b 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/resp.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/resp.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/SigNoz/signoz/pkg/errors" + "github.com/SigNoz/signoz/pkg/types/telemetrystoretypes" "github.com/SigNoz/signoz/pkg/types/telemetrytypes" "github.com/SigNoz/signoz/pkg/valuer" "github.com/swaggest/jsonschema-go" @@ -64,6 +66,64 @@ type QueryRangeResponse struct { QBEvent *QBEvent `json:"-"` } +// QueryRangePreviewResponse is the dry-run output: one QueryPreview per query, +// keyed by the request's query names. +type QueryRangePreviewResponse struct { + CompositeQuery map[string]QueryPreview `json:"compositeQuery" required:"true" nullable:"true"` +} + +// QueryRangePreviewOptions carries per-call options for the dry-run endpoint. +type QueryRangePreviewOptions struct { + Verbose bool +} + +// QueryRangePreviewParams are the query-string parameters of the dry-run endpoint. +type QueryRangePreviewParams struct { + Verbose string `query:"verbose"` +} + +// PrepareJSONSchema adds description to the QueryRangePreviewResponse schema. +func (q *QueryRangePreviewResponse) PrepareJSONSchema(schema *jsonschema.Schema) error { + schema.WithDescription("Response from the v5 query range preview (dry-run) endpoint. For each query in the composite query, returns the underlying ClickHouse statement(s) it renders to without executing them (one per PromQL metric selector; exactly one for builder/ClickHouse/trace-operator queries), with the optional EXPLAIN ESTIMATE and granule analysis attached per statement when requested.") + return nil +} + +// QueryPreview is the dry-run result for a single query. +type QueryPreview struct { + Valid bool `json:"valid" required:"true" nullable:"false"` + Error error `json:"error" required:"true"` + Warnings []string `json:"warnings" required:"true" nullable:"false"` + Statements []PreviewStatement `json:"statements" required:"true" nullable:"false"` +} + +// PreviewStatement is one rendered ClickHouse statement with its args and, when +// requested, its EXPLAIN ESTIMATE and granule breakdown. The query/args JSON +// keys follow the OpenTelemetry db.statement.* convention. +type PreviewStatement struct { + Query string `json:"db.statement.query" required:"true" nullable:"false"` + Args []any `json:"db.statement.args" required:"true" nullable:"false"` + Estimate []telemetrystoretypes.EstimateEntry `json:"estimate" required:"true" nullable:"false"` + Granules *telemetrystoretypes.Granules `json:"granules" required:"true" nullable:"true"` +} + +// MarshalJSON renders Error in its structured form (code/message/suggestions) +// rather than the empty object a bare error produces. The nullable:"false" +// arrays are non-nil from the producer, so they marshal as [] rather than null. +func (p QueryPreview) MarshalJSON() ([]byte, error) { + type alias QueryPreview + out := struct { + alias + Error *errors.JSON `json:"error"` + }{alias: alias(p)} + out.alias.Error = nil + // Derive the verdict so the two can't desync. + out.Valid = p.Error == nil + if p.Error != nil { + out.Error = errors.AsJSON(p.Error) + } + return json.Marshal(out) +} + var _ jsonschema.Preparer = &QueryRangeResponse{} // PrepareJSONSchema adds description to the QueryRangeResponse schema. @@ -256,7 +316,6 @@ type RawStream struct { Error chan error } - func roundToNonZeroDecimals(val float64, n int) float64 { if val == 0 || math.IsNaN(val) || math.IsInf(val, 0) { return val diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/validation.go b/pkg/types/querybuildertypes/querybuildertypesv5/validation.go index 23f4045338..2329e08220 100644 --- a/pkg/types/querybuildertypes/querybuildertypesv5/validation.go +++ b/pkg/types/querybuildertypes/querybuildertypesv5/validation.go @@ -575,6 +575,75 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error { return nil } +// ValidateRequestScope validates request-level invariants (not individual query +// specs) and returns the request type's ValidationOptions. The dry-run path uses +// this so per-query errors can be attributed individually via QueryEnvelope.Validate +// instead of failing fast like Validate does. +func (r *QueryRangeRequest) ValidateRequestScope() ([]ValidationOption, error) { + if r.RequestType != RequestTypeRawStream && r.Start >= r.End { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "start time must be before end time") + } + + var opts []ValidationOption + switch r.RequestType { + case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeTimeSeries, RequestTypeScalar: + opts = GetValidationOptions(r.RequestType) + default: + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid request type: %s", r.RequestType). + WithAdditional("Valid request types are: raw, timeseries, scalar") + } + + if r.RequestType == RequestTypeRaw || r.RequestType == RequestTypeRawStream || r.RequestType == RequestTypeTrace { + for _, envelope := range r.CompositeQuery.Queries { + if envelope.GetSignal() == telemetrytypes.SignalMetrics { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "raw request type is not supported for metric queries") + } + } + } + + if len(r.CompositeQuery.Queries) == 0 { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "at least one query is required") + } + + // Builder query names must be unique across the composite query. + queryNames := make(map[string]bool) + for _, envelope := range r.CompositeQuery.Queries { + if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery { + name := envelope.GetQueryName() + if name != "" { + if queryNames[name] { + return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "duplicate query name '%s'", name) + } + queryNames[name] = true + } + } + } + + if err := r.validateAllQueriesNotDisabled(); err != nil { + return nil, err + } + + return opts, nil +} + +// Validate parses the preview query-string parameters. Verbose defaults to true +// and accepts true/1/false/0; any other value is rejected. +func (p *QueryRangePreviewParams) Validate() (QueryRangePreviewOptions, error) { + switch strings.ToLower(strings.TrimSpace(p.Verbose)) { + case "", "true", "1": + return QueryRangePreviewOptions{Verbose: true}, nil + case "false", "0": + return QueryRangePreviewOptions{Verbose: false}, nil + } + return QueryRangePreviewOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid verbose value %q (allowed: true, false)", p.Verbose) +} + +// Validate validates a single query envelope's spec — the per-query counterpart +// to ValidateRequestScope, letting the dry-run report errors independently. +func (e QueryEnvelope) Validate(opts ...ValidationOption) error { + return validateQueryEnvelope(e, opts...) +} + // validateAllQueriesNotDisabled validates that at least one query in the composite query is enabled. func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error { for _, envelope := range r.CompositeQuery.Queries { diff --git a/pkg/types/telemetrystoretypes/explain.go b/pkg/types/telemetrystoretypes/explain.go new file mode 100644 index 0000000000..0133efa200 --- /dev/null +++ b/pkg/types/telemetrystoretypes/explain.go @@ -0,0 +1,41 @@ +package telemetrystoretypes + +// EstimateEntry is ClickHouse's EXPLAIN ESTIMATE for one table read: the parts, +// rows, and marks it estimates it will scan. +type EstimateEntry struct { + Database string `json:"database" required:"true" nullable:"false"` + Table string `json:"table" required:"true" nullable:"false"` + Parts int64 `json:"parts" required:"true" nullable:"false"` + Rows int64 `json:"rows" required:"true" nullable:"false"` + Marks int64 `json:"marks" required:"true" nullable:"false"` +} + +// Granules is the granule-skip breakdown for one statement, summed from +// `EXPLAIN json = 1, indexes = 1` across every ReadFromMergeTree node. +type Granules struct { + Initial int64 `json:"initial" required:"true" nullable:"false"` + Selected int64 `json:"selected" required:"true" nullable:"false"` + Skipped int64 `json:"skipped" required:"true" nullable:"false"` + Reads []MergeTreeRead `json:"reads" required:"true" nullable:"false"` +} + +// MergeTreeRead is the index-pruning funnel for one ReadFromMergeTree node. Steps +// run in sequence, so each step's Initial* matches the previous Selected*. +type MergeTreeRead struct { + Table string `json:"table" required:"true" nullable:"false"` + Steps []IndexStep `json:"steps" required:"true" nullable:"false"` +} + +// IndexStep is one index applied during a MergeTree read: parts and granules +// entering (Initial*) and surviving (Selected*) it. Type is the index kind +// (MinMax, Partition, PrimaryKey, or Skip). +type IndexStep struct { + Type string `json:"type" required:"true" nullable:"false"` + Name string `json:"name" required:"true" nullable:"false"` + Keys []string `json:"keys" required:"true" nullable:"false"` + Condition string `json:"condition" required:"true" nullable:"false"` + InitialParts int64 `json:"initialParts" required:"true" nullable:"false"` + SelectedParts int64 `json:"selectedParts" required:"true" nullable:"false"` + InitialGranules int64 `json:"initialGranules" required:"true" nullable:"false"` + SelectedGranules int64 `json:"selectedGranules" required:"true" nullable:"false"` +} diff --git a/pkg/types/telemetrytypes/store.go b/pkg/types/telemetrytypes/store.go index db0df7d922..086e21ef24 100644 --- a/pkg/types/telemetrytypes/store.go +++ b/pkg/types/telemetrytypes/store.go @@ -24,7 +24,7 @@ type MetadataStore interface { GetRelatedValues(ctx context.Context, fieldValueSelector *FieldValueSelector) ([]string, bool, error) // GetAllValues returns a list of all values. - GetAllValues(ctx context.Context, fieldValueSelector *FieldValueSelector) (*TelemetryFieldValues, bool, error) + GetAllValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *FieldValueSelector) (*TelemetryFieldValues, bool, error) // FetchTemporality fetches the temporality for metric FetchTemporality(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error) @@ -46,7 +46,7 @@ type MetadataStore interface { // GetFirstSeenFromMetricMetadata gets the first seen timestamp for a metric metadata lookup key. GetFirstSeenFromMetricMetadata(ctx context.Context, lookupKeys []MetricMetadataLookupKey) (map[MetricMetadataLookupKey]int64, error) - FetchLastSeenInfoMulti(ctx context.Context, metricNames ...string) (map[string]int64, error) + FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) } type MetricMetadataLookupKey struct { diff --git a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go index 61c667e58d..8366ceb80b 100644 --- a/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go +++ b/pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go @@ -181,7 +181,7 @@ func (m *MockMetadataStore) GetRelatedValues(ctx context.Context, fieldValueSele } // GetAllValues returns all values for a given field. -func (m *MockMetadataStore) GetAllValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) { +func (m *MockMetadataStore) GetAllValues(ctx context.Context, _ valuer.UUID, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) { if fieldValueSelector == nil { return &telemetrytypes.TelemetryFieldValues{}, true, nil } @@ -419,6 +419,6 @@ func (m *MockMetadataStore) SetFirstSeenFromMetricMetadata(firstSeenMap map[tele } } -func (m *MockMetadataStore) FetchLastSeenInfoMulti(ctx context.Context, metricNames ...string) (map[string]int64, error) { +func (m *MockMetadataStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) { return make(map[string]int64), nil } diff --git a/pkg/web/config.go b/pkg/web/config.go index 4fa3c428e0..a2807a7a09 100644 --- a/pkg/web/config.go +++ b/pkg/web/config.go @@ -28,19 +28,27 @@ type SettingsConfig struct { } type PosthogConfig struct { - Enabled bool `mapstructure:"enabled"` + Enabled bool `mapstructure:"enabled"` + Key string `mapstructure:"key"` + APIHost string `mapstructure:"api_host"` + UIHost string `mapstructure:"ui_host"` } type AppcuesConfig struct { - Enabled bool `mapstructure:"enabled"` + Enabled bool `mapstructure:"enabled"` + AppID string `mapstructure:"app_id"` } type SentryConfig struct { - Enabled bool `mapstructure:"enabled"` + Enabled bool `mapstructure:"enabled"` + DSN string `mapstructure:"dsn"` + Tunnel string `mapstructure:"tunnel"` } type PylonConfig struct { - Enabled bool `mapstructure:"enabled"` + Enabled bool `mapstructure:"enabled"` + AppID string `mapstructure:"app_id"` + IdentitySecret string `mapstructure:"identity_secret"` } func NewConfigFactory() factory.ConfigFactory { diff --git a/pkg/web/config_test.go b/pkg/web/config_test.go index f131e9622b..e71150d60b 100644 --- a/pkg/web/config_test.go +++ b/pkg/web/config_test.go @@ -48,17 +48,26 @@ func TestSettingsConfigWithEnvProvider(t *testing.T) { testCases := []struct { name string env string + value string expected SettingsConfig }{ - {name: "posthog", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_ENABLED", expected: SettingsConfig{Posthog: PosthogConfig{Enabled: true}}}, - {name: "appcues", env: "SIGNOZ_WEB_SETTINGS_APPCUES_ENABLED", expected: SettingsConfig{Appcues: AppcuesConfig{Enabled: true}}}, - {name: "sentry", env: "SIGNOZ_WEB_SETTINGS_SENTRY_ENABLED", expected: SettingsConfig{Sentry: SentryConfig{Enabled: true}}}, - {name: "pylon", env: "SIGNOZ_WEB_SETTINGS_PYLON_ENABLED", expected: SettingsConfig{Pylon: PylonConfig{Enabled: true}}}, + {name: "posthog_enabled", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_ENABLED", value: "true", expected: SettingsConfig{Posthog: PosthogConfig{Enabled: true}}}, + {name: "posthog_key", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_KEY", value: "phc_examplekey", expected: SettingsConfig{Posthog: PosthogConfig{Key: "phc_examplekey"}}}, + {name: "posthog_api_host", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_API__HOST", value: "https://eu.i.posthog.com", expected: SettingsConfig{Posthog: PosthogConfig{APIHost: "https://eu.i.posthog.com"}}}, + {name: "posthog_ui_host", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_UI__HOST", value: "https://eu.posthog.com", expected: SettingsConfig{Posthog: PosthogConfig{UIHost: "https://eu.posthog.com"}}}, + {name: "appcues_enabled", env: "SIGNOZ_WEB_SETTINGS_APPCUES_ENABLED", value: "true", expected: SettingsConfig{Appcues: AppcuesConfig{Enabled: true}}}, + {name: "appcues_app_id", env: "SIGNOZ_WEB_SETTINGS_APPCUES_APP__ID", value: "12345-abcde", expected: SettingsConfig{Appcues: AppcuesConfig{AppID: "12345-abcde"}}}, + {name: "sentry_enabled", env: "SIGNOZ_WEB_SETTINGS_SENTRY_ENABLED", value: "true", expected: SettingsConfig{Sentry: SentryConfig{Enabled: true}}}, + {name: "sentry_dsn", env: "SIGNOZ_WEB_SETTINGS_SENTRY_DSN", value: "https://examplePublicKey@o0.ingest.sentry.io/0", expected: SettingsConfig{Sentry: SentryConfig{DSN: "https://examplePublicKey@o0.ingest.sentry.io/0"}}}, + {name: "sentry_tunnel", env: "SIGNOZ_WEB_SETTINGS_SENTRY_TUNNEL", value: "https://example.com/tunnel", expected: SettingsConfig{Sentry: SentryConfig{Tunnel: "https://example.com/tunnel"}}}, + {name: "pylon_enabled", env: "SIGNOZ_WEB_SETTINGS_PYLON_ENABLED", value: "true", expected: SettingsConfig{Pylon: PylonConfig{Enabled: true}}}, + {name: "pylon_app_id", env: "SIGNOZ_WEB_SETTINGS_PYLON_APP__ID", value: "pylon-app-id", expected: SettingsConfig{Pylon: PylonConfig{AppID: "pylon-app-id"}}}, + {name: "pylon_identity_secret", env: "SIGNOZ_WEB_SETTINGS_PYLON_IDENTITY__SECRET", value: "pylon-secret", expected: SettingsConfig{Pylon: PylonConfig{IdentitySecret: "pylon-secret"}}}, } for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { - t.Setenv(testCase.env, "true") + t.Setenv(testCase.env, testCase.value) conf, err := config.New( context.Background(), diff --git a/pkg/web/routerweb/provider_test.go b/pkg/web/routerweb/provider_test.go index 6436b6c9ae..72ccd54396 100644 --- a/pkg/web/routerweb/provider_test.go +++ b/pkg/web/routerweb/provider_test.go @@ -119,13 +119,49 @@ func TestServeTemplatedIndex(t *testing.T) { Index: "valid_template.html", Directory: "testdata", Settings: web.SettingsConfig{ - Posthog: web.PosthogConfig{Enabled: true}, - Appcues: web.AppcuesConfig{Enabled: true}, + Posthog: web.PosthogConfig{ + Enabled: true, + Key: "phc_examplekey", + APIHost: "https://us.i.posthog.com", + UIHost: "https://us.posthog.com", + }, + Appcues: web.AppcuesConfig{ + Enabled: true, + AppID: "12345-abcde", + }, + Sentry: web.SentryConfig{ + Enabled: true, + DSN: "https://examplePublicKey@o0.ingest.sentry.io/0", + Tunnel: "https://example.com/tunnel", + }, + Pylon: web.PylonConfig{ + Enabled: true, + AppID: "pylon-app-id", + IdentitySecret: "pylon-secret", + }, }, }, expected: expectedHTML("/", web.Settings{ - Posthog: web.Posthog{Enabled: true}, - Appcues: web.Appcues{Enabled: true}, + Posthog: web.Posthog{ + Enabled: true, + Key: "phc_examplekey", + APIHost: "https://us.i.posthog.com", + UIHost: "https://us.posthog.com", + }, + Appcues: web.Appcues{ + Enabled: true, + AppID: "12345-abcde", + }, + Sentry: web.Sentry{ + Enabled: true, + DSN: "https://examplePublicKey@o0.ingest.sentry.io/0", + Tunnel: "https://example.com/tunnel", + }, + Pylon: web.Pylon{ + Enabled: true, + AppID: "pylon-app-id", + IdentitySecret: "pylon-secret", + }, }), }, } diff --git a/pkg/web/settings.go b/pkg/web/settings.go index 9195eae32d..2b5f6b49cb 100644 --- a/pkg/web/settings.go +++ b/pkg/web/settings.go @@ -8,34 +8,50 @@ type Settings struct { } type Posthog struct { - Enabled bool `json:"enabled" required:"true"` + Enabled bool `json:"enabled" required:"true"` + Key string `json:"key"` + APIHost string `json:"apiHost"` + UIHost string `json:"uiHost"` } type Appcues struct { - Enabled bool `json:"enabled" required:"true"` + Enabled bool `json:"enabled" required:"true"` + AppID string `json:"appId"` } type Sentry struct { - Enabled bool `json:"enabled" required:"true"` + Enabled bool `json:"enabled" required:"true"` + DSN string `json:"dsn"` + Tunnel string `json:"tunnel"` } type Pylon struct { - Enabled bool `json:"enabled" required:"true"` + Enabled bool `json:"enabled" required:"true"` + AppID string `json:"appId"` + IdentitySecret string `json:"identitySecret"` } func NewSettings(config Config) Settings { return Settings{ Posthog: Posthog{ Enabled: config.Settings.Posthog.Enabled, + Key: config.Settings.Posthog.Key, + APIHost: config.Settings.Posthog.APIHost, + UIHost: config.Settings.Posthog.UIHost, }, Appcues: Appcues{ Enabled: config.Settings.Appcues.Enabled, + AppID: config.Settings.Appcues.AppID, }, Sentry: Sentry{ Enabled: config.Settings.Sentry.Enabled, + DSN: config.Settings.Sentry.DSN, + Tunnel: config.Settings.Sentry.Tunnel, }, Pylon: Pylon{ - Enabled: config.Settings.Pylon.Enabled, + Enabled: config.Settings.Pylon.Enabled, + AppID: config.Settings.Pylon.AppID, + IdentitySecret: config.Settings.Pylon.IdentitySecret, }, } } diff --git a/tests/integration/tests/cloudintegrations/01_credentials.py b/tests/integration/tests/cloudintegrations/01_credentials.py index 41a9d46633..45be288087 100644 --- a/tests/integration/tests/cloudintegrations/01_credentials.py +++ b/tests/integration/tests/cloudintegrations/01_credentials.py @@ -143,7 +143,7 @@ def test_get_credentials_unsupported_provider( admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) response = requests.get( - signoz.self.host_configs["8080"].get("/api/v1/cloud_integrations/gcp/credentials"), + signoz.self.host_configs["8080"].get("/api/v1/cloud_integrations/unknown/credentials"), headers={"Authorization": f"Bearer {admin_token}"}, timeout=10, ) diff --git a/tests/integration/tests/cloudintegrations/02_create_account.py b/tests/integration/tests/cloudintegrations/02_create_account.py index 2bf88c12a8..97c01ddd5d 100644 --- a/tests/integration/tests/cloudintegrations/02_create_account.py +++ b/tests/integration/tests/cloudintegrations/02_create_account.py @@ -56,14 +56,14 @@ def test_create_account_unsupported_provider( ) -> None: """Test that creating an account with an unsupported cloud provider returns 400.""" admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) - cloud_provider = "gcp" + cloud_provider = "unknown" endpoint = f"/api/v1/cloud_integrations/{cloud_provider}/accounts" response = requests.post( signoz.self.host_configs["8080"].get(endpoint), headers={"Authorization": f"Bearer {admin_token}"}, json={ - "config": {"gcp": {"deploymentRegion": "us-central1", "regions": ["us-central1"]}}, + "config": {"unknown": {"deploymentRegion": "us-central1", "regions": ["us-central1"]}}, "credentials": { "sigNozApiURL": "https://test.signoz.cloud", "sigNozApiKey": "test-key", diff --git a/tests/integration/tests/cloudintegrations/05_services.py b/tests/integration/tests/cloudintegrations/05_services.py index b43e568803..cd6962afeb 100644 --- a/tests/integration/tests/cloudintegrations/05_services.py +++ b/tests/integration/tests/cloudintegrations/05_services.py @@ -341,7 +341,7 @@ def test_list_services_unsupported_provider( admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD) response = requests.get( - signoz.self.host_configs["8080"].get("/api/v1/cloud_integrations/gcp/services"), + signoz.self.host_configs["8080"].get("/api/v1/cloud_integrations/unknown/services"), headers={"Authorization": f"Bearer {admin_token}"}, timeout=10, )