Compare commits

...

2 Commits

Author SHA1 Message Date
srikanthccv
b2b73965a5 chore: the reduced metrics show up in summary page 2026-06-29 02:04:11 +05:30
srikanthccv
518cb01824 chore(metrics): review follow ups for volume control 2026-06-29 01:53:49 +05:30
29 changed files with 550 additions and 192 deletions

View File

@@ -5299,6 +5299,9 @@ components:
type: string
id:
type: string
ingestedSamples:
minimum: 0
type: integer
ingestedSeries:
minimum: 0
type: integer
@@ -5311,9 +5314,9 @@ components:
$ref: '#/components/schemas/MetricreductionruletypesMatchType'
metricName:
type: string
reductionPercent:
format: double
type: number
retainedSamples:
minimum: 0
type: integer
retainedSeries:
minimum: 0
type: integer
@@ -5331,7 +5334,8 @@ components:
- active
- ingestedSeries
- retainedSeries
- reductionPercent
- ingestedSamples
- retainedSamples
type: object
MetricreductionruletypesGettableReductionRulePreview:
properties:
@@ -5374,15 +5378,23 @@ components:
estimatedMonthlySavingsUsd:
format: double
type: number
ingestedSamples:
minimum: 0
type: integer
ingestedSeries:
minimum: 0
type: integer
retainedSamples:
minimum: 0
type: integer
retainedSeries:
minimum: 0
type: integer
required:
- ingestedSeries
- retainedSeries
- ingestedSamples
- retainedSamples
- estimatedMonthlySavingsUsd
type: object
MetricreductionruletypesGettableReductionRules:
@@ -5448,7 +5460,6 @@ components:
- metric
- ingested_volume
- reduced_volume
- reduction
- last_updated
type: string
MetricreductionruletypesUpdatableReductionRule:

View File

@@ -289,12 +289,9 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff
}
ctx = c.withThreads(ctx)
orderExpr := "ingested"
switch orderBy {
case metricreductionruletypes.OrderByReducedVolume:
orderExpr = "reduced"
case metricreductionruletypes.OrderByReduction:
orderExpr = "if(ingested = 0, 0, (toFloat64(ingested) - toFloat64(reduced)) / toFloat64(ingested))"
orderExpr := "ifNull(i.samples, 0)"
if orderBy == metricreductionruletypes.OrderByReducedVolume {
orderExpr = "if(ifNull(d.samples, 0) = 0 OR ifNull(d.samples, 0) > ifNull(i.samples, 0), ifNull(i.samples, 0), ifNull(d.samples, 0))"
}
direction := "ASC"
if order == metricreductionruletypes.OrderDesc {
@@ -310,17 +307,17 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff
sb.From("(SELECT arrayJoin(" + sb.Var(metricNames) + ") AS metric_name) AS base")
sb.JoinWithOption(
sqlbuilder.LeftJoin,
"(SELECT metric_name, uniq(fingerprint) AS cnt FROM "+ingestedTable+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name) AS i",
"(SELECT metric_name, uniq(fingerprint) AS cnt, count() AS samples FROM "+ingestedTable+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name) AS i",
"base.metric_name = i.metric_name",
)
// Reduced series are spread across two type-specific tables; union the per-table distinct
// reduced_fingerprints and sum per metric (a metric only lands in the table matching its type).
sb.JoinWithOption(
sqlbuilder.LeftJoin,
"(SELECT metric_name, sum(cnt) AS cnt FROM ("+
"SELECT metric_name, uniq(reduced_fingerprint) AS cnt FROM "+reducedLast+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+
"(SELECT metric_name, sum(cnt) AS cnt, sum(samples) AS samples FROM ("+
"SELECT metric_name, uniq(reduced_fingerprint) AS cnt, uniq(reduced_fingerprint, unix_milli) AS samples FROM "+reducedLast+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+
" UNION ALL "+
"SELECT metric_name, uniq(reduced_fingerprint) AS cnt FROM "+reducedSum+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+
"SELECT metric_name, uniq(reduced_fingerprint) AS cnt, uniq(reduced_fingerprint, unix_milli) AS samples FROM "+reducedSum+" WHERE has("+sb.Var(metricNames)+", metric_name) AND unix_milli >= "+sb.Var(startMs)+" AND unix_milli < "+sb.Var(endMs)+" AND "+strictEffectiveFrom(sb, metricNames, effectiveFrom)+" GROUP BY metric_name"+
") GROUP BY metric_name) AS d",
"base.metric_name = d.metric_name",
)
@@ -347,74 +344,72 @@ func (c *clickhouse) RankByVolume(ctx context.Context, metricNames []string, eff
return out, rows.Err()
}
func (c *clickhouse) SampleVolume(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (uint64, uint64, error) {
func (c *clickhouse) SampleVolumeByMetric(ctx context.Context, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]volumeRow, error) {
if len(metricNames) == 0 {
return 0, 0, nil
return map[string]volumeRow{}, nil
}
ctx = c.withThreads(ctx)
ingested, err := c.countRawSamples(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, metricNames, effectiveFrom, startMs, endMs)
ingested, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4BufferTableName, "count()", metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return 0, 0, err
return nil, err
}
last, err := c.countReducedSamples(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, metricNames, effectiveFrom, startMs, endMs)
last, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedLastTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return 0, 0, err
return nil, err
}
sum, err := c.countReducedSamples(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, metricNames, effectiveFrom, startMs, endMs)
sum, err := c.countSamplesByMetric(ctx, telemetrymetrics.DBName+"."+telemetrymetrics.SamplesV4ReducedSumTableName, "uniq(reduced_fingerprint, unix_milli)", metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return 0, 0, err
return nil, err
}
return ingested, min(last+sum, ingested), nil
out := make(map[string]volumeRow, len(metricNames))
for _, name := range metricNames {
out[name] = volumeRow{MetricName: name, Ingested: ingested[name], Reduced: last[name] + sum[name]}
}
return out, nil
}
func (c *clickhouse) countRawSamples(ctx context.Context, table string, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (uint64, error) {
func (c *clickhouse) countSamplesByMetric(ctx context.Context, table, countExpr string, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (map[string]uint64, error) {
names := make([]any, len(metricNames))
for i, name := range metricNames {
names[i] = name
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select("count()")
sb.Select("metric_name", countExpr)
sb.From(table)
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
conds := []string{
sb.In("metric_name", names...),
sb.GE("unix_milli", startMs),
sb.LT("unix_milli", endMs),
}
if len(effectiveFrom) > 0 {
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
}
sb.Where(conds...)
sb.GroupBy("metric_name")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
var count uint64
if err := c.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...).Scan(&count); err != nil {
return 0, errors.WrapInternalf(err, errors.CodeInternal, "failed to count ingested samples")
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to count samples")
}
return count, nil
}
defer rows.Close()
func (c *clickhouse) countReducedSamples(ctx context.Context, table string, metricNames []string, effectiveFrom map[string]int64, startMs, endMs int64) (uint64, error) {
names := make([]any, len(metricNames))
for i, name := range metricNames {
names[i] = name
out := make(map[string]uint64, len(metricNames))
for rows.Next() {
var (
metricName string
count uint64
)
if err := rows.Scan(&metricName, &count); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan series count")
}
out[metricName] = count
}
sb := sqlbuilder.NewSelectBuilder()
// Reduced tables key the series on reduced_fingerprint (not fingerprint); dedupe ReplacingMergeTree recomputes.
sb.Select("uniq(reduced_fingerprint, unix_milli)")
sb.From(table)
conds := []string{sb.In("metric_name", names...), sb.GE("unix_milli", startMs), sb.LT("unix_milli", endMs)}
if len(effectiveFrom) > 0 {
conds = append(conds, strictEffectiveFrom(sb, metricNames, effectiveFrom))
}
sb.Where(conds...)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
var count uint64
if err := c.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...).Scan(&count); err != nil {
return 0, errors.WrapInternalf(err, errors.CodeInternal, "failed to count reduced samples")
}
return count, nil
return out, rows.Err()
}
// SeriesTimeseries returns ingested vs reduced series per 60s bucket from the samples tables, gated

View File

@@ -28,8 +28,13 @@ import (
const (
// effectiveFromMargin delays effective_from so the collector picks up the synced rule before it
// goes live; it must be >= the collector's rule-refresh interval (see signoz-otel-collector#839).
effectiveFromMargin = 5 * time.Minute
// goes live; it must be >= the collector's rule-refresh interval (~2m worst case,
// see signoz-otel-collector#839).
effectiveFromMargin = 2 * time.Minute
// uiActivationDelay keeps a rule shown as "pending" in the UI for a while after it goes live to
// the collector, so the user doesn't see "active" before reduced data is actually flowing. The
// user-facing pending window is effectiveFromMargin + uiActivationDelay (~5m).
uiActivationDelay = 3 * time.Minute
defaultPreviewLookback = 24 * time.Hour
pricePerMillionSamplesUSD = 0.1
@@ -107,10 +112,14 @@ func (m *module) listSortedByColumn(ctx context.Context, orgID valuer.UUID, para
if err != nil {
return nil, err
}
sampleVolumes, err := m.ch.SampleVolumeByMetric(ctx, metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
rules := make([]metricreductionruletypes.GettableReductionRule, 0, len(domainRules))
for _, rule := range domainRules {
rules = append(rules, withVolume(toGettableReductionRule(rule), volumes[rule.MetricName]))
rules = append(rules, withVolume(toGettableReductionRule(rule), volumes[rule.MetricName], sampleVolumes[rule.MetricName]))
}
return &metricreductionruletypes.GettableReductionRules{Rules: rules, Total: total}, nil
@@ -139,13 +148,22 @@ func (m *module) listSortedByVolume(ctx context.Context, orgID valuer.UUID, para
return nil, err
}
pageMetricNames := make([]string, 0, len(ranked))
for _, row := range ranked {
pageMetricNames = append(pageMetricNames, row.MetricName)
}
sampleVolumes, err := m.ch.SampleVolumeByMetric(ctx, pageMetricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
rules := make([]metricreductionruletypes.GettableReductionRule, 0, len(ranked))
for _, row := range ranked {
rule, ok := ruleByMetric[row.MetricName]
if !ok {
continue
}
rules = append(rules, withVolume(toGettableReductionRule(rule), row))
rules = append(rules, withVolume(toGettableReductionRule(rule), row, sampleVolumes[row.MetricName]))
}
return &metricreductionruletypes.GettableReductionRules{Rules: rules, Total: total}, nil
@@ -291,7 +309,7 @@ func (m *module) Stats(ctx context.Context, orgID valuer.UUID) (*metricreduction
startMs := now.Add(-defaultPreviewLookback).UnixMilli()
endMs := now.UnixMilli()
allRules, total, err := m.store.List(ctx, orgID, &metricreductionruletypes.ListReductionRulesParams{})
rules, total, err := m.store.List(ctx, orgID, &metricreductionruletypes.ListReductionRulesParams{})
if err != nil {
return nil, err
}
@@ -299,9 +317,9 @@ func (m *module) Stats(ctx context.Context, orgID valuer.UUID) (*metricreduction
return &metricreductionruletypes.GettableReductionRuleStats{}, nil
}
metricNames := make([]string, len(allRules))
effectiveFrom := make(map[string]int64, len(allRules))
for i, rule := range allRules {
metricNames := make([]string, len(rules))
effectiveFrom := make(map[string]int64, len(rules))
for i, rule := range rules {
metricNames[i] = rule.MetricName
effectiveFrom[rule.MetricName] = rule.EffectiveFrom.UnixMilli()
}
@@ -311,27 +329,27 @@ func (m *module) Stats(ctx context.Context, orgID valuer.UUID) (*metricreduction
return nil, err
}
var ingestedSeries, retainedSeries uint64
reducedMetricNames := make([]string, 0, len(volumes))
reducedEffectiveFrom := make(map[string]int64, len(volumes))
for name, volume := range volumes {
for _, volume := range volumes {
ingestedSeries += volume.Ingested
retained := effectiveRetained(volume.Ingested, volume.Reduced)
retainedSeries += retained
if retained < volume.Ingested {
reducedMetricNames = append(reducedMetricNames, name)
reducedEffectiveFrom[name] = effectiveFrom[name]
}
retainedSeries += effectiveRetained(volume.Ingested, volume.Reduced)
}
ingestedSamples, reducedSamples, err := m.ch.SampleVolume(ctx, reducedMetricNames, reducedEffectiveFrom, startMs, endMs)
sampleVolumes, err := m.ch.SampleVolumeByMetric(ctx, metricNames, effectiveFrom, startMs, endMs)
if err != nil {
return nil, err
}
var ingestedSamples, retainedSamples uint64
for _, sv := range sampleVolumes {
ingestedSamples += sv.Ingested
retainedSamples += effectiveRetained(sv.Ingested, sv.Reduced)
}
return &metricreductionruletypes.GettableReductionRuleStats{
IngestedSeries: ingestedSeries,
RetainedSeries: retainedSeries,
EstimatedMonthlySavingsUsd: monthlySavingsUSD(ingestedSamples, reducedSamples, startMs, endMs),
IngestedSamples: ingestedSamples,
RetainedSamples: retainedSamples,
EstimatedMonthlySavingsUsd: monthlySavingsUSD(ingestedSamples, retainedSamples, startMs, endMs),
}, nil
}
@@ -414,7 +432,7 @@ func buildVolumeTimeseries(points []volumePoint) *querybuildertypesv5.QueryRange
}
func (m *module) validateMetricForReduction(ctx context.Context, orgID valuer.UUID, metricName string) error {
lastSeen, err := m.metadataStore.FetchLastSeenInfoMulti(ctx, metricName)
lastSeen, err := m.metadataStore.FetchLastSeenInfoMulti(ctx, orgID, metricName)
if err != nil {
return err
}
@@ -482,7 +500,7 @@ func toGettableReductionRule(rule *metricreductionruletypes.ReductionRule) metri
MatchType: rule.MatchType,
Labels: rule.Labels,
EffectiveFrom: rule.EffectiveFrom,
Active: !rule.EffectiveFrom.After(time.Now()),
Active: !rule.EffectiveFrom.Add(uiActivationDelay).After(time.Now()),
}
}
@@ -493,12 +511,11 @@ func effectiveRetained(ingested, reduced uint64) uint64 {
return reduced
}
func withVolume(rule metricreductionruletypes.GettableReductionRule, volume volumeRow) metricreductionruletypes.GettableReductionRule {
rule.IngestedSeries = volume.Ingested
rule.RetainedSeries = effectiveRetained(volume.Ingested, volume.Reduced)
if volume.Ingested > 0 {
rule.ReductionPercent = (1 - float64(rule.RetainedSeries)/float64(volume.Ingested)) * 100
}
func withVolume(rule metricreductionruletypes.GettableReductionRule, series volumeRow, samples volumeRow) metricreductionruletypes.GettableReductionRule {
rule.IngestedSeries = series.Ingested
rule.RetainedSeries = effectiveRetained(series.Ingested, series.Reduced)
rule.IngestedSamples = samples.Ingested
rule.RetainedSamples = effectiveRetained(samples.Ingested, samples.Reduced)
return rule
}

View File

@@ -6853,6 +6853,11 @@ export interface MetricreductionruletypesGettableReductionRuleDTO {
* @type string
*/
id: string;
/**
* @type integer
* @minimum 0
*/
ingestedSamples: number;
/**
* @type integer
* @minimum 0
@@ -6868,10 +6873,10 @@ export interface MetricreductionruletypesGettableReductionRuleDTO {
*/
metricName: string;
/**
* @type number
* @format double
* @type integer
* @minimum 0
*/
reductionPercent: number;
retainedSamples: number;
/**
* @type integer
* @minimum 0
@@ -6930,11 +6935,21 @@ export interface MetricreductionruletypesGettableReductionRuleStatsDTO {
* @format double
*/
estimatedMonthlySavingsUsd: number;
/**
* @type integer
* @minimum 0
*/
ingestedSamples: number;
/**
* @type integer
* @minimum 0
*/
ingestedSeries: number;
/**
* @type integer
* @minimum 0
*/
retainedSamples: number;
/**
* @type integer
* @minimum 0
@@ -6990,7 +7005,6 @@ export enum MetricreductionruletypesReductionRuleOrderByDTO {
metric = 'metric',
ingested_volume = 'ingested_volume',
reduced_volume = 'reduced_volume',
reduction = 'reduction',
last_updated = 'last_updated',
}
export interface MetricreductionruletypesUpdatableReductionRuleDTO {

View File

@@ -1,4 +1,5 @@
import { Gauge } from '@signozhq/icons';
import { Tooltip } from 'antd';
import { MetricreductionruletypesGettableReductionRuleDTO } from 'api/generated/services/sigNoz.schemas';
import { Badge } from '@signozhq/ui/badge';
@@ -8,16 +9,26 @@ interface VolumeControlBadgeProps {
}
function VolumeControlBadge({ rule }: VolumeControlBadgeProps): JSX.Element {
return (
const badge = (
<Badge
data-testid="vc-badge-active"
variant="outline"
color={!rule.active ? 'success' : 'warning'}
color={rule.active ? 'success' : 'warning'}
>
<Gauge size={12} />
{!rule.active ? 'Active' : 'Pending'}
{rule.active ? 'Active' : 'Pending'}
</Badge>
);
if (rule.active) {
return badge;
}
return (
<Tooltip title="Takes about 5 minutes to take effect">
<span>{badge}</span>
</Tooltip>
);
}
export default VolumeControlBadge;

View File

@@ -15,3 +15,11 @@
.chartBody {
height: 340px;
}
.chartStatus {
display: flex;
align-items: center;
justify-content: center;
width: 100%;
height: 100%;
}

View File

@@ -1,5 +1,6 @@
import { Color } from '@signozhq/design-tokens';
import { Typography } from '@signozhq/ui/typography';
import { Spin } from 'antd';
import { useGetMetricReductionRuleTimeseries } from 'api/generated/services/metrics';
import { PANEL_TYPES } from 'constants/queryBuilder';
import BarChart from 'container/DashboardContainer/visualization/charts/BarChart/BarChart';
@@ -25,7 +26,9 @@ interface VolumeControlChartProps {
}
function VolumeControlChart({ enabled }: VolumeControlChartProps): JSX.Element {
const { data } = useGetMetricReductionRuleTimeseries({ query: { enabled } });
const { data, isLoading, isError } = useGetMetricReductionRuleTimeseries({
query: { enabled },
});
const isDarkMode = useIsDarkMode();
const { timezone } = useTimezone();
@@ -69,7 +72,25 @@ function VolumeControlChart({ enabled }: VolumeControlChartProps): JSX.Element {
Series volume over time · ingested vs retained
</Typography.Text>
<div className={styles.chartBody} ref={graphRef}>
{dimensions.width > 0 && (
{isLoading && (
<div
className={styles.chartStatus}
data-testid="volume-control-chart-loading"
>
<Spin size="small" />
</div>
)}
{!isLoading && isError && (
<div
className={styles.chartStatus}
data-testid="volume-control-chart-error"
>
<Typography.Text size="small" color="danger">
Failed to load chart
</Typography.Text>
</div>
)}
{!isLoading && !isError && dimensions.width > 0 && (
<BarChart
config={config}
data={chartData}

View File

@@ -17,11 +17,27 @@
gap: 5px;
}
.meterLabelRow {
display: flex;
align-items: center;
gap: 4px;
}
.meterLabel {
text-transform: uppercase;
letter-spacing: 0.04em;
}
.meterInfo {
flex-shrink: 0;
color: var(--bg-vanilla-400, #c0c1c3);
line-height: 1;
cursor: help;
}
.meterValue {
display: flex;
align-items: baseline;
gap: 6px;
font-family: 'Geist Mono', monospace;
}

View File

@@ -1,6 +1,8 @@
import { Info } from '@signozhq/icons';
import { Typography } from '@signozhq/ui/typography';
import { Spin } from 'antd';
import { Spin, Tooltip } from 'antd';
import { MetricreductionruletypesGettableReductionRulePreviewDTO } from 'api/generated/services/sigNoz.schemas';
import { popupContainer } from 'utils/selectPopupContainer';
import { formatCompact } from '../../../configUtils';
import { RuleMode } from '../../../types';
@@ -27,6 +29,7 @@ function ImpactPanel({
);
}
const full = preview?.ingestedSeries ?? 0;
const current = preview?.currentRetainedSeries ?? 0;
const proposed = preview?.retainedSeries ?? 0;
const deltaPct = current > 0 ? (1 - proposed / current) * 100 : 0;
@@ -40,31 +43,59 @@ function ImpactPanel({
{!isLoading && preview && (
<div className={styles.meterGrid}>
<div className={styles.meter}>
<Typography.Text size="xs" color="muted" className={styles.meterLabel}>
Current series
<div className={styles.meterLabelRow}>
<Typography.Text size="xs" color="muted" className={styles.meterLabel}>
Full series
</Typography.Text>
<Tooltip
title="Total number of series for this metric before any reduction."
getPopupContainer={popupContainer}
>
<Info size={12} className={styles.meterInfo} />
</Tooltip>
</div>
<Typography.Text size="2xl" className={styles.meterValue}>
{formatCompact(full)}
</Typography.Text>
</div>
<div className={styles.meter}>
<div className={styles.meterLabelRow}>
<Typography.Text size="xs" color="muted" className={styles.meterLabel}>
Current retained
</Typography.Text>
<Tooltip
title="Series kept today under the metric's existing rule, or all of them if it has no rule yet."
getPopupContainer={popupContainer}
>
<Info size={12} className={styles.meterInfo} />
</Tooltip>
</div>
<Typography.Text size="2xl" className={styles.meterValue}>
{formatCompact(current)}
</Typography.Text>
</div>
<div className={styles.meter}>
<Typography.Text size="xs" color="muted" className={styles.meterLabel}>
Proposed series
</Typography.Text>
<div className={styles.meterLabelRow}>
<Typography.Text size="xs" color="muted" className={styles.meterLabel}>
Potential retained
</Typography.Text>
<Tooltip
title="Series that would be kept if you save this rule, with the reduction vs what's retained today."
getPopupContainer={popupContainer}
>
<Info size={12} className={styles.meterInfo} />
</Tooltip>
</div>
<Typography.Text size="2xl" className={styles.meterValue}>
{formatCompact(proposed)}
</Typography.Text>
</div>
<div className={styles.meter}>
<Typography.Text size="xs" color="muted" className={styles.meterLabel}>
Reduction
</Typography.Text>
<Typography.Text
size="2xl"
color={deltaPct >= 0 ? 'success' : undefined}
className={styles.meterValue}
>
{reductionLabel}
{deltaPct !== 0 && (
<Typography.Text
size="small"
color={deltaPct >= 0 ? 'success' : undefined}
>
{reductionLabel}
</Typography.Text>
)}
</Typography.Text>
</div>
</div>

View File

@@ -18,12 +18,12 @@ const MODE_OPTIONS: ModeOption[] = [
},
{
mode: 'include',
title: 'Include attributes',
title: 'Include',
description: 'Allowlist: only the selected attributes stay queryable.',
},
{
mode: 'exclude',
title: 'Exclude attributes',
title: 'Exclude',
description: 'Blocklist: the selected attributes are aggregated away.',
},
];

View File

@@ -42,6 +42,10 @@ function VolumeControlConfigDrawer({
const footer = (
<div className={styles.footer}>
<Typography.Text size="small" color="muted">
Changes take effect about 5 minutes after saving.
</Typography.Text>
<div className={styles.footerSpacer} />
<Button
variant="outlined"
color="secondary"
@@ -50,7 +54,6 @@ function VolumeControlConfigDrawer({
>
Cancel
</Button>
<div className={styles.footerSpacer} />
{hasExistingRule && (
<Button
variant="ghost"

View File

@@ -8,8 +8,8 @@ function PendingActivationBanner(): JSX.Element {
<div className={styles.banner} data-testid="volume-control-pending-banner">
<Info size={13} />
<Typography.Text size="sm" color="muted">
This metric&apos;s configuration was recently updated. Volume changes will
take effect within a few minutes.
This metric&apos;s configuration was recently updated. Volume changes take
effect within about 5 minutes.
</Typography.Text>
</div>
);

View File

@@ -22,7 +22,7 @@ function VolumeControlSection({
useVolumeControlFeatureGate();
const [isConfigOpen, setIsConfigOpen] = useState(false);
const { data, isLoading, error } = useListMetricReductionRules(
const { data, isLoading, isError } = useListMetricReductionRules(
{ metricName },
{
query: {
@@ -37,7 +37,7 @@ function VolumeControlSection({
}
const rule = data?.data.rules?.[0];
const hasRule = !!rule && !error;
const hasRule = !!rule && !isError;
const openConfig = (): void => setIsConfigOpen(true);
const closeConfig = (): void => setIsConfigOpen(false);
@@ -53,6 +53,16 @@ function VolumeControlSection({
{isLoading && <Skeleton active title={false} paragraph={{ rows: 2 }} />}
{!isLoading && isError && (
<Typography.Text
size="small"
color="danger"
data-testid="volume-control-section-error"
>
Failed to load volume control. Please try again.
</Typography.Text>
)}
{!isLoading && hasRule && rule && !rule.active && (
<PendingActivationBanner />
)}
@@ -65,7 +75,7 @@ function VolumeControlSection({
/>
)}
{!isLoading && !hasRule && (
{!isLoading && !isError && !hasRule && (
<NoRuleEmptyState canManage={canManageVolumeControl} onSetup={openConfig} />
)}

View File

@@ -21,11 +21,24 @@
background: var(--callout-success-background);
}
.statCardLabelRow {
display: flex;
align-items: center;
gap: 4px;
}
.statCardLabel {
text-transform: uppercase;
letter-spacing: 0.05em;
}
.statCardInfo {
flex-shrink: 0;
color: var(--bg-vanilla-400, #c0c1c3);
line-height: 1;
cursor: help;
}
.statCardValue {
display: flex;
align-items: baseline;

View File

@@ -1,4 +1,6 @@
import { Info } from '@signozhq/icons';
import { Typography } from '@signozhq/ui/typography';
import { Skeleton, Tooltip } from 'antd';
import cx from 'classnames';
import { formatCompact, formatUsd } from '../../../configUtils';
@@ -8,12 +10,17 @@ interface VolumeControlStatsProps {
activeRules: number;
ingestedSeries: number;
retainedSeries: number;
ingestedSamples: number;
retainedSamples: number;
estimatedMonthlySavingsUsd: number;
isLoading?: boolean;
isError?: boolean;
}
interface StatItem {
label: string;
value: string;
tooltip: string;
delta?: string;
unit?: string;
highlighted?: boolean;
@@ -24,20 +31,53 @@ function VolumeControlStats({
activeRules,
ingestedSeries,
retainedSeries,
ingestedSamples,
retainedSamples,
estimatedMonthlySavingsUsd,
isLoading = false,
isError = false,
}: VolumeControlStatsProps): JSX.Element {
const overallReduction =
ingestedSeries > 0
? Math.round((1 - retainedSeries / ingestedSeries) * 100)
: 0;
const sampleReduction =
ingestedSamples > 0
? Math.round((1 - retainedSamples / ingestedSamples) * 100)
: 0;
const items: StatItem[] = [
{ label: 'Active rules', value: String(activeRules) },
{ label: 'Ingested series', value: formatCompact(ingestedSeries) },
{
label: 'Configured rules',
value: String(activeRules),
tooltip: 'Volume-control rules currently configured for this workspace.',
},
{
label: 'Ingested series',
value: formatCompact(ingestedSeries),
tooltip:
'Distinct time series observed across active rules in the last 24 hours, from when each rule took effect, before any reduction.',
},
{
label: 'Retained series',
value: formatCompact(retainedSeries),
delta: overallReduction > 0 ? `${overallReduction}%` : undefined,
tooltip:
'Distinct time series retained after reduction across active rules over the same window. Lower means more reduction.',
},
{
label: 'Ingested samples',
value: formatCompact(ingestedSamples),
tooltip:
'Sample data points observed across active rules in the last 24 hours, from when each rule took effect, before any reduction.',
},
{
label: 'Retained samples',
value: formatCompact(retainedSamples),
delta: sampleReduction > 0 ? `${sampleReduction}%` : undefined,
tooltip:
'Sample data points retained after reduction across active rules over the same window. Samples reduction can exceed series reduction since series do not all contribute the same sample volume.',
},
{
label: 'Est. monthly savings',
@@ -45,6 +85,8 @@ function VolumeControlStats({
unit: '/mo',
highlighted: true,
valueGood: true,
tooltip:
'Estimated from the reduced sample volume assuming 1-month standard retention.',
},
];
@@ -57,28 +99,48 @@ function VolumeControlStats({
[styles.statCardHighlighted]: item.highlighted,
})}
>
<Typography.Text size="sm" color="muted" className={styles.statCardLabel}>
{item.label}
</Typography.Text>
<Typography.Text
as="div"
size="large"
weight="semibold"
color={item.valueGood ? 'success' : undefined}
className={styles.statCardValue}
>
{item.value}
{item.delta && (
<Typography.Text size="small" weight="semibold" color="success">
{item.delta}
</Typography.Text>
)}
{item.unit && (
<Typography.Text size="small" weight="medium" color="muted">
{item.unit}
</Typography.Text>
)}
</Typography.Text>
<div className={styles.statCardLabelRow}>
<Typography.Text size="sm" color="muted" className={styles.statCardLabel}>
{item.label}
</Typography.Text>
<Tooltip title={item.tooltip}>
<Info size={12} className={styles.statCardInfo} />
</Tooltip>
</div>
{isLoading && (
<Skeleton.Button active size="small" className={styles.statCardValue} />
)}
{!isLoading && isError && (
<Typography.Text
as="div"
size="small"
color="danger"
className={styles.statCardValue}
>
Failed to load
</Typography.Text>
)}
{!isLoading && !isError && (
<Typography.Text
as="div"
size="large"
weight="semibold"
color={item.valueGood ? 'success' : undefined}
className={styles.statCardValue}
>
{item.value}
{item.delta && (
<Typography.Text size="small" weight="semibold" color="success">
{item.delta}
</Typography.Text>
)}
{item.unit && (
<Typography.Text size="small" weight="medium" color="muted">
{item.unit}
</Typography.Text>
)}
</Typography.Text>
)}
</div>
))}
</div>

View File

@@ -13,8 +13,10 @@
font-family: 'Geist Mono', monospace;
}
.reductionCell {
font-family: 'Geist Mono', monospace;
.volumeCell {
display: flex;
flex-direction: column;
gap: 2px;
}
.empty {

View File

@@ -36,7 +36,7 @@ type VolumeControlTableParams = Required<
>;
const DEFAULT_PARAMS: VolumeControlTableParams = {
orderBy: OrderBy.reduction,
orderBy: OrderBy.ingested_volume,
order: SortOrder.desc,
search: '',
offset: 0,
@@ -60,11 +60,20 @@ function VolumeControlTab(): JSX.Element {
);
}, [debouncedSearch]);
const { data, isLoading } = useListMetricReductionRules(params, {
const {
data,
isLoading,
isError: isListError,
} = useListMetricReductionRules(params, {
query: { enabled: isVolumeControlEnabled },
});
const { data: statsData } = useGetMetricReductionRuleStats({
const {
data: statsData,
isLoading: isStatsLoading,
isFetching: isStatsFetching,
isError: isStatsError,
} = useGetMetricReductionRuleStats({
query: { enabled: isVolumeControlEnabled },
});
const stats = statsData?.data;
@@ -111,7 +120,7 @@ function VolumeControlTab(): JSX.Element {
{
title: 'MODE',
key: 'mode',
width: 160,
width: 110,
render: (
_value: unknown,
rule: MetricreductionruletypesGettableReductionRuleDTO,
@@ -147,9 +156,17 @@ function VolumeControlTab(): JSX.Element {
_value: unknown,
rule: MetricreductionruletypesGettableReductionRuleDTO,
): JSX.Element => (
<Typography.Text size="small" color="muted">
{formatCompact(rule.ingestedSeries)}
</Typography.Text>
<div className={styles.volumeCell}>
<Typography.Text size="small">
{formatCompact(rule.ingestedSeries)}{' '}
<Typography.Text size="small" color="muted">
series
</Typography.Text>
</Typography.Text>
<Typography.Text size="small" color="muted">
{formatCompact(rule.ingestedSamples)} samples
</Typography.Text>
</div>
),
},
{
@@ -162,22 +179,35 @@ function VolumeControlTab(): JSX.Element {
_value: unknown,
rule: MetricreductionruletypesGettableReductionRuleDTO,
): JSX.Element => (
<Typography.Text size="small">
{formatCompact(rule.retainedSeries)}
</Typography.Text>
<div className={styles.volumeCell}>
<Typography.Text size="small">
{formatCompact(rule.retainedSeries)}{' '}
<Typography.Text size="small" color="muted">
series
</Typography.Text>
</Typography.Text>
<Typography.Text size="small" color="muted">
{formatCompact(rule.retainedSamples)} samples
</Typography.Text>
</div>
),
},
{
title: 'CHANGE',
key: OrderBy.reduction,
width: 110,
sorter: true,
sortOrder: sortOrderFor(OrderBy.reduction),
width: 140,
render: (
_value: unknown,
rule: MetricreductionruletypesGettableReductionRuleDTO,
): JSX.Element => {
if (rule.reductionPercent <= 0) {
const seriesReduction =
rule.ingestedSeries > 0
? (1 - rule.retainedSeries / rule.ingestedSeries) * 100
: 0;
const samplesReduction =
rule.ingestedSamples > 0
? (1 - rule.retainedSamples / rule.ingestedSamples) * 100
: 0;
if (seriesReduction <= 0 && samplesReduction <= 0) {
return (
<Typography.Text size="small" color="muted">
@@ -185,14 +215,18 @@ function VolumeControlTab(): JSX.Element {
);
}
return (
<Typography.Text
size="small"
weight="semibold"
color="success"
className={styles.reductionCell}
>
{Math.round(rule.reductionPercent)}%
</Typography.Text>
<div className={styles.volumeCell}>
<Typography.Text size="small" weight="semibold" color="success">
{seriesReduction > 0 ? `${Math.round(seriesReduction)}%` : '0%'}{' '}
<Typography.Text size="small" color="muted">
series
</Typography.Text>
</Typography.Text>
<Typography.Text size="small" color="muted">
{samplesReduction > 0 ? `${Math.round(samplesReduction)}%` : '0%'}{' '}
samples
</Typography.Text>
</div>
);
},
},
@@ -273,7 +307,11 @@ function VolumeControlTab(): JSX.Element {
activeRules={total}
ingestedSeries={stats?.ingestedSeries ?? 0}
retainedSeries={stats?.retainedSeries ?? 0}
ingestedSamples={stats?.ingestedSamples ?? 0}
retainedSamples={stats?.retainedSamples ?? 0}
estimatedMonthlySavingsUsd={stats?.estimatedMonthlySavingsUsd ?? 0}
isLoading={isStatsLoading || isStatsFetching}
isError={isStatsError}
/>
<VolumeControlChart enabled={isVolumeControlEnabled} />
@@ -293,7 +331,13 @@ function VolumeControlTab(): JSX.Element {
showSizeChanger: false,
}}
locale={{
emptyText: (
emptyText: isListError ? (
<div className={styles.empty} data-testid="volume-control-tab-error">
<Typography.Text color="danger">
Failed to load volume control rules. Please try again.
</Typography.Text>
</div>
) : (
<div className={styles.empty} data-testid="volume-control-tab-empty">
<Typography.Text color="muted">
No volume control rules yet. Open a metric and set one up to start

View File

@@ -49,6 +49,7 @@ export interface UseVolumeControlConfigResult {
const PREVIEW_DEBOUNCE_MS = 400;
const SAVE_ERROR_MESSAGE = 'Failed to save volume control rule';
const REMOVE_ERROR_MESSAGE = 'Failed to remove volume control rule';
const PREVIEW_ERROR_MESSAGE = 'Failed to preview volume control rule';
export function useVolumeControlConfig({
metricName,
@@ -95,11 +96,25 @@ export function useVolumeControlConfig({
const timer = setTimeout(() => {
previewMutate(
{ data: { metricName, matchType: matchTypeForMode(mode), labels } },
{ onSettled: () => setIsPreviewPending(false) },
{
onError: (error) =>
notifications.error({
message: error.response?.data?.error?.message ?? PREVIEW_ERROR_MESSAGE,
}),
onSettled: () => setIsPreviewPending(false),
},
);
}, PREVIEW_DEBOUNCE_MS);
return (): void => clearTimeout(timer);
}, [open, mode, labels, metricName, previewMutate, previewReset]);
}, [
open,
mode,
labels,
metricName,
previewMutate,
previewReset,
notifications,
]);
const createMutation = useCreateMetricReductionRule();
const updateMutation = useUpdateMetricReductionRuleByID();
@@ -142,7 +157,10 @@ export function useVolumeControlConfig({
}
const onSuccess = (): void => {
notifications.success({ message: 'Volume control rule saved' });
notifications.success({
message:
'Volume control rule saved. It takes about 5 minutes to take effect.',
});
invalidate();
onClose();
};

View File

@@ -9,7 +9,7 @@ export function isKeepMode(
export function getMatchTypeLabel(
matchType: MetricreductionruletypesMatchTypeDTO,
): string {
return isKeepMode(matchType) ? 'Include attributes' : 'Exclude attributes';
return isKeepMode(matchType) ? 'Include' : 'Exclude';
}
export function getLabelVerb(

View File

@@ -1031,17 +1031,76 @@ func (m *module) fetchMetricsStatsWithSamples(
samplesSB.GroupBy("metric_name")
ctes = append(ctes, sqlbuilder.CTEQuery("__sample_counts").As(samplesSB))
// reduced tables
reducedTsSB := sqlbuilder.NewSelectBuilder()
reducedTsSB.Select(
"metric_name",
"uniq(fingerprint) AS timeseries",
)
// the data lands either of the _reduced_last/_reduced_sum tables
reducedLastSB := sqlbuilder.NewSelectBuilder()
reducedLastSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedLastSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedLastTableName))
reducedLastSB.Where(reducedLastSB.Between("unix_milli", req.Start, req.End))
reducedLastSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedSumSB := sqlbuilder.NewSelectBuilder()
reducedSumSB.Select("metric_name", "uniq(reduced_fingerprint, unix_milli) AS cnt")
reducedSumSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.SamplesV4ReducedSumTableName))
reducedSumSB.Where(reducedSumSB.Between("unix_milli", req.Start, req.End))
reducedSumSB.Where("NOT startsWith(metric_name, 'signoz')")
if filterWhereClause == nil {
reducedTsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedTsSB.Where(reducedTsSB.Between("unix_milli", start, end))
reducedTsSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedTsSB.Where(reducedTsSB.E("__normalized", normalized))
reducedTsSB.GroupBy("metric_name")
} else {
reducedFpSB := sqlbuilder.NewSelectBuilder()
reducedFpSB.Select("metric_name", "fingerprint")
reducedFpSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, telemetrymetrics.TimeseriesV4ReducedTableName))
reducedFpSB.Where(reducedFpSB.Between("unix_milli", start, end))
reducedFpSB.Where("NOT startsWith(metric_name, 'signoz')")
reducedFpSB.Where(reducedFpSB.E("__normalized", normalized))
reducedFpSB.AddWhereClause(sqlbuilder.CopyWhereClause(filterWhereClause))
reducedFpSB.GroupBy("metric_name", "fingerprint")
ctes = append(ctes, sqlbuilder.CTEQuery("__reduced_filtered_fingerprints").As(reducedFpSB))
reducedTsSB.From("__reduced_filtered_fingerprints")
reducedTsSB.GroupBy("metric_name")
reducedLastSB.Where("reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints)")
reducedSumSB.Where("reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints)")
}
reducedLastSB.GroupBy("metric_name")
reducedSumSB.GroupBy("metric_name")
reducedSamplesSB := sqlbuilder.NewSelectBuilder()
reducedSamplesSB.Select("metric_name", "sum(cnt) AS samples")
reducedSamplesSB.From(reducedSamplesSB.BuilderAs(sqlbuilder.UnionAll(reducedLastSB, reducedSumSB), "reduced_samples"))
reducedSamplesSB.GroupBy("metric_name")
ctes = append(ctes,
sqlbuilder.CTEQuery("__reduced_time_series_counts").As(reducedTsSB),
sqlbuilder.CTEQuery("__reduced_sample_counts").As(reducedSamplesSB),
)
cteBuilder := sqlbuilder.With(ctes...)
finalSB := cteBuilder.Select(
"COALESCE(ts.metric_name, s.metric_name) AS metric_name",
"COALESCE(ts.timeseries, 0) AS timeseries",
"COALESCE(s.samples, 0) AS samples",
"COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name",
"COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries",
"COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples",
"COUNT(*) OVER() AS total",
)
finalSB.From("__time_series_counts ts")
finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__sample_counts s", "ts.metric_name = s.metric_name")
finalSB.Where("(COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0)")
finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__reduced_time_series_counts rts", "ts.metric_name = rts.metric_name")
finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__sample_counts s", "COALESCE(ts.metric_name, rts.metric_name) = s.metric_name")
finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__reduced_sample_counts rs", "COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name")
finalSB.Where("(COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0)")
orderByColumn, orderDirection, err := getStatsOrderByColumn(req.OrderBy)
if err != nil {

View File

@@ -28,9 +28,9 @@ const (
// to stable table names that the expected SQL strings can hard-code).
testStartMillis int64 = 1700000000000
testEndMillis int64 = 1700003600000 // +1h
statsNoFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ?) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?"
statsOrderTimeseriesSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ?) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY timeseries ASC, metric_name ASC LIMIT ? OFFSET ?"
statsWithFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name) SELECT COALESCE(ts.metric_name, s.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __sample_counts s ON ts.metric_name = s.metric_name WHERE (COALESCE(ts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?"
statsNoFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ?) GROUP BY metric_name), __reduced_time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name) SELECT COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __reduced_time_series_counts rts ON ts.metric_name = rts.metric_name FULL OUTER JOIN __sample_counts s ON COALESCE(ts.metric_name, rts.metric_name) = s.metric_name FULL OUTER JOIN __reduced_sample_counts rs ON COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name WHERE (COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?"
statsOrderTimeseriesSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND metric_name IN (SELECT DISTINCT metric_name FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ?) GROUP BY metric_name), __reduced_time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name) SELECT COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __reduced_time_series_counts rts ON ts.metric_name = rts.metric_name FULL OUTER JOIN __sample_counts s ON COALESCE(ts.metric_name, rts.metric_name) = s.metric_name FULL OUTER JOIN __reduced_sample_counts rs ON COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name WHERE (COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0) ORDER BY timeseries ASC, metric_name ASC LIMIT ? OFFSET ?"
statsWithFilterSQL = "WITH __time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name), __filtered_fingerprints AS (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY fingerprint), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND fingerprint IN (SELECT fingerprint FROM __filtered_fingerprints) GROUP BY metric_name), __reduced_filtered_fingerprints AS (SELECT metric_name, fingerprint FROM signoz_metrics.distributed_time_series_v4_reduced WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name, fingerprint), __reduced_time_series_counts AS (SELECT metric_name, uniq(fingerprint) AS timeseries FROM __reduced_filtered_fingerprints GROUP BY metric_name), __reduced_sample_counts AS (SELECT metric_name, sum(cnt) AS samples FROM ((SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints) GROUP BY metric_name) UNION ALL (SELECT metric_name, uniq(reduced_fingerprint, unix_milli) AS cnt FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND reduced_fingerprint IN (SELECT fingerprint FROM __reduced_filtered_fingerprints) GROUP BY metric_name)) AS reduced_samples GROUP BY metric_name) SELECT COALESCE(ts.metric_name, rts.metric_name, s.metric_name, rs.metric_name) AS metric_name, COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) AS timeseries, COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) AS samples, COUNT(*) OVER() AS total FROM __time_series_counts ts FULL OUTER JOIN __reduced_time_series_counts rts ON ts.metric_name = rts.metric_name FULL OUTER JOIN __sample_counts s ON COALESCE(ts.metric_name, rts.metric_name) = s.metric_name FULL OUTER JOIN __reduced_sample_counts rs ON COALESCE(ts.metric_name, rts.metric_name, s.metric_name) = rs.metric_name WHERE (COALESCE(ts.timeseries, 0) + COALESCE(rts.timeseries, 0) > 0 OR COALESCE(s.samples, 0) + COALESCE(rs.samples, 0) > 0) ORDER BY samples DESC, metric_name ASC LIMIT ? OFFSET ?"
treemapTimeseriesNoFilterSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND __normalized = ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? GROUP BY metric_name) SELECT mt.metric_name, mt.total_value, CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage FROM __metric_totals mt JOIN __total_time_series tts ON 1=1 ORDER BY percentage DESC LIMIT ?"
treemapTimeseriesWithFilterSQL = "WITH __total_time_series AS (SELECT uniq(fingerprint) AS total_time_series FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND __normalized = ?), __metric_totals AS (SELECT metric_name, uniq(fingerprint) AS total_value FROM signoz_metrics.distributed_time_series_v4 WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND JSONExtractString(labels, 'host.name') = ? GROUP BY metric_name) SELECT mt.metric_name, mt.total_value, CASE WHEN tts.total_time_series = 0 THEN 0 ELSE (mt.total_value * 100.0 / tts.total_time_series) END AS percentage FROM __metric_totals mt JOIN __total_time_series tts ON 1=1 ORDER BY percentage DESC LIMIT ?"
treemapSamplesNoFilterSQL = "WITH __metric_candidates AS (SELECT metric_name FROM signoz_metrics.distributed_time_series_v4 WHERE NOT startsWith(metric_name, 'signoz') AND __normalized = ? AND unix_milli BETWEEN ? AND ? GROUP BY metric_name ORDER BY uniq(fingerprint) DESC LIMIT ?), __sample_counts AS (SELECT metric_name, count(*) AS samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ? AND metric_name GLOBAL IN (SELECT metric_name FROM __metric_candidates) GROUP BY metric_name), __total_samples AS (SELECT count(*) AS total_samples FROM signoz_metrics.distributed_samples_v4 WHERE unix_milli BETWEEN ? AND ?) SELECT mc.metric_name, COALESCE(sc.samples, 0) AS samples, CASE WHEN ts.total_samples = 0 THEN 0 ELSE (COALESCE(sc.samples, 0) * 100.0 / ts.total_samples) END AS percentage FROM __metric_candidates mc LEFT JOIN __sample_counts sc ON mc.metric_name = sc.metric_name JOIN __total_samples ts ON 1=1 ORDER BY percentage DESC LIMIT ?"
@@ -158,12 +158,12 @@ func TestGetStats(t *testing.T) {
noQuery bool // SQL never reaches clickhouse (validation/build error)
wantCode errors.Code
}{
{name: "NoFilter_FastPathSQL", expectSQL: statsNoFilterSQL, argCount: 10},
{name: "WhitespaceFilter_FastPathSQL", opts: []statsOpt{withStatsFilter(" ")}, expectSQL: statsNoFilterSQL, argCount: 10},
{name: "WithFilter_FingerprintSQL", opts: []statsOpt{withStatsFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: statsWithFilterSQL, argCount: 12},
{name: "OrderByTimeseriesAsc", opts: []statsOpt{withStatsOrderBy("timeseries", qbtypes.OrderDirectionAsc)}, expectSQL: statsOrderTimeseriesSQL, argCount: 10},
{name: "NoFilter_FastPathSQL", expectSQL: statsNoFilterSQL, argCount: 17},
{name: "WhitespaceFilter_FastPathSQL", opts: []statsOpt{withStatsFilter(" ")}, expectSQL: statsNoFilterSQL, argCount: 17},
{name: "WithFilter_FingerprintSQL", opts: []statsOpt{withStatsFilter("host.name = 'foo'")}, seedKey: "host.name", expectSQL: statsWithFilterSQL, argCount: 20},
{name: "OrderByTimeseriesAsc", opts: []statsOpt{withStatsOrderBy("timeseries", qbtypes.OrderDirectionAsc)}, expectSQL: statsOrderTimeseriesSQL, argCount: 17},
{name: "OrderByInvalid", opts: []statsOpt{withStatsOrderBy("nonsense", qbtypes.OrderDirectionAsc)}, noQuery: true, wantCode: errors.CodeInvalidInput},
{name: "QueryError", queryErr: assert.AnError, expectSQL: statsNoFilterSQL, argCount: 10, wantCode: errors.CodeInternal},
{name: "QueryError", queryErr: assert.AnError, expectSQL: statsNoFilterSQL, argCount: 17, wantCode: errors.CodeInternal},
{name: "InvalidRequest_Limit", opts: []statsOpt{withStatsLimit(0)}, noQuery: true, wantCode: errors.CodeInvalidInput},
}

View File

@@ -408,7 +408,7 @@ func (q *querier) resolveMetricMetadata(ctx context.Context, orgID valuer.UUID,
// Classify each missing metric: never-seen -> warning with empty result;
// seen-but-no-data-in-window -> dormant warning.
lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, externalMissingMetrics...)
lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, orgID, externalMissingMetrics...)
var nonExistentMetrics []string
var dormantMetrics []string
for _, name := range externalMissingMetrics {

View File

@@ -2525,12 +2525,33 @@ func (t *telemetryMetaStore) GetFirstSeenFromMetricMetadata(ctx context.Context,
return result, nil
}
func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, metricNames ...string) (map[string]int64, error) {
func (t *telemetryMetaStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) {
lastSeenInfo, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4TableName, metricNames)
if err != nil {
return nil, err
}
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID)) {
reducedLastSeen, err := t.fetchLastSeenInfoForTable(ctx, telemetrymetrics.TimeseriesV4ReducedTableName, metricNames)
if err != nil {
return nil, err
}
for metricName, ts := range reducedLastSeen {
if ts > lastSeenInfo[metricName] {
lastSeenInfo[metricName] = ts
}
}
}
return lastSeenInfo, nil
}
func (t *telemetryMetaStore) fetchLastSeenInfoForTable(ctx context.Context, tableName string, metricNames []string) (map[string]int64, error) {
sb := sqlbuilder.Select(
"metric_name",
"max(unix_milli)",
).
From(t.metricsDBName + "." + telemetrymetrics.TimeseriesV4TableName)
From(t.metricsDBName + "." + tableName)
sb.Where(sb.In("metric_name", metricNames))
sb.GroupBy("metric_name")

View File

@@ -26,16 +26,15 @@ var (
OrderByMetricName = ReductionRuleOrderBy{valuer.NewString("metric")}
OrderByIngestedVolume = ReductionRuleOrderBy{valuer.NewString("ingested_volume")}
OrderByReducedVolume = ReductionRuleOrderBy{valuer.NewString("reduced_volume")}
OrderByReduction = ReductionRuleOrderBy{valuer.NewString("reduction")}
OrderByLastUpdated = ReductionRuleOrderBy{valuer.NewString("last_updated")}
)
func (ReductionRuleOrderBy) Enum() []any {
return []any{OrderByMetricName, OrderByIngestedVolume, OrderByReducedVolume, OrderByReduction, OrderByLastUpdated}
return []any{OrderByMetricName, OrderByIngestedVolume, OrderByReducedVolume, OrderByLastUpdated}
}
type ListReductionRulesParams struct {
OrderBy ReductionRuleOrderBy `query:"orderBy,default=reduction" json:"orderBy"`
OrderBy ReductionRuleOrderBy `query:"orderBy,default=ingested_volume" json:"orderBy"`
Order Order `query:"order,default=desc" json:"order"`
Search string `query:"search" json:"search"`
MetricName string `query:"metricName" json:"metricName,omitempty"`

View File

@@ -13,7 +13,7 @@ func TestListReductionRulesParamsSortDefaults(t *testing.T) {
var params metricreductionruletypes.ListReductionRulesParams
require.NoError(t, binding.Query.BindQuery(map[string][]string{"limit": {"10"}}, &params))
assert.Equal(t, metricreductionruletypes.OrderByReduction, params.OrderBy, "orderBy defaults to reduction")
assert.Equal(t, metricreductionruletypes.OrderByIngestedVolume, params.OrderBy, "orderBy defaults to ingested volume")
assert.Equal(t, metricreductionruletypes.OrderDesc, params.Order, "order defaults to desc")
}

View File

@@ -102,7 +102,8 @@ type GettableReductionRule struct {
Active bool `json:"active" required:"true"`
IngestedSeries uint64 `json:"ingestedSeries" required:"true"`
RetainedSeries uint64 `json:"retainedSeries" required:"true"`
ReductionPercent float64 `json:"reductionPercent" required:"true"`
IngestedSamples uint64 `json:"ingestedSamples" required:"true"`
RetainedSamples uint64 `json:"retainedSamples" required:"true"`
}
type GettableReductionRules struct {

View File

@@ -4,5 +4,7 @@ package metricreductionruletypes
type GettableReductionRuleStats struct {
IngestedSeries uint64 `json:"ingestedSeries" required:"true"`
RetainedSeries uint64 `json:"retainedSeries" required:"true"`
IngestedSamples uint64 `json:"ingestedSamples" required:"true"`
RetainedSamples uint64 `json:"retainedSamples" required:"true"`
EstimatedMonthlySavingsUsd float64 `json:"estimatedMonthlySavingsUsd" required:"true"`
}

View File

@@ -46,7 +46,7 @@ type MetadataStore interface {
// GetFirstSeenFromMetricMetadata gets the first seen timestamp for a metric metadata lookup key.
GetFirstSeenFromMetricMetadata(ctx context.Context, lookupKeys []MetricMetadataLookupKey) (map[MetricMetadataLookupKey]int64, error)
FetchLastSeenInfoMulti(ctx context.Context, metricNames ...string) (map[string]int64, error)
FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error)
}
type MetricMetadataLookupKey struct {

View File

@@ -419,6 +419,6 @@ func (m *MockMetadataStore) SetFirstSeenFromMetricMetadata(firstSeenMap map[tele
}
}
func (m *MockMetadataStore) FetchLastSeenInfoMulti(ctx context.Context, metricNames ...string) (map[string]int64, error) {
func (m *MockMetadataStore) FetchLastSeenInfoMulti(ctx context.Context, orgID valuer.UUID, metricNames ...string) (map[string]int64, error) {
return make(map[string]int64), nil
}