Compare commits

...

1 Commits

Author SHA1 Message Date
Naman Verma
5a400fb44a fix(rules): keep rule labels from overriding engine labels 2026-11-16 21:01:45 +05:30
10 changed files with 140 additions and 18 deletions

View File

@@ -11992,8 +11992,9 @@ components:
additionalProperties:
type: string
description: Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*;
values may use Go templates such as {{$labels.service_name}}. severity
is the conventional routing label.
values may use Go templates such as {{$labels.service_name}}. A label
named after one the engine writes, such as severity or threshold.name,
is ignored.
type: object
notificationSettings:
allOf:
@@ -12110,8 +12111,9 @@ components:
additionalProperties:
type: string
description: Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*;
values may use Go templates such as {{$labels.service_name}}. severity
is the conventional routing label.
values may use Go templates such as {{$labels.service_name}}. A label
named after one the engine writes, such as severity or threshold.name,
is ignored.
type: object
notificationSettings:
allOf:

View File

@@ -267,9 +267,7 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (int, error) {
lb := ruletypes.NewBuilder(smpl.Metric...).Del(ruletypes.MetricNameLabel).Del(ruletypes.TemporalityLabel)
resultLabels := ruletypes.NewBuilder(smpl.Metric...).Del(ruletypes.MetricNameLabel).Del(ruletypes.TemporalityLabel).Labels()
for name, value := range r.Labels().Map() {
lb.Set(name, expand(value))
}
lb.SetRuleLabels(r.Labels().Map(), expand)
lb.Set(ruletypes.AlertNameLabel, r.Name())
lb.Set(ruletypes.AlertRuleIDLabel, r.ID())

View File

@@ -13156,7 +13156,7 @@ export interface RuletypesNotificationSettingsDTO {
export type RuletypesPostableRuleDTOAnnotations = { [key: string]: string };
/**
* Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. severity is the conventional routing label.
* Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. A label named after one the engine writes, such as severity or threshold.name, is ignored.
*/
export type RuletypesPostableRuleDTOLabels = { [key: string]: string };
@@ -13273,7 +13273,7 @@ export interface RuletypesPostableRuleDTO {
evaluation: RuletypesEvaluationEnvelopeDTO;
/**
* @type object
* @description Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. severity is the conventional routing label.
* @description Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. A label named after one the engine writes, such as severity or threshold.name, is ignored.
*/
labels?: RuletypesPostableRuleDTOLabels;
/**
@@ -13306,7 +13306,7 @@ export interface RuletypesPostableRuleViewDTO {
export type RuletypesRuleDTOAnnotations = { [key: string]: string };
/**
* Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. severity is the conventional routing label.
* Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. A label named after one the engine writes, such as severity or threshold.name, is ignored.
*/
export type RuletypesRuleDTOLabels = { [key: string]: string };
@@ -13365,7 +13365,7 @@ export interface RuletypesRuleDTO {
id: string;
/**
* @type object
* @description Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. severity is the conventional routing label.
* @description Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. A label named after one the engine writes, such as severity or threshold.name, is ignored.
*/
labels?: RuletypesRuleDTOLabels;
/**

View File

@@ -201,9 +201,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (int, error) {
lb := ruletypes.NewBuilder(result.Metric...).Del(ruletypes.MetricNameLabel)
resultLabels := ruletypes.NewBuilder(result.Metric...).Del(ruletypes.MetricNameLabel).Labels()
for name, value := range r.labels.Map() {
lb.Set(name, expand(value))
}
lb.SetRuleLabels(r.labels.Map(), expand)
lb.Set(ruletypes.AlertNameLabel, r.Name())
lb.Set(ruletypes.AlertRuleIDLabel, r.ID())

View File

@@ -278,9 +278,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
lb := ruletypes.NewBuilder(smpl.Metric...).Del(ruletypes.MetricNameLabel).Del(ruletypes.TemporalityLabel)
resultLabels := ruletypes.NewBuilder(smpl.Metric...).Del(ruletypes.MetricNameLabel).Del(ruletypes.TemporalityLabel).Labels()
for name, value := range r.labels.Map() {
lb.Set(name, expand(value))
}
lb.SetRuleLabels(r.labels.Map(), expand)
lb.Set(ruletypes.AlertNameLabel, r.Name())
lb.Set(ruletypes.AlertRuleIDLabel, r.ID())

View File

@@ -71,7 +71,7 @@ type PostableRule struct {
RuleType RuleType `json:"ruleType" required:"true" description:"The evaluation strategy: threshold_rule compares against thresholds, promql_rule evaluates a PromQL expression, anomaly_rule scores deviation from seasonal history. Defaults to threshold_rule for builder and clickhouse_sql queries and is forced to promql_rule for promql queries."`
RuleCondition *RuleCondition `json:"condition" required:"true" description:"The query to evaluate and the thresholds applied to its result."`
Labels map[string]string `json:"labels,omitempty" description:"Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. severity is the conventional routing label."`
Labels map[string]string `json:"labels,omitempty" description:"Labels attached to every alert the rule fires. Keys match [a-zA-Z_.][a-zA-Z0-9_.]*; values may use Go templates such as {{$labels.service_name}}. A label named after one the engine writes, such as severity or threshold.name, is ignored."`
Annotations map[string]string `json:"annotations,omitempty" description:"Free text attached to every alert, conventionally summary and description. Values may use Go templates with $value, $threshold and $labels."`
Disabled bool `json:"disabled" description:"Whether the rule is paused. A disabled rule is neither evaluated nor notifies. Defaults to false."`

View File

@@ -31,6 +31,23 @@ const (
AlertDescriptionLabel = "description"
)
// reservedLabelNames are written by the rule engine; a rule label with one of
// these names never reaches the alert.
var reservedLabelNames = map[string]struct{}{
MetricNameLabel: {},
TemporalityLabel: {},
AlertNameLabel: {},
AlertRuleIDLabel: {},
RuleSourceLabel: {},
NoDataLabel: {},
LabelThresholdName: {},
LabelSeverityName: {},
LabelLastSeen: {},
// TODO: the template layer reads is_recovering but nothing writes it;
// set it from Alert.IsRecovering.
LabelIsRecovering: {},
}
// Label is a key/value pair of strings.
// TODO(srikanthccv): https://github.com/SigNoz/signoz/issues/9232?
type Label struct {
@@ -285,6 +302,20 @@ func (b *Builder) Set(n, v string) *Builder {
return b
}
// SetRuleLabels adds the rule's labels, each expanded, skipping the names the
// engine writes.
// TODO: a rule label still overwrites a query result label of the same name;
// decide whether those are locked too.
func (b *Builder) SetRuleLabels(ruleLabels map[string]string, expand func(string) string) *Builder {
for name, value := range ruleLabels {
if _, reserved := reservedLabelNames[name]; reserved {
continue
}
b.Set(name, expand(value))
}
return b
}
// Labels returns the labels from the builder. If no modifications
// were made, the original labels are returned.
func (b *Builder) Labels() Labels {

View File

@@ -0,0 +1,12 @@
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:01:00+00:00","value":1,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:02:00+00:00","value":2,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:03:00+00:00","value":4,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:04:00+00:00","value":4,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:05:00+00:00","value":15,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:06:00+00:00","value":10,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:07:00+00:00","value":36,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:08:00+00:00","value":25,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:09:00+00:00","value":37,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:10:00+00:00","value":35,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:11:00+00:00","value":39,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"request_total_reserved_labels_from_engine","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:12:00+00:00","value":25,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}

View File

@@ -0,0 +1,57 @@
{
"alert": "reserved_labels_from_engine",
"ruleType": "threshold_rule",
"alertType": "METRIC_BASED_ALERT",
"condition": {
"thresholds": {
"kind": "basic",
"spec": [
{
"name": "critical",
"target": 10,
"matchType": "at_least_once",
"op": "above",
"channels": [
"test channel"
]
}
]
},
"compositeQuery": {
"queryType": "clickhouse_sql",
"panelType": "graph",
"queries": [
{
"type": "clickhouse_sql",
"spec": {
"name": "A",
"query": "WITH __temporal_aggregation_cte AS (\n SELECT \n fingerprint, \n toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, \n avg(value) AS per_series_value \n FROM signoz_metrics.distributed_samples_v4 AS points \n INNER JOIN (\n SELECT fingerprint \n FROM signoz_metrics.time_series_v4 \n WHERE metric_name IN ('request_total_reserved_labels_from_engine') \n AND LOWER(temporality) LIKE LOWER('cumulative') \n GROUP BY fingerprint\n ) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint \n WHERE metric_name IN ('request_total_reserved_labels_from_engine') \n AND unix_milli >= $start_timestamp_ms \n AND unix_milli < $end_timestamp_ms \n GROUP BY fingerprint, ts \n ORDER BY fingerprint, ts\n), \n__spatial_aggregation_cte AS (\n SELECT \n ts, \n sum(per_series_value) AS value \n FROM __temporal_aggregation_cte \n WHERE isNaN(per_series_value) = 0 \n GROUP BY ts\n) \nSELECT * FROM __spatial_aggregation_cte \nORDER BY ts"
}
}
]
},
"selectedQueryName": "A"
},
"evaluation": {
"kind": "rolling",
"spec": {
"evalWindow": "5m0s",
"frequency": "15s"
}
},
"labels": {"threshold.name": "spoofed", "severity": "spoofed", "team": "payments"},
"annotations": {
"description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
"summary": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})"
},
"notificationSettings": {
"groupBy": [],
"usePolicy": false,
"renotify": {
"enabled": false,
"interval": "30m",
"alertStates": []
}
},
"schemaVersion": "v2alpha1"
}

View File

@@ -543,6 +543,32 @@ TEST_RULES_MISCELLANEOUS = [
],
),
),
# the rule carries threshold.name and severity labels of its own; the alert
# keeps the values the engine wrote and only the plain label comes through
types.AlertTestCase(
name="test_rule_labels_do_not_override_engine_labels",
rule_path="ruler/test_scenarios/reserved_labels_from_engine/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="ruler/test_scenarios/reserved_labels_from_engine/alert_data.jsonl",
),
],
alert_expectation=types.AlertExpectation(
should_alert=True,
wait_time_seconds=30,
expected_alerts=[
types.FiringAlert(
labels={
"alertname": "reserved_labels_from_engine",
"threshold.name": "critical",
"severity": "critical",
"team": "payments",
}
),
],
),
),
# TODO: @abhishekhugetech enable the test for multi threshold rule, pylint: disable=W0511
# after the [issue](https://github.com/SigNoz/engineering-pod/issues/3934) with alertManager is resolved
# types.AlertTestCase(