fix: ensure that alerts created as disabled don't actually fire

pkg/query-service/rules/manager.go

@@ -544,7 +544,7 @@ func (m *Manager) deleteTask(taskName string) {
 }
 
 // CreateRule stores rule def into db and also
-// starts an executor for the rule
+// starts an executor for the rule, unless the rule is disabled
 func (m *Manager) CreateRule(ctx context.Context, ruleStr string) (*ruletypes.GettableRule, error) {
 	claims, err := authtypes.ClaimsFromContext(ctx)
 	if err != nil {
@@ -611,7 +611,7 @@ func (m *Manager) CreateRule(ctx context.Context, ruleStr string) (*ruletypes.Ge
 		}
 
 		taskName := prepareTaskName(id.StringValue())
-		if err = m.addTask(ctx, orgID, &parsedRule, taskName); err != nil {
+		if err = m.syncRuleStateWithTask(ctx, orgID, taskName, &parsedRule); err != nil {
 			return err
 		}
 

tests/fixtures/alerts.py

@@ -143,6 +143,16 @@ def wait_for_firing_timeline_entry(signoz: types.SigNoz, token: str, rule_id: st
     raise AssertionError(f"No firing entry recorded in rule state history within {wait_seconds}s, items: {items}")
 
 
+def get_rule(signoz: types.SigNoz, token: str, rule_id: str) -> dict:
+    response = requests.get(
+        signoz.self.host_configs["8080"].get(f"/api/v2/rules/{rule_id}"),
+        headers={"Authorization": f"Bearer {token}"},
+        timeout=5,
+    )
+    assert response.status_code == HTTPStatus.OK, f"Failed to get rule, api returned {response.status_code} with response: {response.text}"
+    return response.json()["data"]
+
+
 def get_rule_history_top_contributors(signoz: types.SigNoz, token: str, rule_id: str, start_ms: int, end_ms: int) -> list[dict]:
     response = requests.get(
         signoz.self.host_configs["8080"].get(f"/api/v2/rules/{rule_id}/history/top_contributors"),

tests/integration/testdata/alerts/test_scenarios/disabled_rule/alert_data.jsonl

@@ -0,0 +1,12 @@
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:01:00+00:00","value":1,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:02:00+00:00","value":2,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:03:00+00:00","value":4,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:04:00+00:00","value":4,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:05:00+00:00","value":15,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:06:00+00:00","value":10,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:07:00+00:00","value":36,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:08:00+00:00","value":25,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:09:00+00:00","value":37,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:10:00+00:00","value":35,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:11:00+00:00","value":39,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}
+{"metric_name":"request_total_disabled_rule","labels":{"service":"api","endpoint":"/health","status_code":"200"},"timestamp":"2026-01-29T10:12:00+00:00","value":25,"temporality":"Cumulative","type_":"Sum","is_monotonic":false,"flags":0,"description":"","unit":"","env":"default","resource_attrs":{},"scope_attrs":{}}

tests/integration/testdata/alerts/test_scenarios/disabled_rule/rule.json

@@ -0,0 +1,59 @@
+{
+  "alert": "disabled_rule",
+  "ruleType": "threshold_rule",
+  "alertType": "METRIC_BASED_ALERT",
+  "disabled": true,
+  "condition": {
+    "thresholds": {
+      "kind": "basic",
+      "spec": [
+        {
+          "name": "critical",
+          "target": 10,
+          "matchType": "at_least_once",
+          "op": "above",
+          "channels": [
+            "test channel"
+          ]
+        }
+      ]
+    },
+    "compositeQuery": {
+      "queryType": "clickhouse_sql",
+      "panelType": "graph",
+      "queries": [
+        {
+          "type": "clickhouse_sql",
+          "spec": {
+            "name": "A",
+            "query": "WITH __temporal_aggregation_cte AS (\n SELECT \n fingerprint, \n toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts, \n avg(value) AS per_series_value \n FROM signoz_metrics.distributed_samples_v4 AS points \n INNER JOIN (\n SELECT fingerprint \n FROM signoz_metrics.time_series_v4 \n WHERE metric_name IN ('request_total_disabled_rule') \n AND LOWER(temporality) LIKE LOWER('cumulative') \n GROUP BY fingerprint\n ) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint \n WHERE metric_name IN ('request_total_disabled_rule') \n AND unix_milli >= $start_timestamp_ms \n AND unix_milli < $end_timestamp_ms \n GROUP BY fingerprint, ts \n ORDER BY fingerprint, ts\n), \n__spatial_aggregation_cte AS (\n SELECT \n ts, \n sum(per_series_value) AS value \n FROM __temporal_aggregation_cte \n WHERE isNaN(per_series_value) = 0 \n GROUP BY ts\n) \nSELECT * FROM __spatial_aggregation_cte \nORDER BY ts"
+          }
+        }
+      ]
+    },
+    "selectedQueryName": "A"
+  },
+  "evaluation": {
+    "kind": "rolling",
+    "spec": {
+      "evalWindow": "5m0s",
+      "frequency": "15s"
+    }
+  },
+  "labels": {},
+  "annotations": {
+    "description": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})",
+    "summary": "This alert is fired when the defined metric (current value: {{$value}}) crosses the threshold ({{$threshold}})"
+  },
+  "notificationSettings": {
+    "groupBy": [],
+    "usePolicy": false,
+    "renotify": {
+      "enabled": false,
+      "interval": "30m",
+      "alertStates": []
+    }
+  },
+  "version": "v5",
+  "schemaVersion": "v2alpha1"
+}

tests/integration/tests/alerts/05_disabled_rule.py

@@ -0,0 +1,109 @@
+import json
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    collect_webhook_firing_alerts,
+    get_rule,
+    update_rule_channel_name,
+)
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.fs import get_testdata_file_path
+from fixtures.logger import setup_logger
+
+logger = setup_logger(__name__)
+
+# The rule evaluates every 15s and the alert data is set up to fire on the
+# first evaluation, so a buggy evaluator would transition the rule and fire
+# well within this window.
+OBSERVATION_WINDOW_SECONDS = 35
+
+
+def test_disabled_rule_does_not_evaluate_or_notify(
+    signoz: types.SigNoz,
+    # Notification channel related fixtures
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_webhook_notification_channel: Callable[[str, str, dict, bool], str],
+    # Alert rule related fixtures
+    create_alert_rule: Callable[[dict], str],
+    # Alert data insertion related fixtures
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    get_token: Callable[[str, str], str],
+):
+    """
+    A rule created with disabled: true must not be evaluated: its state must
+    stay "disabled" and it must not send any notification, even though the
+    inserted data would fire the rule if it were evaluated. The companion
+    scenario threshold_above_at_least_once in 02_basic_alert_conditions.py
+    uses the same data shape and fires when the rule is enabled.
+    """
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    # Prepare notification channel name and webhook endpoint
+    notification_channel_name = str(uuid.uuid4())
+    webhook_endpoint_path = f"/alert/{notification_channel_name}"
+    notification_url = notification_channel.container_configs["8080"].get(webhook_endpoint_path)
+
+    # register the mock endpoint in notification channel
+    make_http_mocks(
+        notification_channel,
+        [
+            Mapping(
+                request=MappingRequest(
+                    method=HttpMethods.POST,
+                    url=webhook_endpoint_path,
+                ),
+                response=MappingResponse(
+                    status=200,
+                    json_body={},
+                ),
+                persistent=False,
+            )
+        ],
+    )
+
+    # Create an alert channel using the given route
+    create_webhook_notification_channel(
+        channel_name=notification_channel_name,
+        webhook_url=notification_url,
+        http_config={},
+        send_resolved=False,
+    )
+
+    # Insert alert data that would fire the rule if it were evaluated
+    insert_alert_data(
+        [types.AlertData(type="metrics", data_path="alerts/test_scenarios/disabled_rule/alert_data.jsonl")],
+        base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
+    )
+
+    # Create the disabled alert rule
+    rule_path = get_testdata_file_path("alerts/test_scenarios/disabled_rule/rule.json")
+    with open(rule_path, encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, notification_channel_name)
+    rule_id = create_alert_rule(rule_data)
+    logger.info(
+        "disabled rule created with id: %s",
+        {"rule_id": rule_id, "rule_name": rule_data["alert"]},
+    )
+
+    # The rule must stay disabled and must not fire for the whole observation
+    # window; poll to give a buggy evaluator several chances to run.
+    deadline = time.time() + OBSERVATION_WINDOW_SECONDS
+    while time.time() < deadline:
+        rule = get_rule(signoz, token, rule_id)
+        assert rule["state"] == "disabled", f"disabled rule transitioned to unexpected state: {rule['state']}"
+        assert rule["disabled"] is True, "disabled rule was unexpectedly re-enabled"
+
+        firing_alerts = collect_webhook_firing_alerts(notification_channel, notification_channel_name)
+        assert len(firing_alerts) == 0, f"disabled rule fired alerts: {[alert.labels for alert in firing_alerts]}"
+
+        time.sleep(2)
+
+    logger.info("disabled rule stayed disabled and sent no notifications, as expected")