test(alert-channel-integrations): recreate stale notification_channel cache lacking https:8443

tests/fixtures/alerts.py

@@ -335,27 +335,69 @@ def _is_json_subset(subset, superset) -> bool:
     return subset == superset
 
 
+def _match_query_params(expected: dict, req: dict) -> bool:
+    """Match a wiremock request's query params. Each expected value may be a string
+    (exact), an re.Pattern (search), or None (presence only, e.g. a dynamic hash)."""
+    query_params = req.get("queryParams", {})
+    for name, want in expected.items():
+        if name not in query_params:
+            return False
+        values = query_params[name].get("values", [])
+        if want is None:
+            if not values:
+                return False
+        elif isinstance(want, re.Pattern):
+            if not any(want.search(v) for v in values):
+                return False
+        elif want not in values:
+            return False
+    return True
+
+
 def verify_webhook_notification_expectation(
     notification_channel: types.TestContainerDocker,
     validation_data: dict,
 ) -> bool:
-    """Check if wiremock received a request at the given path
-    whose JSON body is a superset of the expected json_body."""
+    """Check that wiremock received the expected request(s) at the given path.
+
+    validation_data supports (all optional except path):
+      - path: request url path (matched as urlPath, so query strings are ignored)
+      - json_body: expected JSON subset of the request body
+      - query_params: {name: str|re.Pattern|None} matched against the request query
+      - count: exact number of requests required at the path
+      - min_count: minimum number of requests required (e.g. retries)
+    Body/query constraints must be satisfied by a single request; count constraints
+    apply to the total at the path."""
     path = validation_data["path"]
-    json_body = validation_data["json_body"]
+    json_body = validation_data.get("json_body")
+    query_params = validation_data.get("query_params")
 
     url = notification_channel.host_configs["8080"].get("__admin/requests/find")
     try:
-        res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)
+        # urlPath matches the path only; the notifier appends a dynamic threadKey.
+        res = requests.post(url, json={"method": "POST", "urlPath": path}, timeout=10)
     except requests.exceptions.RequestException:
         return False
     if res.status_code != HTTPStatus.OK:
         return False
 
-    for req in res.json()["requests"]:
-        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
-        if _is_json_subset(json_body, body):
-            return True
+    reqs = res.json()["requests"]
+    if "count" in validation_data and len(reqs) != validation_data["count"]:
+        return False
+    if "min_count" in validation_data and len(reqs) < validation_data["min_count"]:
+        return False
+
+    if json_body is None and query_params is None:
+        return True
+
+    for req in reqs:
+        if json_body is not None:
+            body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+            if not _is_json_subset(json_body, body):
+                continue
+        if query_params is not None and not _match_query_params(query_params, req):
+            continue
+        return True
     return False
 
 
@@ -416,7 +458,7 @@ def _received_notifications(
                 continue
             url = notification_channel.host_configs["8080"].get("__admin/requests/find")
             try:
-                res = requests.post(url, json={"method": "POST", "url": validation.validation_data["path"]}, timeout=10)
+                res = requests.post(url, json={"method": "POST", "urlPath": validation.validation_data["path"]}, timeout=10)
                 webhook_bodies.extend(json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8")) for req in res.json()["requests"])
             except requests.exceptions.RequestException as exc:
                 webhook_bodies.append(f"<failed to fetch wiremock journal: {exc}>")
@@ -455,4 +497,11 @@ def update_raw_channel_config(
                     path = urlparse(original_url).path
                     entry[url_field] = notification_channel.container_configs["8080"].get(path)
 
+    # Google Chat validates the webhook host, so route via the https alias config
+    # (chat.googleapis.com:8443) keeping the path, and skip tls for wiremock's cert.
+    for entry in config.get("googlechat_configs", []):
+        path = urlparse(entry["webhook_url"]).path
+        entry["webhook_url"] = notification_channel.container_configs["8443"].get(path)
+        entry.setdefault("http_config", {}).setdefault("tls_config", {})["insecure_skip_verify"] = True
+
     return config

tests/fixtures/notification_channel.py

@@ -18,6 +18,10 @@ from fixtures.maildev import MAILDEV_INCOMING_PASS, SMTP_TEST_FROM
 
 logger = setup_logger(__name__)
 
+# Google Chat validates the webhook host, so the WireMock container is aliased as
+# this hostname on the docker network and channels point at https://<host>:8443/...
+GOOGLE_CHAT_HOST = "chat.googleapis.com"
+
 
 EMAIL_TRANSPORT_KEYS = [
     "from",
@@ -124,6 +128,19 @@ email_default_config = {
 }
 
 
+def googlechat_config(space: str) -> dict:
+    """Google Chat channel config for a per-test WireMock space path. Title/text are
+    omitted so the backend applies its default templates. The host + tls-skip are
+    injected at runtime by update_raw_channel_config."""
+    return {
+        "googlechat_configs": [
+            {
+                "webhook_url": f"/v1/spaces/{space}/messages",  # host set on runtime
+            }
+        ],
+    }
+
+
 @pytest.fixture(name="notification_channel", scope="package")
 def notification_channel(
     network: Network,
@@ -134,9 +151,27 @@ def notification_channel(
     Package-scoped fixture for WireMock container to receive notifications for Alert rules.
     """
 
+    # A --reuse cache from before the https:8443 alias was added lacks the "8443"
+    # config that Google Chat delivery needs (and its container lacks the port/alias).
+    # Drop such a stale cache + container so the fixture recreates a correct one,
+    # instead of raising KeyError on container_configs["8443"].
+    cached = pytestconfig.cache.get("notification_channel", None)
+    if cached and "8443" not in (cached.get("container_configs") or {}):
+        logger.info("Recreating stale notification_channel (cache missing https:8443)")
+        try:
+            docker.from_env().containers.get(cached["id"]).remove(force=True)
+        except docker.errors.NotFound:
+            pass
+        pytestconfig.cache.set("notification_channel", None)
+
     def create() -> types.TestContainerDocker:
+        # http:8080 admin/webhook delivery, plus https:8443 aliased as
+        # chat.googleapis.com so Google Chat's validated webhook host routes here.
         container = WireMockContainer(image="wiremock/wiremock:2.35.1-1", secure=False)
+        container.with_cli_arg("--https-port", "8443")
+        container.with_exposed_ports(8080)  # 8443 reached in-network via the alias, no host mapping needed
         container.with_network(network)
+        container.with_network_aliases(GOOGLE_CHAT_HOST)
         container.start()
 
         return types.TestContainerDocker(
@@ -148,7 +183,11 @@ def notification_channel(
                     container.get_exposed_port(8080),
                 )
             },
-            container_configs={"8080": types.TestContainerUrlConfig("http", container.get_wrapped_container().name, 8080)},
+            container_configs={
+                "8080": types.TestContainerUrlConfig("http", container.get_wrapped_container().name, 8080),
+                # Google Chat delivery: https to the validated host via the network alias.
+                "8443": types.TestContainerUrlConfig("https", GOOGLE_CHAT_HOST, 8443),
+            },
         )
 
     def delete(container: types.TestContainerDocker):

tests/integration/tests/alertmanager/04_googlechat.py

@@ -0,0 +1,232 @@
+"""Google Chat notifier integration tests driven through the real alerting path:
+create a rule pointing at a Google Chat channel, insert breaching telemetry, let
+the ruler fire, and assert on the cardsV2 payload WireMock received.
+
+WireMock stands in for chat.googleapis.com (network alias + https:8443, see the
+notification_channel fixture). Assertions check the actual card structure, deep
+links and threading query params so behavioural regressions are caught.
+"""
+
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.notification_channel import googlechat_config
+
+logger = setup_logger(__name__)
+
+METRICS_DATA = "alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl"
+METRICS_RULE = "alerts/test_scenarios/threshold_above_at_least_once/rule.json"
+LOGS_DATA = "alerts/test_scenarios/threshold_below_at_least_once/alert_data.jsonl"
+LOGS_RULE = "alerts/test_scenarios/threshold_below_at_least_once/rule.json"
+TRACES_DATA = "alerts/test_scenarios/threshold_above_average/alert_data.jsonl"
+TRACES_RULE = "alerts/test_scenarios/threshold_above_average/rule.json"
+
+# threading query params the notifier always appends
+THREAD_QUERY = {
+    "messageReplyOption": "REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD",
+    "threadKey": None,  # dynamic hash; presence only
+}
+
+
+def _path(space: str) -> str:
+    return f"/v1/spaces/{space}/messages"
+
+
+def _stub_200(path: str) -> list[Mapping]:
+    return [
+        Mapping(
+            request=MappingRequest(method=HttpMethods.POST, url_path=path),
+            response=MappingResponse(status=200, json_body={"name": "spaces/x/messages/x"}),
+            persistent=True,
+        )
+    ]
+
+
+def _stub_retry(path: str) -> list[Mapping]:
+    """429 on the first call then 200, via a wiremock scenario transition."""
+    scenario = f"gc-retry-{path}"
+    return [
+        Mapping(
+            request=MappingRequest(method=HttpMethods.POST, url_path=path),
+            response=MappingResponse(status=429, json_body={"error": {"code": 429, "status": "RESOURCE_EXHAUSTED"}}),
+            scenario_name=scenario,
+            required_scenario_state="Started",
+            new_scenario_state="ok",
+            persistent=True,
+        ),
+        Mapping(
+            request=MappingRequest(method=HttpMethods.POST, url_path=path),
+            response=MappingResponse(status=200, json_body={"name": "spaces/x/messages/x"}),
+            scenario_name=scenario,
+            required_scenario_state="ok",
+            persistent=True,
+        ),
+    ]
+
+
+def _card_subset(alertname: str, buttons: list[tuple[str, str]]) -> dict:
+    """A cardsV2 subset asserting title, firing banner, rendered body, and each
+    button's text AND deep-link url (as a regex), so a broken link is caught too.
+    buttons: list of (text, url_regex)."""
+    return {
+        "text": f"[FIRING:1] {alertname}",
+        "cardsV2": [
+            {
+                "cardId": "signoz-alert",
+                "card": {
+                    "header": {"title": f"[FIRING:1] {alertname}"},
+                    "sections": [
+                        # firing banner
+                        {"widgets": [{"textParagraph": {"text": re.compile("FIRING")}}]},
+                        # rendered alert body mentions the alertname
+                        {"widgets": [{"textParagraph": {"text": re.compile(re.escape(alertname))}}]},
+                    ]
+                    + [{"widgets": [{"buttonList": {"buttons": [{"text": text, "onClick": {"openLink": {"url": re.compile(url)}}}]}}]} for text, url in buttons],
+                },
+            }
+        ],
+    }
+
+
+GOOGLECHAT_CASES = [
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_default_metrics_firing",
+        rule_path=METRICS_RULE,
+        alert_data=[types.AlertData(type="metrics", data_path=METRICS_DATA)],
+        channel_config=googlechat_config("gc-metrics"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": _path("gc-metrics"),
+                        "query_params": THREAD_QUERY,
+                        "json_body": _card_subset("threshold_above_at_least_once", [("Open in SigNoz", r"/alerts/overview\?ruleId=")]),
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_rich_card_logs",
+        rule_path=LOGS_RULE,
+        alert_data=[types.AlertData(type="logs", data_path=LOGS_DATA)],
+        channel_config=googlechat_config("gc-logs"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": _path("gc-logs"),
+                        "json_body": _card_subset(
+                            "threshold_below_at_least_once",
+                            [("View Related Logs", r"/logs/logs-explorer\?"), ("Open in SigNoz", r"/alerts/overview\?ruleId=")],
+                        ),
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_rich_card_traces",
+        rule_path=TRACES_RULE,
+        alert_data=[types.AlertData(type="traces", data_path=TRACES_DATA)],
+        channel_config=googlechat_config("gc-traces"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": _path("gc-traces"),
+                        "json_body": _card_subset(
+                            "threshold_above_average",
+                            [("View Related Traces", r"traces-explorer\?"), ("Open in SigNoz", r"/alerts/overview\?ruleId=")],
+                        ),
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_retry_429_then_200",
+        rule_path=METRICS_RULE,
+        alert_data=[types.AlertData(type="metrics", data_path=METRICS_DATA)],
+        channel_config=googlechat_config("gc-retry"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        # a retryable 429 is followed by a successful re-POST => >=2 hits
+                        "path": _path("gc-retry"),
+                        "min_count": 2,
+                        "json_body": {"cardsV2": [{"cardId": "signoz-alert"}]},
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+# per-case wiremock stubs (retry needs a stateful scenario, the rest a plain 200)
+CASE_STUBS: dict[str, Callable[[str], list[Mapping]]] = {
+    "googlechat_retry_429_then_200": _stub_retry,
+}
+
+
+@pytest.mark.parametrize(
+    "gc_test_case",
+    GOOGLECHAT_CASES,
+    ids=lambda c: c.name,
+)
+def test_googlechat_notifier(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    gc_test_case: types.AlertManagerNotificationTestCase,
+) -> None:
+    channel_name = str(uuid.uuid4())
+    path = gc_test_case.notification_expectation.notification_validations[0].validation_data["path"]
+
+    channel_config = update_raw_channel_config(gc_test_case.channel_config, channel_name, notification_channel)
+
+    stub_factory = CASE_STUBS.get(gc_test_case.name, _stub_200)
+    make_http_mocks(notification_channel, stub_factory(path))
+
+    create_notification_channel(channel_config)
+    time.sleep(12)  # org registration in alertmanager
+
+    insert_alert_data(gc_test_case.alert_data, base_time=datetime.now(tz=UTC) - timedelta(minutes=5))
+
+    with open(get_testdata_file_path(gc_test_case.rule_path), encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    create_alert_rule(rule_data)
+
+    verify_notification_expectation(notification_channel, maildev, gc_test_case.notification_expectation)

tests/integration/tests/alerts/02_googlechat_test_channel.py

@@ -0,0 +1,112 @@
+"""Google Chat coverage for the testChannel API (POST /api/v1/testChannel).
+
+testChannel drives the notifier once, synchronously, with a hardcoded test alert
+and no retry. It is the button users click in the UI, and the deterministic place
+to assert permanent-failure / no-retry behaviour. Rich-card and retry behaviour is
+covered via the firing-rule path in alertmanager/04_googlechat.py.
+"""
+
+import base64
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+
+import pytest
+import requests
+from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import update_raw_channel_config
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.logger import setup_logger
+from fixtures.notification_channel import googlechat_config
+
+logger = setup_logger(__name__)
+
+
+def _path(space: str) -> str:
+    return f"/v1/spaces/{space}/messages"
+
+
+# name, space, stub status, stub body, expect testChannel 204
+TEST_CHANNEL_CASES = [
+    ("googlechat_test_channel_success", "gc-tc-ok", 200, {"name": "spaces/x/messages/x"}, True),
+    ("googlechat_test_channel_permanent_400", "gc-tc-400", 400, {"error": {"code": 400, "status": "INVALID_ARGUMENT", "message": "Message cannot be empty."}}, False),
+    ("googlechat_test_channel_permission_403", "gc-tc-403", 403, {"error": {"code": 403, "status": "PERMISSION_DENIED", "message": "Method doesn't allow unregistered callers"}}, False),
+]
+
+
+@pytest.mark.parametrize(
+    "name,space,status,body,expect_delivered",
+    TEST_CHANNEL_CASES,
+    ids=lambda v: v if isinstance(v, str) else "",
+)
+def test_googlechat_test_channel(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    signoz: types.SigNoz,
+    get_token: Callable[[str, str], str],
+    create_user_admin: None,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    name: str,  # pylint: disable=unused-argument
+    space: str,
+    status: int,
+    body: dict,
+    expect_delivered: bool,
+) -> None:
+    path = _path(space)
+    make_http_mocks(
+        notification_channel,
+        [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url_path=path),
+                response=MappingResponse(status=status, json_body=body),
+                persistent=True,
+            )
+        ],
+    )
+
+    channel_name = str(uuid.uuid4())
+    receiver = update_raw_channel_config(googlechat_config(space), channel_name, notification_channel)
+
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    # org registration in alertmanager
+    time.sleep(10)
+
+    response = requests.post(
+        signoz.self.host_configs["8080"].get("/api/v1/testChannel"),
+        json=receiver,
+        headers={"Authorization": f"Bearer {admin_token}"},
+        timeout=30,
+    )
+
+    if expect_delivered:
+        assert response.status_code == HTTPStatus.NO_CONTENT, f"expected 204, got {response.status_code}: {response.text}"
+    else:
+        # a 400/403 is a permanent failure: testChannel surfaces it, does not retry
+        assert response.status_code != HTTPStatus.NO_CONTENT, f"expected failure status, got 204 for {status} stub"
+
+    # exactly one delivery attempt either way (testChannel never retries)
+    count = requests.post(
+        notification_channel.host_configs["8080"].get("/__admin/requests/count"),
+        json={"method": "POST", "urlPath": path},
+        timeout=10,
+    )
+    assert count.json()["count"] == 1, f"expected exactly 1 request (no retry), got {count.text}"
+
+    if expect_delivered:
+        find = requests.post(
+            notification_channel.host_configs["8080"].get("/__admin/requests/find"),
+            json={"method": "POST", "urlPath": path},
+            timeout=10,
+        )
+        req = find.json()["requests"][0]
+        # threading query params are always appended
+        assert "messageReplyOption=REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD" in req["url"]
+        assert "threadKey=" in req["url"]
+        # cardsV2 shape with the hardcoded test alert
+        card = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        assert card["cardsV2"][0]["cardId"] == "signoz-alert"
+        assert re.search(r"Test Alert \(", card["cardsV2"][0]["card"]["header"]["title"])