test(googlechat): address review feedback on assertions and flakiness

- switch testChannel calls to the non-deprecated /api/v1/channels/test
- pin failure cases to 500 + 'unexpected status code <N>' body instead
  of a weak != 204, distinguishing downstream 400 from 403
- replace the fixed sleep(10) with a poll: channels/test 404s until the
  org's alertmanager registers, without reaching the notifier, so the
  first non-404 response is the single authoritative delivery attempt
- parametrize testChannel cases via a NamedTuple with short ids
- firing path: exactly-once (count 1) guard on non-retry cases,
  threading params asserted on the retry delivery, and both retry
  attempts must share the same threadKey

tests/fixtures/alerts.py

@@ -335,27 +335,69 @@ def _is_json_subset(subset, superset) -> bool:
     return subset == superset
 
 
+def _match_query_params(expected: dict, req: dict) -> bool:
+    """Match a wiremock request's query params. Each expected value may be a string
+    (exact), an re.Pattern (search), or None (presence only, e.g. a dynamic hash)."""
+    query_params = req.get("queryParams", {})
+    for name, want in expected.items():
+        if name not in query_params:
+            return False
+        values = query_params[name].get("values", [])
+        if want is None:
+            if not values:
+                return False
+        elif isinstance(want, re.Pattern):
+            if not any(want.search(v) for v in values):
+                return False
+        elif want not in values:
+            return False
+    return True
+
+
 def verify_webhook_notification_expectation(
     notification_channel: types.TestContainerDocker,
     validation_data: dict,
 ) -> bool:
-    """Check if wiremock received a request at the given path
-    whose JSON body is a superset of the expected json_body."""
+    """Check that wiremock received the expected request(s) at the given path.
+
+    validation_data supports (all optional except path):
+      - path: request url path (matched as urlPath, so query strings are ignored)
+      - json_body: expected JSON subset of the request body
+      - query_params: {name: str|re.Pattern|None} matched against the request query
+      - count: exact number of requests required at the path
+      - min_count: minimum number of requests required (e.g. retries)
+    Body/query constraints must be satisfied by a single request; count constraints
+    apply to the total at the path."""
     path = validation_data["path"]
-    json_body = validation_data["json_body"]
+    json_body = validation_data.get("json_body")
+    query_params = validation_data.get("query_params")
 
     url = notification_channel.host_configs["8080"].get("__admin/requests/find")
     try:
-        res = requests.post(url, json={"method": "POST", "url": path}, timeout=10)
+        # urlPath matches the path only; the notifier appends a dynamic threadKey.
+        res = requests.post(url, json={"method": "POST", "urlPath": path}, timeout=10)
     except requests.exceptions.RequestException:
         return False
     if res.status_code != HTTPStatus.OK:
         return False
 
-    for req in res.json()["requests"]:
-        body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
-        if _is_json_subset(json_body, body):
-            return True
+    reqs = res.json()["requests"]
+    if "count" in validation_data and len(reqs) != validation_data["count"]:
+        return False
+    if "min_count" in validation_data and len(reqs) < validation_data["min_count"]:
+        return False
+
+    if json_body is None and query_params is None:
+        return True
+
+    for req in reqs:
+        if json_body is not None:
+            body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+            if not _is_json_subset(json_body, body):
+                continue
+        if query_params is not None and not _match_query_params(query_params, req):
+            continue
+        return True
     return False
 
 
@@ -416,7 +458,7 @@ def _received_notifications(
                 continue
             url = notification_channel.host_configs["8080"].get("__admin/requests/find")
             try:
-                res = requests.post(url, json={"method": "POST", "url": validation.validation_data["path"]}, timeout=10)
+                res = requests.post(url, json={"method": "POST", "urlPath": validation.validation_data["path"]}, timeout=10)
                 webhook_bodies.extend(json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8")) for req in res.json()["requests"])
             except requests.exceptions.RequestException as exc:
                 webhook_bodies.append(f"<failed to fetch wiremock journal: {exc}>")
@@ -455,4 +497,9 @@ def update_raw_channel_config(
                     path = urlparse(original_url).path
                     entry[url_field] = notification_channel.container_configs["8080"].get(path)
 
+    # Google Chat validates the webhook host
+    for entry in config.get("googlechat_configs", []):
+        https = notification_channel.container_configs["443"]
+        entry["webhook_url"] = f"{https.scheme}://{https.address}{urlparse(entry['webhook_url']).path}"
+
     return config

tests/fixtures/notification_channel.py

@@ -1,23 +1,32 @@
 # pylint: disable=line-too-long
 import json
+import re
 import time
 from collections.abc import Callable
 from http import HTTPStatus
+from pathlib import Path
 
 import docker
 import docker.errors
 import pytest
 import requests
 from testcontainers.core.container import Network
+from wiremock.resources.mappings import HttpMethods, Mapping, MappingRequest, MappingResponse
 from wiremock.testing.testcontainer import WireMockContainer
 
 from fixtures import reuse, types
 from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
 from fixtures.logger import setup_logger
 from fixtures.maildev import MAILDEV_INCOMING_PASS, SMTP_TEST_FROM
+from fixtures.tls import CA_ID_LABEL, KEYSTORE_PASSWORD, ca_id, issue_server_keystore
 
 logger = setup_logger(__name__)
 
+# Google Chat validates the webhook host, so the WireMock container joins the
+# network under this alias and serves HTTPS on 443 with a certificate issued by
+# the integration CA that signoz trusts; channels point at https://<host>/...
+GOOGLE_CHAT_HOST = "chat.googleapis.com"
+
 
 EMAIL_TRANSPORT_KEYS = [
     "from",
@@ -124,9 +133,77 @@ email_default_config = {
 }
 
 
+def googlechat_config(space: str) -> dict:
+    """Google Chat channel config for a per-test WireMock space path. Title/text are
+    omitted so the backend applies its default templates. The host is injected at
+    runtime by update_raw_channel_config."""
+    return {
+        "googlechat_configs": [
+            {
+                "webhook_url": f"/v1/spaces/{space}/messages",  # host set on runtime
+            }
+        ],
+    }
+
+
+def googlechat_ok_mappings(path: str) -> list[Mapping]:
+    return [
+        Mapping(
+            request=MappingRequest(method=HttpMethods.POST, url_path=path),
+            response=MappingResponse(status=200, json_body={"name": "spaces/x/messages/x"}),
+        )
+    ]
+
+
+def googlechat_retry_mappings(path: str) -> list[Mapping]:
+    """429 on the first call then 200, via a wiremock scenario transition."""
+    scenario = f"gc-retry-{path}"
+    return [
+        Mapping(
+            request=MappingRequest(method=HttpMethods.POST, url_path=path),
+            response=MappingResponse(status=429, json_body={"error": {"code": 429, "status": "RESOURCE_EXHAUSTED"}}),
+            scenario_name=scenario,
+            required_scenario_state="Started",
+            new_scenario_state="ok",
+        ),
+        Mapping(
+            request=MappingRequest(method=HttpMethods.POST, url_path=path),
+            response=MappingResponse(status=200, json_body={"name": "spaces/x/messages/x"}),
+            scenario_name=scenario,
+            required_scenario_state="ok",
+        ),
+    ]
+
+
+def googlechat_card_subset(alertname: str, buttons: list[tuple[str, str]]) -> dict:
+    """A cardsV2 subset asserting title, firing banner, rendered body, and each
+    button's text AND deep-link url (as a regex), so a broken link is caught too.
+    buttons: list of (text, url_regex)."""
+    return {
+        "text": f"[FIRING:1] {alertname}",
+        "cardsV2": [
+            {
+                "cardId": "signoz-alert",
+                "card": {
+                    "header": {"title": f"[FIRING:1] {alertname}"},
+                    "sections": [
+                        # firing banner
+                        {"widgets": [{"textParagraph": {"text": re.compile("FIRING")}}]},
+                        # rendered alert body mentions the alertname
+                        {"widgets": [{"textParagraph": {"text": re.compile(re.escape(alertname))}}]},
+                    ]
+                    + [{"widgets": [{"buttonList": {"buttons": [{"text": text, "onClick": {"openLink": {"url": re.compile(url)}}}]}}]} for text, url in buttons],
+                },
+            }
+        ],
+    }
+
+
 @pytest.fixture(name="notification_channel", scope="package")
-def notification_channel(
+def notification_channel(  # pylint: disable=too-many-arguments,too-many-positional-arguments
     network: Network,
+    tls: types.TLS,
+    tmpfs: Callable[[str], Path],
     request: pytest.FixtureRequest,
     pytestconfig: pytest.Config,
 ) -> types.TestContainerDocker:
@@ -135,9 +212,25 @@ def notification_channel(
     """
 
     def create() -> types.TestContainerDocker:
+        # http:8080 for admin API + plain webhook delivery; https:443 aliased as
+        # chat.googleapis.com with a CA-issued cert so Google Chat's validated
+        # webhook host routes here over real TLS (signoz trusts the integration CA).
+        keystore_path = issue_server_keystore(tls, tmpfs("notification-channel-certs"), GOOGLE_CHAT_HOST)
+
         container = WireMockContainer(image="wiremock/wiremock:2.35.1-1", secure=False)
+        container.with_volume_mapping(str(keystore_path.parent), "/certs", "ro")
         container.with_network(network)
-        container.start()
+        container.with_network_aliases(GOOGLE_CHAT_HOST)
+        container.with_kwargs(labels={CA_ID_LABEL: ca_id(tls)})
+
+        try:
+            container.start(f"--port 8080 --https-port 443 --https-keystore /certs/keystore.p12 --keystore-type PKCS12 --keystore-password {KEYSTORE_PASSWORD}")
+        except Exception:
+            # Ryuk is disabled: a started-but-unready container would survive and
+            # keep squatting on the chat.googleapis.com alias, poisoning DNS for
+            # any replacement on the shared network.
+            container.stop()
+            raise
 
         return types.TestContainerDocker(
             id=container.get_wrapped_container().id,
@@ -148,7 +241,11 @@ def notification_channel(
                     container.get_exposed_port(8080),
                 )
             },
-            container_configs={"8080": types.TestContainerUrlConfig("http", container.get_wrapped_container().name, 8080)},
+            container_configs={
+                "8080": types.TestContainerUrlConfig("http", container.get_wrapped_container().name, 8080),
+                # Google Chat delivery: https to the validated host via the network alias.
+                "443": types.TestContainerUrlConfig("https", GOOGLE_CHAT_HOST, 443),
+            },
         )
 
     def delete(container: types.TestContainerDocker):
@@ -165,6 +262,16 @@ def notification_channel(
     def restore(cache: dict) -> types.TestContainerDocker:
         return types.TestContainerDocker.from_cache(cache)
 
+    def stale(container: types.TestContainerDocker) -> bool:
+        # A container built against a rotated/absent CA can't serve a cert signoz
+        # trusts; recreate it instead of failing TLS opaquely.
+        client = docker.from_env()
+        try:
+            labels = client.containers.get(container_id=container.id).attrs["Config"]["Labels"]
+        except docker.errors.NotFound:
+            return True
+        return labels.get(CA_ID_LABEL) != ca_id(tls)
+
     return reuse.wrap(
         request,
         pytestconfig,
@@ -173,6 +280,7 @@ def notification_channel(
         create,
         delete,
         restore,
+        stale=stale,
     )
 
 

tests/integration/tests/alertmanager/04_googlechat.py

@@ -0,0 +1,182 @@
+import json
+import time
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+import pytest
+import requests
+from wiremock.resources.mappings import Mapping
+
+from fixtures import types
+from fixtures.alerts import (
+    get_testdata_file_path,
+    update_raw_channel_config,
+    update_rule_channel_name,
+    verify_notification_expectation,
+)
+from fixtures.logger import setup_logger
+from fixtures.notification_channel import (
+    googlechat_card_subset,
+    googlechat_config,
+    googlechat_ok_mappings,
+    googlechat_retry_mappings,
+)
+
+logger = setup_logger(__name__)
+
+METRICS_DATA = "alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl"
+METRICS_RULE = "alerts/test_scenarios/threshold_above_at_least_once/rule.json"
+LOGS_DATA = "alerts/test_scenarios/threshold_below_at_least_once/alert_data.jsonl"
+LOGS_RULE = "alerts/test_scenarios/threshold_below_at_least_once/rule.json"
+TRACES_DATA = "alerts/test_scenarios/threshold_above_average/alert_data.jsonl"
+TRACES_RULE = "alerts/test_scenarios/threshold_above_average/rule.json"
+
+# threading query params the notifier always appends
+THREAD_QUERY = {
+    "messageReplyOption": "REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD",
+    "threadKey": None,  # dynamic hash; presence only
+}
+
+
+GOOGLECHAT_CASES = [
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_default_metrics_firing",
+        rule_path=METRICS_RULE,
+        alert_data=[types.AlertData(type="metrics", data_path=METRICS_DATA)],
+        channel_config=googlechat_config("gc-metrics"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v1/spaces/gc-metrics/messages",
+                        "count": 1,
+                        "query_params": THREAD_QUERY,
+                        "json_body": googlechat_card_subset("threshold_above_at_least_once", [("Open in SigNoz", r"/alerts/overview\?ruleId=")]),
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_rich_card_logs",
+        rule_path=LOGS_RULE,
+        alert_data=[types.AlertData(type="logs", data_path=LOGS_DATA)],
+        channel_config=googlechat_config("gc-logs"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v1/spaces/gc-logs/messages",
+                        "count": 1,
+                        "json_body": googlechat_card_subset(
+                            "threshold_below_at_least_once",
+                            [("View Related Logs", r"/logs/logs-explorer\?"), ("Open in SigNoz", r"/alerts/overview\?ruleId=")],
+                        ),
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_rich_card_traces",
+        rule_path=TRACES_RULE,
+        alert_data=[types.AlertData(type="traces", data_path=TRACES_DATA)],
+        channel_config=googlechat_config("gc-traces"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        "path": "/v1/spaces/gc-traces/messages",
+                        "count": 1,
+                        "json_body": googlechat_card_subset(
+                            "threshold_above_average",
+                            [("View Related Traces", r"traces-explorer\?"), ("Open in SigNoz", r"/alerts/overview\?ruleId=")],
+                        ),
+                    },
+                ),
+            ],
+        ),
+    ),
+    types.AlertManagerNotificationTestCase(
+        name="googlechat_retry_429_then_200",
+        rule_path=METRICS_RULE,
+        alert_data=[types.AlertData(type="metrics", data_path=METRICS_DATA)],
+        channel_config=googlechat_config("gc-retry"),
+        notification_expectation=types.AMNotificationExpectation(
+            should_notify=True,
+            wait_time_seconds=150,
+            notification_validations=[
+                types.NotificationValidation(
+                    destination_type="webhook",
+                    validation_data={
+                        # a retryable 429 is followed by a successful re-POST => >=2 hits
+                        "path": "/v1/spaces/gc-retry/messages",
+                        "min_count": 2,
+                        "query_params": THREAD_QUERY,
+                        "json_body": {"cardsV2": [{"cardId": "signoz-alert"}]},
+                    },
+                ),
+            ],
+        ),
+    ),
+]
+
+# per-case wiremock stubs (retry needs a stateful scenario, the rest a plain 200)
+CASE_STUBS: dict[str, Callable[[str], list[Mapping]]] = {
+    "googlechat_retry_429_then_200": googlechat_retry_mappings,
+}
+
+
+@pytest.mark.parametrize(
+    "gc_test_case",
+    GOOGLECHAT_CASES,
+    ids=lambda c: c.name,
+)
+def test_googlechat_notifier(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    create_notification_channel: Callable[[dict], str],
+    create_alert_rule: Callable[[dict], str],
+    insert_alert_data: Callable[[list[types.AlertData], datetime], None],
+    maildev: types.TestContainerDocker,
+    gc_test_case: types.AlertManagerNotificationTestCase,
+) -> None:
+    channel_name = str(uuid.uuid4())
+    path = gc_test_case.notification_expectation.notification_validations[0].validation_data["path"]
+
+    channel_config = update_raw_channel_config(gc_test_case.channel_config, channel_name, notification_channel)
+
+    stub_factory = CASE_STUBS.get(gc_test_case.name, googlechat_ok_mappings)
+    make_http_mocks(notification_channel, stub_factory(path))
+
+    create_notification_channel(channel_config)
+    time.sleep(12)  # org registration in alertmanager
+
+    insert_alert_data(gc_test_case.alert_data, base_time=datetime.now(tz=UTC) - timedelta(minutes=5))
+
+    with open(get_testdata_file_path(gc_test_case.rule_path), encoding="utf-8") as f:
+        rule_data = json.loads(f.read())
+    update_rule_channel_name(rule_data, channel_name)
+    create_alert_rule(rule_data)
+
+    verify_notification_expectation(notification_channel, maildev, gc_test_case.notification_expectation)
+
+    if gc_test_case.name == "googlechat_retry_429_then_200":
+        find = requests.post(
+            notification_channel.host_configs["8080"].get("/__admin/requests/find"),
+            json={"method": "POST", "urlPath": path},
+            timeout=10,
+        )
+        # the retried POST must land in the same chat thread as the 429'd attempt
+        thread_keys = {req["queryParams"]["threadKey"]["values"][0] for req in find.json()["requests"]}
+        assert len(thread_keys) == 1 and "" not in thread_keys, f"expected one shared threadKey across retry attempts, got {thread_keys}"

tests/integration/tests/alertmanager/conftest.py

@@ -13,6 +13,7 @@ def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
     gateway: types.TestContainerDocker,
     sqlstore: types.TestContainerSQL,
     clickhouse: types.TestContainerClickhouse,
+    tls: types.TLS,
     request: pytest.FixtureRequest,
     pytestconfig: pytest.Config,
     maildev: types.TestContainerDocker,
@@ -24,6 +25,7 @@ def signoz(  # pylint: disable=too-many-arguments,too-many-positional-arguments
         gateway=gateway,
         sqlstore=sqlstore,
         clickhouse=clickhouse,
+        tls=tls,
         request=request,
         pytestconfig=pytestconfig,
         cache_key="signoz_alertmanager",

tests/integration/tests/alerts/02_googlechat_test_channel.py

@@ -0,0 +1,115 @@
+import base64
+import json
+import re
+import time
+import uuid
+from collections.abc import Callable
+from http import HTTPStatus
+from typing import NamedTuple
+
+import pytest
+import requests
+from wiremock.resources.mappings import HttpMethods, Mapping, MappingRequest, MappingResponse
+
+from fixtures import types
+from fixtures.alerts import update_raw_channel_config
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.logger import setup_logger
+from fixtures.notification_channel import googlechat_config
+
+logger = setup_logger(__name__)
+
+
+# channel test (POST /api/v1/channels/test) drives the notifier once, synchronously,
+# with a hardcoded test alert and no retry — the deterministic place to assert
+# permanent-failure behaviour. Rich cards + retry are covered in alertmanager/04_googlechat.py.
+class TestChannelCase(NamedTuple):
+    __test__ = False
+    name: str
+    space: str
+    status: int  # stub status
+    body: dict  # stub body
+    expect_delivered: bool  # expect channels/test 204
+
+
+TEST_CHANNEL_CASES = [
+    TestChannelCase("success", "gc-tc-ok", 200, {"name": "spaces/x/messages/x"}, True),
+    TestChannelCase("permanent_400", "gc-tc-400", 400, {"error": {"code": 400, "status": "INVALID_ARGUMENT", "message": "Message cannot be empty."}}, False),
+    TestChannelCase("permission_403", "gc-tc-403", 403, {"error": {"code": 403, "status": "PERMISSION_DENIED", "message": "Method doesn't allow unregistered callers"}}, False),
+]
+
+
+@pytest.mark.parametrize(
+    "case",
+    TEST_CHANNEL_CASES,
+    ids=lambda c: c.name,
+)
+def test_googlechat_test_channel(  # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals
+    signoz: types.SigNoz,
+    get_token: Callable[[str, str], str],
+    create_user_admin: None,  # pylint: disable=unused-argument
+    notification_channel: types.TestContainerDocker,
+    make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
+    case: TestChannelCase,
+) -> None:
+    path = f"/v1/spaces/{case.space}/messages"
+    make_http_mocks(
+        notification_channel,
+        [
+            Mapping(
+                request=MappingRequest(method=HttpMethods.POST, url_path=path),
+                response=MappingResponse(status=case.status, json_body=case.body),
+            )
+        ],
+    )
+
+    channel_name = str(uuid.uuid4())
+    receiver = update_raw_channel_config(googlechat_config(case.space), channel_name, notification_channel)
+
+    admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    # channels/test 404s until the org's alertmanager registers (one poll tick),
+    # without reaching the notifier — so the first non-404 response is the single
+    # authoritative delivery attempt and the count == 1 assertion below holds
+    deadline = time.time() + 60
+    while True:
+        response = requests.post(
+            signoz.self.host_configs["8080"].get("/api/v1/channels/test"),
+            json=receiver,
+            headers={"Authorization": f"Bearer {admin_token}"},
+            timeout=30,
+        )
+        if response.status_code != HTTPStatus.NOT_FOUND or time.time() > deadline:
+            break
+        time.sleep(2)
+
+    if case.expect_delivered:
+        assert response.status_code == HTTPStatus.NO_CONTENT, f"expected 204, got {response.status_code}: {response.text}"
+    else:
+        # a downstream 400/403 surfaces as a 500 (untyped notify error) whose body
+        # carries the real downstream status code; pin it to distinguish 400 vs 403
+        assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, f"expected 500, got {response.status_code}: {response.text}"
+        assert f"unexpected status code {case.status}" in response.text, f"expected downstream {case.status} in error body: {response.text}"
+
+    # exactly one delivery attempt either way (testChannel never retries)
+    count = requests.post(
+        notification_channel.host_configs["8080"].get("/__admin/requests/count"),
+        json={"method": "POST", "urlPath": path},
+        timeout=10,
+    )
+    assert count.json()["count"] == 1, f"expected exactly 1 request (no retry), got {count.text}"
+
+    if case.expect_delivered:
+        find = requests.post(
+            notification_channel.host_configs["8080"].get("/__admin/requests/find"),
+            json={"method": "POST", "urlPath": path},
+            timeout=10,
+        )
+        req = find.json()["requests"][0]
+        # threading query params are always appended
+        assert "messageReplyOption=REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD" in req["url"]
+        assert "threadKey=" in req["url"]
+        # cardsV2 shape with the hardcoded test alert
+        card = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
+        assert card["cardsV2"][0]["cardId"] == "signoz-alert"
+        assert re.search(r"Test Alert \(", card["cardsV2"][0]["card"]["header"]["title"])