Compare commits

...

2 Commits

Author SHA1 Message Date
Naman Verma
31b8681ca5 test: add integration test 2026-11-18 14:21:18 +05:30
Naman Verma
d9d410b8a7 fix: use default title and body when notification content rendering fails 2026-11-18 14:10:52 +05:30
7 changed files with 222 additions and 28 deletions

View File

@@ -154,8 +154,8 @@ func (n *Notifier) encodeSendMessage(chatID int64, text string) (*bytes.Buffer,
}
// buildMessage sends the title in bold above the message. Each half comes from
// the rule's template when it has one and from the channel's otherwise; an
// empty title sends the message alone.
// the rule's template when it has one that renders and from the channel's
// otherwise; an empty title sends the message alone.
func (n *Notifier) buildMessage(ctx context.Context, logger *slog.Logger, as []*types.Alert) (string, error) {
customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(as)
@@ -179,7 +179,7 @@ func (n *Notifier) buildMessage(ctx context.Context, logger *slog.Logger, as []*
title, body string
err error
)
if customTitle != "" {
if result != nil && !result.IsDefaultTitle {
title, err = n.renderRuleTemplate(result.Title)
} else {
title, err = n.renderChannelTemplate(ctx, as, n.conf.Title)
@@ -188,7 +188,7 @@ func (n *Notifier) buildMessage(ctx context.Context, logger *slog.Logger, as []*
return "", err
}
if customBody != "" {
if result != nil && !result.IsDefaultBody {
body, err = n.renderRuleTemplate(strings.Join(nonEmpty(result.Body), "\n\n"))
} else {
body, err = n.renderChannelTemplate(ctx, as, n.conf.Message)

View File

@@ -225,6 +225,16 @@ func TestNotifyBuildsTheMessage(t *testing.T) {
},
wantContains: []string{"<b>Summary:</b> p99 is 612ms"},
},
{
// Templates written against the channel data model do not render
// against the rule's, and the channel's own title and message stand in.
name: "UnrenderableRuleTemplates_FallBackToChannel",
alerts: []*types.Alert{newAlert("web-3", model.LabelSet{
ruletypes.AnnotationTitleTemplate: "Rule title: {{ .CommonLabels.alertname }}",
ruletypes.AnnotationBodyTemplate: "{{ .Labels.alertname }}: {{ .Annotations.summary }}",
})},
wantContains: []string{"<b>[FIRING:1] checkout-latency-high</b>\n\n<b>Alert:</b>", "<b>Summary:</b> p99 is 612ms"},
},
}
for _, testCase := range testCases {

View File

@@ -36,11 +36,14 @@ func (at *templater) Expand(
title, titleMissingVars, err := at.expandTitle(req.TitleTemplate, ntd)
if err != nil {
return nil, err
at.logger.WarnContext(ctx, "custom title template failed, falling back to default template", slog.String("rule.id", ntd.Rule.ID), errors.Attr(err))
title, titleMissingVars = "", nil
}
// if title template results in empty string, use default template
// this happens for rules where custom title annotation was not set
if title == "" && req.DefaultTitleTemplate != "" {
// or failed to render
isDefaultTitle := title == ""
if isDefaultTitle && req.DefaultTitleTemplate != "" {
title, err = at.expandDefaultTemplate(ctx, req.DefaultTitleTemplate, alerts)
if err != nil {
return nil, err
@@ -52,10 +55,12 @@ func (at *templater) Expand(
isDefaultBody := false
body, bodyMissingVars, err := at.expandBody(req.BodyTemplate, ntd)
if err != nil {
return nil, err
at.logger.WarnContext(ctx, "custom body template failed, falling back to default template", slog.String("rule.id", ntd.Rule.ID), errors.Attr(err))
body, bodyMissingVars = nil, nil
}
// if body template results in nil, use default template
// this happens for rules where custom body annotation was not set
// or failed to render
if body == nil {
isDefaultBody = true
defaultBody, err := at.expandDefaultTemplate(ctx, req.DefaultBodyTemplate, alerts)
@@ -78,6 +83,7 @@ func (at *templater) Expand(
Title: title,
Body: body,
MissingVars: missingVarsList,
IsDefaultTitle: isDefaultTitle,
IsDefaultBody: isDefaultBody,
NotificationData: ntd,
}, nil

View File

@@ -54,14 +54,15 @@ func TestExpandTemplates(t *testing.T) {
at, ctx := testSetup(t)
tests := []struct {
name string
alerts []*types.Alert
input alertmanagertypes.ExpandRequest
wantTitle string
wantBody []string
wantMissingVars []string
errorContains string
wantIsDefaultBody bool
name string
alerts []*types.Alert
input alertmanagertypes.ExpandRequest
wantTitle string
wantBody []string
wantMissingVars []string
errorContains string
wantIsDefaultTitle bool
wantIsDefaultBody bool
}{
{
// High request throughput on a service — service is a custom label.
@@ -155,7 +156,8 @@ Description: Request rate exceeded 10k/s`},
" • *severity:* critical\n" +
" \n" +
" "},
wantIsDefaultBody: true,
wantIsDefaultTitle: true,
wantIsDefaultBody: true,
},
{
// Pod crash loop on multiple pods — body is expanded once per alert
@@ -238,9 +240,10 @@ Description: Request rate exceeded 10k/s`},
DefaultTitleTemplate: "{{ .CommonLabels.alertname }}",
DefaultBodyTemplate: `{{ range .Alerts }}{{ template "signoz.relatedLinks.slack" . }}{{ end }}`,
},
wantTitle: "HighLatency",
wantBody: []string{"<https://signoz.example/traces?q=2|View Related Traces>"},
wantIsDefaultBody: true,
wantTitle: "HighLatency",
wantBody: []string{"<https://signoz.example/traces?q=2|View Related Traces>"},
wantIsDefaultTitle: true,
wantIsDefaultBody: true,
},
{
// $environment is not a known AlertData or NotificationTemplateData field,
@@ -266,8 +269,9 @@ Description: Request rate exceeded 10k/s`},
input: alertmanagertypes.ExpandRequest{
BodyTemplate: "$rule.name: see runbook at $runbook_url",
},
wantBody: []string{"PodOOMKilled: see runbook at <no value>"},
wantMissingVars: []string{"runbook_url"},
wantBody: []string{"PodOOMKilled: see runbook at <no value>"},
wantMissingVars: []string{"runbook_url"},
wantIsDefaultTitle: true,
},
{
// Both title and body use unknown variables; MissingVars is the union of both.
@@ -295,17 +299,68 @@ Description: Request rate exceeded 10k/s`},
DefaultTitleTemplate: "{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})",
DefaultBodyTemplate: "Runbook: https://runbook.example.com",
},
wantTitle: "HighCPU (RESOLVED)",
wantBody: []string{"Runbook: https://runbook.example.com"},
wantIsDefaultBody: true,
wantTitle: "HighCPU (RESOLVED)",
wantBody: []string{"Runbook: https://runbook.example.com"},
wantIsDefaultTitle: true,
wantIsDefaultBody: true,
},
{
name: "using non-existing function in template",
// A custom template that does not render must not cost the
// notification; the channel default stands in for that half.
name: "fallback: custom title using non-existing function falls back to default",
alerts: []*types.Alert{
createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
},
input: alertmanagertypes.ExpandRequest{
TitleTemplate: "$rule.name ({{$severity | toUpperAndTrim}}) for $alertname",
TitleTemplate: "$rule.name ({{$severity | toUpperAndTrim}}) for $alertname",
DefaultTitleTemplate: "{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})",
DefaultBodyTemplate: "Runbook: https://runbook.example.com",
},
wantTitle: "HighCPU (FIRING)",
wantBody: []string{"Runbook: https://runbook.example.com"},
wantIsDefaultTitle: true,
wantIsDefaultBody: true,
},
{
// A title copied from a channel template reads .CommonLabels, which
// the rule data model does not carry.
name: "fallback: channel-style custom title falls back to default",
alerts: []*types.Alert{
createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
},
input: alertmanagertypes.ExpandRequest{
TitleTemplate: "Rule title: {{ .CommonLabels.alertname }}",
BodyTemplate: "$rule.name is $alert.status",
DefaultTitleTemplate: "[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}",
DefaultBodyTemplate: "Runbook: https://runbook.example.com",
},
wantTitle: "[FIRING] HighCPU",
wantBody: []string{"HighCPU is firing"},
wantIsDefaultTitle: true,
},
{
name: "fallback: channel-style custom body falls back to default",
alerts: []*types.Alert{
createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, map[string]string{"summary": "CPU above 90%"}, true),
},
input: alertmanagertypes.ExpandRequest{
TitleTemplate: "$rule.name",
BodyTemplate: "{{ .Labels.alertname }}: {{ .Annotations.summary }}",
DefaultTitleTemplate: "[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}",
DefaultBodyTemplate: "{{ range .Alerts }}{{ .Labels.alertname }}: {{ .Annotations.summary }}{{ end }}",
},
wantTitle: "HighCPU",
wantBody: []string{"HighCPU: CPU above 90%"},
wantIsDefaultBody: true,
},
{
// The channel's own template has no fallback left.
name: "default title template using non-existing function fails",
alerts: []*types.Alert{
createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
},
input: alertmanagertypes.ExpandRequest{
DefaultTitleTemplate: "{{ .CommonLabels.alertname | toUpperAndTrim }}",
},
errorContains: "function \"toUpperAndTrim\" not defined",
},
@@ -326,6 +381,7 @@ Description: Request rate exceeded 10k/s`},
if tc.wantBody != nil {
require.Equal(t, tc.wantBody, got.Body)
}
require.Equal(t, tc.wantIsDefaultTitle, got.IsDefaultTitle)
require.Equal(t, tc.wantIsDefaultBody, got.IsDefaultBody)
if len(tc.wantMissingVars) == 0 {

View File

@@ -37,7 +37,7 @@ func FilterPublicAnnotations(kv template.KV) template.KV {
// ExpandRequest carries the title/body templates and their defaults handed to
// Templater.Expand. Default templates are used when the custom templates
// expand to empty strings.
// expand to empty strings or fail to render.
type ExpandRequest struct {
TitleTemplate string
BodyTemplate string
@@ -52,8 +52,13 @@ type ExpandResult struct {
// Body is the expanded notification body, one entry per input alert. The
// body template is applied per-alert and concatenated by the caller.
Body []string
// IsDefaultTitle is true when Title came from the default template (no
// user-authored title was supplied, it expanded to nothing, or it failed
// to render), false when a custom template was used.
IsDefaultTitle bool
// IsDefaultBody is true when Body came from the default template (no
// user-authored body was supplied), false when a custom template was used.
// user-authored body was supplied, or it failed to render), false when a
// custom template was used.
IsDefaultBody bool
// MissingVars is the union of $-references in the title and body templates
// that did not resolve to any known field. Surfaced for preview warnings;

View File

@@ -0,0 +1,73 @@
{
"alert": "content_templating_data_dependent",
"ruleType": "threshold_rule",
"alertType": "METRIC_BASED_ALERT",
"condition": {
"thresholds": {
"kind": "basic",
"spec": [
{
"name": "critical",
"target": 100,
"matchType": "1",
"op": "1",
"channels": [
"test channel"
]
}
]
},
"compositeQuery": {
"queryType": "builder",
"panelType": "graph",
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "metrics",
"aggregations": [
{
"metricName": "container_memory_bytes_content_templating",
"timeAggregation": "avg",
"spaceAggregation": "max"
}
],
"groupBy": [
{"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
]
}
}
]
},
"selectedQueryName": "A"
},
"evaluation": {
"kind": "rolling",
"spec": {
"evalWindow": "5m0s",
"frequency": "15s"
}
},
"labels": {},
"annotations": {
"description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
"summary": "High container memory in $namespace/$pod",
"_title_template": "{{ slice .labels.pod 0 40 }} memory high",
"_body_template": "Pod {{ slice .labels.pod 0 40 }} in {{ .labels.namespace }} exceeded memory threshold"
},
"notificationSettings": {
"groupBy": [],
"usePolicy": false,
"renotify": {
"enabled": false,
"interval": "30m",
"alertStates": []
}
},
"schemaVersion": "v2alpha1"
}

View File

@@ -71,6 +71,50 @@ CONTENT_TEMPLATING_TEST = [
],
),
),
types.AlertManagerNotificationTestCase(
# The rule's templates slice 40 characters off a pod label that is 24 long.
name="msteams_metrics_rule_templates_failing_on_data_fall_back_to_channel",
rule_path="alertmanager/content_templating/metrics_rule_fallback.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alertmanager/content_templating/metrics_data.jsonl",
),
],
channel_config=msteams_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
wait_time_seconds=120,
notification_validations=[
types.NotificationValidation(
destination_type="webhook",
validation_data={
"path": "/msteams/webhook_url",
"json_body": {
"type": "message",
"attachments": [
{
"contentType": "application/vnd.microsoft.card.adaptive",
"content": {
"type": "AdaptiveCard",
"body": [
{
"type": "TextBlock",
"text": re.compile(
r'\[FIRING:1\] content_templating_data_dependent for \(alertname="content_templating_data_dependent", container="checkout", namespace="production", node="ip-10-0-1-23", pod="checkout-7d9c8b5f4-x2k9p", ruleSource="http://localhost:8080/alerts/overview\?ruleId=[0-9a-f-]+", severity="critical", threshold\.name="critical"\)'
),
},
{"type": "TextBlock", "text": "Alerts"},
],
},
}
],
},
},
),
],
),
),
types.AlertManagerNotificationTestCase(
name="opsgenie_metrics_default_templating",
rule_path="alertmanager/content_templating/metrics_rule.json",