Compare commits

..

8 Commits

Author SHA1 Message Date
Nikhil Soni
0bd9889226 chore: fix comment 2026-05-26 01:40:33 +05:30
Nikhil Soni
51da3e0d72 refactor: use sqlbuider for queries 2026-05-25 23:49:10 +05:30
Nikhil Soni
a004ba8d06 chore: update openapi specs 2026-05-25 21:07:43 +05:30
Nikhil Soni
e41b46bbb4 refactor: move conversion logic to types 2026-05-25 21:05:53 +05:30
Nikhil Soni
15ac97f49f chore: avoid unnecessary diffs 2026-05-25 21:05:53 +05:30
Nikhil Soni
f6971c8f9f refactor: keep the waterfall changes in new api version
This is to avoid the contract change in existing v3
2026-05-25 21:05:53 +05:30
Nikhil Soni
72c65d7dd9 feat: break down waterfall module to handle large spans
Handling large traces in two steps to avoid high
memory allocation
2026-05-25 21:05:53 +05:30
Nikhil Soni
7a88dbabdd feat: add store methods for minimal trace fetch 2026-05-25 21:05:53 +05:30
68 changed files with 747 additions and 3161 deletions

View File

@@ -39,7 +39,6 @@ jobs:
matrix:
suite:
- alerts
- alertmanager
- callbackauthn
- cloudintegrations
- dashboard

View File

@@ -11,7 +11,7 @@ RUN apk update && \
COPY ./target/${OS}-${TARGETARCH}/signoz-community /root/signoz
COPY ./templates /root/templates
COPY ./templates/email /root/templates
COPY frontend/build/ /etc/signoz/web/
RUN chmod 755 /root /root/signoz

View File

@@ -12,7 +12,7 @@ RUN apk update && \
rm -rf /var/cache/apk/*
COPY ./target/${OS}-${ARCH}/signoz-community /root/signoz-community
COPY ./templates /root/templates
COPY ./templates/email /root/templates
COPY frontend/build/ /etc/signoz/web/
RUN chmod 755 /root /root/signoz-community

View File

@@ -11,7 +11,7 @@ RUN apk update && \
COPY ./target/${OS}-${TARGETARCH}/signoz /root/signoz
COPY ./templates /root/templates
COPY ./templates/email /root/templates
COPY frontend/build/ /etc/signoz/web/
RUN chmod 755 /root /root/signoz

View File

@@ -26,7 +26,7 @@ RUN go mod download
COPY ./cmd/ ./cmd/
COPY ./ee/ ./ee/
COPY ./pkg/ ./pkg/
COPY ./templates /root/templates
COPY ./templates/email /root/templates
COPY Makefile Makefile
RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race

View File

@@ -12,7 +12,7 @@ RUN apk update && \
rm -rf /var/cache/apk/*
COPY ./target/${OS}-${ARCH}/signoz /root/signoz
COPY ./templates /root/templates
COPY ./templates/email /root/templates
COPY frontend/build/ /etc/signoz/web/
RUN chmod 755 /root /root/signoz

View File

@@ -35,7 +35,7 @@ RUN go mod download
COPY ./cmd/ ./cmd/
COPY ./ee/ ./ee/
COPY ./pkg/ ./pkg/
COPY ./templates /root/templates
COPY ./templates/email /root/templates
COPY Makefile Makefile
RUN TARGET_DIR=/root ARCHS=${TARGETARCH} ZEUS_URL=${ZEUSURL} LICENSE_URL=${ZEUSURL}/api/v1 make go-build-enterprise-race

View File

@@ -182,11 +182,6 @@ alertmanager:
poll_interval: 1m
# The URL under which Alertmanager is externally reachable (for example, if Alertmanager is served via a reverse proxy). Used for generating relative and absolute links back to Alertmanager itself.
external_url: http://localhost:8080
# The list of globs from which SigNoz's alertmanager notification templates are loaded (e.g. the email.signoz.html layout).
# This mirrors the upstream alertmanager `templates` config option. The upstream default templates (default.tmpl, email.tmpl)
# are always loaded from the embedded alertmanager assets, so only SigNoz's own templates need to be listed here.
templates:
- /opt/signoz/conf/templates/alertmanager/*.gotmpl
# The global configuration for the alertmanager. All the exahustive fields can be found in the upstream: https://github.com/prometheus/alertmanager/blob/efa05feffd644ba4accb526e98a8c6545d26a783/config/config.go#L833
global:
# ResolveTimeout is the time after which an alert is declared resolved if it has not been updated.

View File

@@ -18948,6 +18948,77 @@ paths:
summary: Get waterfall view for a trace
tags:
- tracedetail
/api/v4/traces/{traceID}/waterfall:
post:
deprecated: false
description: 'Two-step fetch: minimal fields for all spans to build the tree,
full fields only for the visible window. Aggregations are not included in
the response.'
operationId: GetWaterfallV4
parameters:
- in: path
name: traceID
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/SpantypesPostableWaterfall'
responses:
"200":
content:
application/json:
schema:
properties:
data:
$ref: '#/components/schemas/SpantypesGettableWaterfallTrace'
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"404":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Not Found
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Get waterfall view for a trace (OOM-safe)
tags:
- tracedetail
/api/v5/query_range:
post:
deprecated: false

View File

@@ -9232,6 +9232,17 @@ export type GetWaterfall200 = {
status: string;
};
export type GetWaterfallV4PathParameters = {
traceID: string;
};
export type GetWaterfallV4200 = {
data: SpantypesGettableWaterfallTraceDTO;
/**
* @type string
*/
status: string;
};
export type QueryRangeV5200 = {
data: Querybuildertypesv5QueryRangeResponseDTO;
/**

View File

@@ -14,6 +14,8 @@ import type {
import type {
GetWaterfall200,
GetWaterfallPathParameters,
GetWaterfallV4200,
GetWaterfallV4PathParameters,
RenderErrorResponseDTO,
SpantypesPostableWaterfallDTO,
} from '../sigNoz.schemas';
@@ -120,3 +122,102 @@ export const useGetWaterfall = <
> => {
return useMutation(getGetWaterfallMutationOptions(options));
};
/**
* Two-step fetch: minimal fields for all spans to build the tree, full fields only for the visible window. Aggregations are not included in the response.
* @summary Get waterfall view for a trace (OOM-safe)
*/
export const getWaterfallV4 = (
{ traceID }: GetWaterfallV4PathParameters,
spantypesPostableWaterfallDTO?: BodyType<SpantypesPostableWaterfallDTO>,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<GetWaterfallV4200>({
url: `/api/v4/traces/${traceID}/waterfall`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: spantypesPostableWaterfallDTO,
signal,
});
};
export const getGetWaterfallV4MutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof getWaterfallV4>>,
TError,
{
pathParams: GetWaterfallV4PathParameters;
data?: BodyType<SpantypesPostableWaterfallDTO>;
},
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof getWaterfallV4>>,
TError,
{
pathParams: GetWaterfallV4PathParameters;
data?: BodyType<SpantypesPostableWaterfallDTO>;
},
TContext
> => {
const mutationKey = ['getWaterfallV4'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof getWaterfallV4>>,
{
pathParams: GetWaterfallV4PathParameters;
data?: BodyType<SpantypesPostableWaterfallDTO>;
}
> = (props) => {
const { pathParams, data } = props ?? {};
return getWaterfallV4(pathParams, data);
};
return { mutationFn, ...mutationOptions };
};
export type GetWaterfallV4MutationResult = NonNullable<
Awaited<ReturnType<typeof getWaterfallV4>>
>;
export type GetWaterfallV4MutationBody =
| BodyType<SpantypesPostableWaterfallDTO>
| undefined;
export type GetWaterfallV4MutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Get waterfall view for a trace (OOM-safe)
*/
export const useGetWaterfallV4 = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof getWaterfallV4>>,
TError,
{
pathParams: GetWaterfallV4PathParameters;
data?: BodyType<SpantypesPostableWaterfallDTO>;
},
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof getWaterfallV4>>,
TError,
{
pathParams: GetWaterfallV4PathParameters;
data?: BodyType<SpantypesPostableWaterfallDTO>;
},
TContext
> => {
return useMutation(getGetWaterfallV4MutationOptions(options));
};

View File

@@ -10,6 +10,9 @@ export const DEFAULT_AUTH0_APP_REDIRECTION_PATH = ROUTES.APPLICATION;
export const INVITE_MEMBERS_HASH = '#invite-team-members';
export const SIGNOZ_UPGRADE_PLAN_URL =
'https://upgrade.signoz.io/upgrade-from-app';
export const DASHBOARD_TIME_IN_DURATION = 'refreshInterval';
export const DEFAULT_ENTITY_VERSION = 'v3';

View File

@@ -1,3 +1,4 @@
import { SIGNOZ_UPGRADE_PLAN_URL } from 'constants/app';
import CreateAlertChannels from 'container/CreateAlertChannels';
import { ChannelType } from 'container/CreateAlertChannels/config';
import {
@@ -312,6 +313,16 @@ describe('Create Alert Channel (Normal User)', () => {
expect(screen.getByText('Microsoft Teams')).toBeInTheDocument();
});
it.skip('Should check if the upgrade plan message is shown', () => {
expect(screen.getByText('Upgrade to a Paid Plan')).toBeInTheDocument();
expect(
screen.getByText(/This feature is available for paid plans only./),
).toBeInTheDocument();
const link = screen.getByRole('link', { name: 'Click here' });
expect(link).toBeInTheDocument();
expect(link).toHaveAttribute('href', SIGNOZ_UPGRADE_PLAN_URL);
expect(screen.getByText(/to Upgrade/)).toBeInTheDocument();
});
it('Should check if the form buttons are displayed properly (Save, Test, Back)', () => {
expect(
screen.getByRole('button', { name: 'button_save_channel' }),

View File

@@ -81,20 +81,6 @@
}
}
.alert-rule-scope {
margin-bottom: 12px;
.ant-radio-wrapper {
color: var(--l1-foreground);
}
}
.alert-rule-all-warning {
font-size: 12px;
font-weight: 400;
color: var(--l2-foreground);
}
.formItemWithBullet {
margin-bottom: 0;
}

View File

@@ -8,7 +8,6 @@ import {
FormInstance,
Input,
Modal,
Radio,
Select,
SelectProps,
Spin,
@@ -72,14 +71,11 @@ const TZ_OPTIONS: DefaultOptionType[] = ALL_TIME_ZONES.map(
}),
);
type AlertRuleScope = 'all' | 'specific';
interface PlannedDowntimeFormData {
name: string;
startTime: dayjs.Dayjs | null;
endTime: dayjs.Dayjs | null;
recurrence?: AlertmanagertypesRecurrenceDTO;
alertRuleScope: AlertRuleScope;
alertRules: DefaultOptionType[];
recurrenceSelect?: AlertmanagertypesRecurrenceDTO;
timezone?: string;
@@ -133,12 +129,6 @@ export function PlannedDowntimeForm(
recurrenceOptions.doesNotRepeat.value,
);
const [alertRuleScope, setAlertRuleScope] = useState<AlertRuleScope>(
initialValues.id && (initialValues.alertIds || []).length === 0
? 'all'
: 'specific',
);
const { notifications } = useNotifications();
const { showErrorModal } = useErrorModal();
@@ -152,12 +142,9 @@ export function PlannedDowntimeForm(
const saveHandler = useCallback(
async (values: PlannedDowntimeFormData) => {
const data: AlertmanagertypesPostablePlannedMaintenanceDTO = {
alertIds:
values.alertRuleScope === 'all'
? []
: (values.alertRules
.map((alert) => alert.value)
.filter((alert) => alert !== undefined) as string[]),
alertIds: values.alertRules
.map((alert) => alert.value)
.filter((alert) => alert !== undefined) as string[],
name: values.name,
scope: values.scope,
schedule: {
@@ -278,13 +265,12 @@ export function PlannedDowntimeForm(
const startTime = schedule?.recurrence?.startTime || schedule?.startTime;
const endTime = schedule?.recurrence?.endTime || schedule?.endTime;
const initialAlertIds = initialValues.alertIds || [];
return {
name: defaultTo(initialValues.name, ''),
alertRuleScope:
isEditMode && initialAlertIds.length === 0 ? 'all' : 'specific',
alertRules: getAlertOptionsFromIds(initialAlertIds, alertOptions),
alertRules: getAlertOptionsFromIds(
initialValues.alertIds || [],
alertOptions,
),
startTime: startTime ? dayjs(startTime).tz(schedule.timezone) : null,
endTime: endTime ? dayjs(endTime).tz(schedule.timezone) : null,
recurrence: {
@@ -301,7 +287,6 @@ export function PlannedDowntimeForm(
useEffect(() => {
setSelectedTags(formattedInitialValues.alertRules);
setAlertRuleScope(formattedInitialValues.alertRuleScope);
form.setFieldsValue({ ...formattedInitialValues });
}, [form, formattedInitialValues, initialValues]);
@@ -364,7 +349,6 @@ export function PlannedDowntimeForm(
onFinish={onFinish}
onValuesChange={(): void => {
setRecurrenceType(form.getFieldValue('recurrence')?.repeatType as string);
setAlertRuleScope(form.getFieldValue('alertRuleScope') as AlertRuleScope);
handleFormData(form.getFieldsValue());
}}
autoComplete="off"
@@ -464,76 +448,49 @@ export function PlannedDowntimeForm(
<div className="scheduleTimeInfoText">{endTimeText}</div>
)}
<div>
<Typography style={{ marginBottom: 8 }}>Silence Alerts</Typography>
<Form.Item
name="alertRuleScope"
initialValue="specific"
className="alert-rule-scope"
>
<Radio.Group>
<Radio value="all">All alert rules</Radio>
<Radio value="specific">Specific alert rules</Radio>
</Radio.Group>
<div className="alert-rule-form">
<Typography style={{ marginBottom: 8 }}>Silence Alerts</Typography>
<Typography style={{ marginBottom: 8 }} className="alert-rule-info">
(Leave empty to silence all alerts)
</Typography>
</div>
<Form.Item noStyle shouldUpdate>
<AlertRuleTags
closable
selectedTags={selectedTags}
handleClose={handleClose}
/>
</Form.Item>
<Form.Item name={alertRuleFormName}>
<Select
placeholder="Search for alerts rules or groups..."
mode="multiple"
status={isError ? 'error' : undefined}
loading={isLoading}
tagRender={noTagRenderer}
onChange={handleAlertRulesChange}
showSearch
options={alertOptions}
filterOption={(input, option): boolean =>
(option?.label as string)?.toLowerCase()?.includes(input.toLowerCase())
}
notFoundContent={
isLoading ? (
<span>
<Spin size="small" /> Loading...
</span>
) : (
<span>No alert available.</span>
)
}
>
{alertOptions?.map((option) => (
<Select.Option key={option.value} value={option.value}>
{option.label}
</Select.Option>
))}
</Select>
</Form.Item>
{alertRuleScope === 'specific' && (
<>
<Form.Item noStyle shouldUpdate>
<AlertRuleTags
closable
selectedTags={selectedTags}
handleClose={handleClose}
/>
</Form.Item>
<Form.Item
name={alertRuleFormName}
rules={[
{
validator: async (
_rule,
value: DefaultOptionType[] | undefined,
): Promise<void> => {
if (!value || value.length === 0) {
throw new Error(
'Select at least one alert rule, or choose "All alert rules" to silence everything.',
);
}
},
},
]}
>
<Select
placeholder="Search for alert rules or groups..."
mode="multiple"
status={isError ? 'error' : undefined}
loading={isLoading}
tagRender={noTagRenderer}
onChange={handleAlertRulesChange}
showSearch
options={alertOptions}
filterOption={(input, option): boolean =>
(option?.label as string)
?.toLowerCase()
?.includes(input.toLowerCase())
}
notFoundContent={
isLoading ? (
<span>
<Spin size="small" /> Loading...
</span>
) : (
<span>No alert available.</span>
)
}
>
{alertOptions?.map((option) => (
<Select.Option key={option.value} value={option.value}>
{option.label}
</Select.Option>
))}
</Select>
</Form.Item>
</>
)}
</div>
<Form.Item
label={

View File

@@ -204,7 +204,7 @@ export function CollapseListContent({
selectedTags={alertOptions}
/>
) : (
<Tag className="all-alerts-tag">All alert rules</Tag>
'-'
),
)}
</Flex>

View File

@@ -23,13 +23,7 @@ import (
"sync"
"time"
htmltemplate "html/template"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/alertmanager/config"
@@ -40,39 +34,20 @@ import (
const (
Integration = "email"
// alertEmailLayoutTemplate is the name of the HTML layout template that
// wraps the rendered alert bodies. It is loaded into the notification
// template (n.tmpl) from the alertmanager templates config and lives at
// templates/alertmanager/email.gotmpl.
alertEmailLayoutTemplate = "email.signoz.html"
)
// Email implements a Notifier for email notifications.
type Email struct {
conf *config.EmailConfig
tmpl *template.Template
logger *slog.Logger
hostname string
templater alertmanagertypes.Templater
}
// layoutData is the value passed to the email.signoz.html layout
// template. It embeds NotificationTemplateData so templates can reference
// `.Alert.Status`, `.Alert.TotalFiring`, `.Alert.TotalResolved`,
// `.NotificationTemplateData.ExternalURL`, etc. alongside the rendered
// Title and per-alert Bodies.
type layoutData struct {
alertmanagertypes.NotificationTemplateData
Title string
Bodies []htmltemplate.HTML
conf *config.EmailConfig
tmpl *template.Template
logger *slog.Logger
hostname string
}
var errNoAuthUsernameConfigured = errors.NewInternalf(errors.CodeInternal, "no auth username configured")
// New returns a new Email notifier. When the email.signoz.html layout is
// not defined in t, custom-body alerts fall back to plain <div>-wrapped HTML.
func New(c *config.EmailConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater) *Email {
// New returns a new Email notifier.
func New(c *config.EmailConfig, t *template.Template, l *slog.Logger) *Email {
if _, ok := c.Headers["Subject"]; !ok {
c.Headers["Subject"] = config.DefaultEmailSubject
}
@@ -88,7 +63,7 @@ func New(c *config.EmailConfig, t *template.Template, l *slog.Logger, templater
if err != nil {
h = "localhost.localdomain"
}
return &Email{conf: c, tmpl: t, logger: l, hostname: h, templater: templater}
return &Email{conf: c, tmpl: t, logger: l, hostname: h}
}
// auth resolves a string of authentication mechanisms.
@@ -224,9 +199,9 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
if ok, mech := c.Extension("AUTH"); ok {
auth, err := n.auth(mech)
if err != nil && !errors.Is(err, errNoAuthUsernameConfigured) {
if err != nil && err != errNoAuthUsernameConfigured {
return true, errors.WrapInternalf(err, errors.CodeInternal, "find auth mechanism")
} else if errors.Is(err, errNoAuthUsernameConfigured) {
} else if err == errNoAuthUsernameConfigured {
n.logger.DebugContext(ctx, "no auth username configured. Attempting to send email without authenticating")
}
if auth != nil {
@@ -270,16 +245,6 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
}
}
// Prepare the content for the email. subject, when non-empty, overrides
// the configured Subject header for this notification only. We deliberately
// do not mutate n.conf.Headers here: the config map is shared across
// concurrent notifications to the same receiver.
subject, htmlBody, err := n.prepareContent(ctx, as)
if err != nil {
n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
return false, err
}
// Send the email headers and body.
message, err := c.Data()
if err != nil {
@@ -297,10 +262,6 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
buffer := &bytes.Buffer{}
for header, t := range n.conf.Headers {
if header == "Subject" {
fmt.Fprintf(buffer, "%s: %s\r\n", header, mime.QEncoding.Encode("utf-8", subject))
continue
}
value, err := n.tmpl.ExecuteTextString(t, data)
if err != nil {
return false, errors.WrapInternalf(err, errors.CodeInternal, "execute %q header template", header)
@@ -375,7 +336,7 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
}
}
if htmlBody != "" {
if len(n.conf.HTML) > 0 {
// Html template
// Preferred alternative placed last per section 5.1.4 of RFC 2046
// https://www.ietf.org/rfc/rfc2046.txt
@@ -386,8 +347,12 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
if err != nil {
return false, errors.WrapInternalf(err, errors.CodeInternal, "create part for html template")
}
body, err := n.tmpl.ExecuteHTMLString(n.conf.HTML, data)
if err != nil {
return false, errors.WrapInternalf(err, errors.CodeInternal, "execute html template")
}
qw := quotedprintable.NewWriter(w)
_, err = qw.Write([]byte(htmlBody))
_, err = qw.Write([]byte(body))
if err != nil {
return true, errors.WrapInternalf(err, errors.CodeInternal, "write HTML part")
}
@@ -416,124 +381,6 @@ func (n *Email) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
return false, nil
}
// prepareContent returns a subject override (empty when the default config
// Subject should be used) and the HTML body for the email. Callers must treat
// the subject as local state and never write it back to n.conf.Headers.
func (n *Email) prepareContent(ctx context.Context, alerts []*types.Alert) (string, string, error) {
customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
TitleTemplate: customTitle,
BodyTemplate: customBody,
DefaultTitleTemplate: n.conf.Headers["Subject"],
DefaultBodyTemplate: n.conf.HTML,
}, alerts)
if err != nil {
return "", "", err
}
subject := result.Title
if !result.IsDefaultBody {
// Custom-body path: render each expanded markdown body to HTML, then
// wrap the whole thing in the email.signoz.html layout (or fall
// back to plain <div> wrapping when the layout template is not loaded).
for i, body := range result.Body {
if body == "" {
continue
}
rendered, err := markdownrenderer.RenderHTML(body)
if err != nil {
return "", "", err
}
result.Body[i] = rendered
}
appendRelatedLinkButtons(alerts, result.Body)
html, err := n.renderLayout(result)
if err != nil {
n.logger.WarnContext(ctx, "custom email template rendering failed, falling back to plain <div> wrap", errors.Attr(err))
return subject, wrapBodiesAsDivs(result.Body), nil
}
return subject, html, nil
}
return subject, result.Body[0], nil
}
// renderLayout wraps result in the email.signoz.html HTML layout loaded
// into n.tmpl from the alertmanager templates config. Returns an error when the
// layout template is not defined (e.g. in tests where no templates are loaded)
// so prepareContent can fall back to plain <div> wrapping.
func (n *Email) renderLayout(result *alertmanagertypes.ExpandResult) (string, error) {
bodies := make([]htmltemplate.HTML, 0, len(result.Body))
for _, b := range result.Body {
bodies = append(bodies, htmltemplate.HTML(b))
}
data := layoutData{Title: result.Title, Bodies: bodies}
if result.NotificationData != nil {
data.NotificationTemplateData = *result.NotificationData
}
html, err := n.tmpl.ExecuteHTMLString(`{{ template "`+alertEmailLayoutTemplate+`" . }}`, data)
if err != nil {
return "", errors.WrapInternalf(err, errors.CodeInternal, "failed to render email layout")
}
return html, nil
}
// appendRelatedLinkButtons appends "View Related Logs/Traces" buttons to each
// per-alert body when the rule manager attached the corresponding annotation.
// bodies is positionally aligned with alerts (see alertmanagertemplate.Prepare);
// empty bodies are skipped so we never attach a button to an alert that produced
// no visible content.
func appendRelatedLinkButtons(alerts []*types.Alert, bodies []string) {
for i := range bodies {
if i >= len(alerts) || bodies[i] == "" {
continue
}
if link := alerts[i].Annotations[ruletypes.AnnotationRelatedLogs]; link != "" {
bodies[i] += htmlButton("View Related Logs", string(link))
}
if link := alerts[i].Annotations[ruletypes.AnnotationRelatedTraces]; link != "" {
bodies[i] += htmlButton("View Related Traces", string(link))
}
}
}
func wrapBodiesAsDivs(bodies []string) string {
var b strings.Builder
for _, part := range bodies {
if part == "" {
continue
}
b.WriteString("<div>")
b.WriteString(part)
b.WriteString("</div>")
}
return b.String()
}
func htmlButton(text, url string) string {
return fmt.Sprintf(`
<a href="%s" target="_blank" style="text-decoration: none;">
<button style="
padding: 6px 16px;
/* Default System Font */
font-family: sans-serif;
font-size: 14px;
font-weight: 500;
line-height: 1.5;
/* Light Theme & Dynamic Background (Solid) */
color: #111827;
background-color: #f9fafb;
/* Static Outline */
border: 1px solid #d1d5db;
border-radius: 4px;
cursor: pointer;
">
%s
</button>
</a>`, url, text)
}
type loginAuth struct {
username, password string
}

View File

@@ -8,7 +8,6 @@ import (
"context"
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
@@ -18,10 +17,7 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
"github.com/emersion/go-smtp"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
@@ -46,11 +42,6 @@ const (
emailFrom = "alertmanager@example.com"
)
// testTemplater returns a Templater bound to tmpl with a discard logger.
func testTemplater(tmpl *template.Template) alertmanagertypes.Templater {
return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
}
// email represents an email returned by the MailDev REST API.
// See https://github.com/djfarrelly/MailDev/blob/master/docs/rest.md.
type email struct {
@@ -171,7 +162,7 @@ func notifyEmailWithContext(ctx context.Context, t *testing.T, cfg *config.Email
return nil, false, err
}
email := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl))
email := New(cfg, tmpl, promslog.NewNopLogger())
retry, err := email.Notify(ctx, firingAlert)
if err != nil {
@@ -715,7 +706,7 @@ func TestEmailRejected(t *testing.T) {
tmpl, firingAlert, err := prepare(cfg)
require.NoError(t, err)
e := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl))
e := New(cfg, tmpl, promslog.NewNopLogger())
// Send the alert to mock SMTP server.
retry, err := e.Notify(context.Background(), firingAlert)
@@ -1039,135 +1030,6 @@ func TestEmailImplicitTLS(t *testing.T) {
}
}
func TestPrepareContent(t *testing.T) {
t.Run("default title template; custom body template", func(t *testing.T) {
tmpl, err := template.FromGlobs([]string{})
require.NoError(t, err)
tmpl.ExternalURL, _ = url.Parse("http://am")
bodyTpl := "line $labels.instance"
a1 := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{
model.LabelName("instance"): model.LabelValue("one"),
},
Annotations: model.LabelSet{
model.LabelName(ruletypes.AnnotationBodyTemplate): model.LabelValue(bodyTpl),
},
},
}
a2 := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{
model.LabelName("instance"): model.LabelValue("two"),
},
Annotations: model.LabelSet{
model.LabelName(ruletypes.AnnotationBodyTemplate): model.LabelValue(bodyTpl),
},
},
}
alerts := []*types.Alert{a1, a2}
cfg := &config.EmailConfig{Headers: map[string]string{"Subject": "subj"}}
n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl))
ctx := context.Background()
subject, htmlBody, err := n.prepareContent(ctx, alerts)
require.NoError(t, err)
require.Equal(t, "subj", subject)
require.Equal(t, "<div><p>line one</p>\n</div><div><p>line two</p>\n</div>", htmlBody)
})
t.Run("custom title template; default body HTML template", func(t *testing.T) {
tmpl, err := template.FromGlobs([]string{})
require.NoError(t, err)
tmpl.ExternalURL, _ = url.Parse("http://am")
firingAlert := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{},
Annotations: model.LabelSet{
model.LabelName(ruletypes.AnnotationTitleTemplate): model.LabelValue("fixed from $alert.status"),
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
}
alerts := []*types.Alert{firingAlert}
cfg := &config.EmailConfig{
Headers: map[string]string{},
HTML: "Status: {{ .Status }}",
}
n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl))
ctx := context.Background()
subject, htmlBody, err := n.prepareContent(ctx, alerts)
require.NoError(t, err)
require.Equal(t, "Status: firing", htmlBody)
require.Equal(t, "fixed from firing", subject)
})
t.Run("default template without HTML", func(t *testing.T) {
cfg := &config.EmailConfig{Headers: map[string]string{"Subject": "the email subject"}}
tmpl, err := template.FromGlobs([]string{})
require.NoError(t, err)
tmpl.ExternalURL, _ = url.Parse("http://am")
n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl))
firingAlert := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
}
alerts := []*types.Alert{firingAlert}
ctx := context.Background()
subject, htmlBody, err := n.prepareContent(ctx, alerts)
require.NoError(t, err)
require.Equal(t, "", htmlBody)
require.Equal(t, "the email subject", subject)
})
t.Run("custom title template; custom body template", func(t *testing.T) {
// Load the email.signoz.html layout into the notification template
// the same way the alertmanager server does via the templates config.
tmpl, err := template.FromGlobs([]string{"../../../../templates/alertmanager/*.gotmpl"})
require.NoError(t, err)
tmpl.ExternalURL, _ = url.Parse("http://am")
firingAlert := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{
model.LabelName("instance"): model.LabelValue("two"),
},
Annotations: model.LabelSet{
model.LabelName(ruletypes.AnnotationTitleTemplate): model.LabelValue("fixed from $alert.status"),
model.LabelName(ruletypes.AnnotationBodyTemplate): model.LabelValue("line $labels.instance"),
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
}
alerts := []*types.Alert{firingAlert}
cfg := &config.EmailConfig{
Headers: map[string]string{"Subject": "subject"},
HTML: "Well, what are you?",
}
n := New(cfg, tmpl, promslog.NewNopLogger(), testTemplater(tmpl))
ctx := context.Background()
subject, htmlBody, err := n.prepareContent(ctx, alerts)
require.NoError(t, err)
require.Contains(t, htmlBody, "<!DOCTYPE html>")
require.Contains(t, htmlBody, "<p>line two</p>")
require.NotContains(t, htmlBody, "Well, what are you?")
require.Equal(t, subject, "fixed from firing")
require.NotContains(t, subject, "subject")
})
}
func ptrTo(b bool) *bool {
return &b
}

View File

@@ -15,9 +15,7 @@ import (
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
@@ -46,7 +44,6 @@ type Notifier struct {
retrier *notify.Retrier
webhookURL *config.SecretURL
postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
templater alertmanagertypes.Templater
}
// https://learn.microsoft.com/en-us/connectors/teams/?tabs=text1#adaptivecarditemschema
@@ -55,7 +52,7 @@ type Content struct {
Type string `json:"type"`
Version string `json:"version"`
Body []Body `json:"body"`
Msteams Msteams `json:"msteams,omitzero"`
Msteams Msteams `json:"msteams,omitempty"`
Actions []Action `json:"actions"`
}
@@ -97,7 +94,7 @@ type teamsMessage struct {
}
// New returns a new notifier that uses the Microsoft Teams Power Platform connector.
func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
if err != nil {
return nil, err
@@ -112,7 +109,6 @@ func New(c *config.MSTeamsV2Config, t *template.Template, titleLink string, l *s
retrier: &notify.Retrier{},
webhookURL: c.WebhookURL,
postJSONFunc: notify.PostJSON,
templater: templater,
}
return n, nil
@@ -132,11 +128,25 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
return false, err
}
title := tmpl(n.conf.Title)
if err != nil {
return false, err
}
titleLink := tmpl(n.titleLink)
if err != nil {
return false, err
}
alerts := types.Alerts(as...)
color := colorGrey
switch alerts.Status() {
case model.AlertFiring:
color = colorRed
case model.AlertResolved:
color = colorGreen
}
var url string
if n.conf.WebhookURL != nil {
url = n.conf.WebhookURL.String()
@@ -148,12 +158,6 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
url = strings.TrimSpace(string(content))
}
bodyBlocks, err := n.prepareContent(ctx, as)
if err != nil {
n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
return false, err
}
// A message as referenced in https://learn.microsoft.com/en-us/connectors/teams/?tabs=text1%2Cdotnet#request-body-schema
t := teamsMessage{
Type: "message",
@@ -165,7 +169,17 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
Schema: "http://adaptivecards.io/schemas/adaptive-card.json",
Type: "AdaptiveCard",
Version: "1.2",
Body: bodyBlocks,
Body: []Body{
{
Type: "TextBlock",
Text: title,
Weight: "Bolder",
Size: "Medium",
Wrap: true,
Style: "heading",
Color: color,
},
},
Actions: []Action{
{
Type: "Action.OpenUrl",
@@ -181,6 +195,20 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
},
}
// add labels and annotations to the body of all alerts
for _, alert := range as {
t.Attachments[0].Content.Body = append(t.Attachments[0].Content.Body, Body{
Type: "TextBlock",
Text: "Alerts",
Weight: "Bolder",
Size: "Medium",
Wrap: true,
Color: color,
})
t.Attachments[0].Content.Body = append(t.Attachments[0].Content.Body, n.createLabelsAndAnnotationsBody(alert)...)
}
var payload bytes.Buffer
if err = json.NewEncoder(&payload).Encode(t); err != nil {
return false, err
@@ -200,75 +228,6 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
return shouldRetry, err
}
// prepareContent builds the Adaptive Card body blocks for the notification.
// The first block is always the title; the remainder depends on whether the
// alerts carried a custom body template.
func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) ([]Body, error) {
customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
TitleTemplate: customTitle,
BodyTemplate: customBody,
DefaultTitleTemplate: n.conf.Title,
DefaultBodyTemplate: n.conf.Text,
}, alerts)
if err != nil {
return nil, err
}
color := colorGrey
switch types.Alerts(alerts...).Status() {
case model.AlertFiring:
color = colorRed
case model.AlertResolved:
color = colorGreen
}
blocks := []Body{{
Type: "TextBlock",
Text: result.Title,
Weight: "Bolder",
Size: "Medium",
Wrap: true,
Style: "heading",
Color: color,
}}
if result.IsDefaultBody {
for _, alert := range alerts {
blocks = append(blocks, Body{
Type: "TextBlock",
Text: "Alerts",
Weight: "Bolder",
Size: "Medium",
Wrap: true,
Color: color,
})
blocks = append(blocks, n.createLabelsAndAnnotationsBody(alert)...)
}
return blocks, nil
}
// Custom body path: result.Body is positionally aligned with alerts;
// entries for alerts whose template rendered empty are kept as "" so we
// can skip them here without shifting the per-alert color index.
for i, body := range result.Body {
if body == "" || i >= len(alerts) {
continue
}
perAlertColor := colorRed
if alerts[i].Resolved() {
perAlertColor = colorGreen
}
blocks = append(blocks, Body{
Type: "TextBlock",
Text: body,
Wrap: true,
Color: perAlertColor,
})
}
return blocks, nil
}
func (*Notifier) createLabelsAndAnnotationsBody(alert *types.Alert) []Body {
bodies := []Body{}
bodies = append(bodies, Body{
@@ -299,8 +258,7 @@ func (*Notifier) createLabelsAndAnnotationsBody(alert *types.Alert) []Body {
annotationsFacts := []Fact{}
for k, v := range alert.Annotations {
if slices.Contains([]string{"summary", "related_logs", "related_traces"}, string(k)) ||
alertmanagertypes.IsPrivateAnnotation(string(k)) {
if slices.Contains([]string{"summary", "related_logs", "related_traces"}, string(k)) {
continue
}
annotationsFacts = append(annotationsFacts, Fact{Title: string(k), Value: string(v)})

View File

@@ -8,7 +8,6 @@ import (
"context"
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
@@ -16,9 +15,6 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
@@ -27,28 +23,21 @@ import (
test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
)
func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
}
// This is a test URL that has been modified to not be valid.
var testWebhookURL, _ = url.Parse("https://example.westeurope.logic.azure.com:443/workflows/xxx/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=xxx")
func TestMSTeamsV2Retry(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.MSTeamsV2Config{
WebhookURL: &config.SecretURL{URL: testWebhookURL},
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
`{{ template "msteamsv2.default.titleLink" . }}`,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -75,16 +64,14 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.MSTeamsV2Config{
WebhookURL: &config.SecretURL{URL: testWebhookURL},
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
`{{ template "msteamsv2.default.titleLink" . }}`,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -166,8 +153,7 @@ func TestMSTeamsV2Templating(t *testing.T) {
t.Run(tc.title, func(t *testing.T) {
tc.cfg.WebhookURL = &config.SecretURL{URL: u}
tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
tmpl := test.CreateTmpl(t)
pd, err := New(tc.cfg, tmpl, tc.titleLink, promslog.NewNopLogger(), newTestTemplater(tmpl))
pd, err := New(tc.cfg, test.CreateTmpl(t), tc.titleLink, promslog.NewNopLogger())
require.NoError(t, err)
ctx := context.Background()
@@ -200,124 +186,20 @@ func TestMSTeamsV2RedactedURL(t *testing.T) {
defer fn()
secret := "secret"
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.MSTeamsV2Config{
WebhookURL: &config.SecretURL{URL: u},
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
`{{ template "msteamsv2.default.titleLink" . }}`,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
test.AssertNotifyLeaksNoSecret(ctx, t, notifier, secret)
}
func TestPrepareContent(t *testing.T) {
t.Run("default template - firing alerts", func(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.MSTeamsV2Config{
WebhookURL: &config.SecretURL{URL: testWebhookURL},
HTTPConfig: &commoncfg.HTTPClientConfig{},
Title: "Alertname: {{ .CommonLabels.alertname }}",
},
tmpl,
`{{ template "msteamsv2.default.titleLink" . }}`,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
ctx := context.Background()
ctx = notify.WithGroupKey(ctx, "1")
alerts := []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "test"},
// Custom body template
Annotations: model.LabelSet{
ruletypes.AnnotationBodyTemplate: "Firing alert: $alertname",
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
},
}
blocks, err := notifier.prepareContent(ctx, alerts)
require.NoError(t, err)
require.NotEmpty(t, blocks)
// First block should be the title with color (firing = red)
require.Equal(t, "Bolder", blocks[0].Weight)
require.Equal(t, colorRed, blocks[0].Color)
// verify title text
require.Equal(t, "Alertname: test", blocks[0].Text)
// verify body text
require.Equal(t, "Firing alert: test", blocks[1].Text)
})
t.Run("custom template - per-alert color", func(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.MSTeamsV2Config{
WebhookURL: &config.SecretURL{URL: testWebhookURL},
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
`{{ template "msteamsv2.default.titleLink" . }}`,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
ctx := context.Background()
ctx = notify.WithGroupKey(ctx, "1")
alerts := []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "test1"},
Annotations: model.LabelSet{
"summary": "test",
ruletypes.AnnotationTitleTemplate: "Custom Title",
ruletypes.AnnotationBodyTemplate: "custom body $alertname",
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
},
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "test2"},
Annotations: model.LabelSet{
"summary": "test",
ruletypes.AnnotationTitleTemplate: "Custom Title",
ruletypes.AnnotationBodyTemplate: "custom body $alertname",
},
StartsAt: time.Now().Add(-time.Hour),
EndsAt: time.Now().Add(-time.Minute),
},
},
}
blocks, err := notifier.prepareContent(ctx, alerts)
require.NoError(t, err)
require.NotEmpty(t, blocks)
// total 3 blocks: title and 2 body blocks
require.True(t, len(blocks) == 3)
// First block: title color is overall color of the alerts
require.Equal(t, colorRed, blocks[0].Color)
// verify title text
require.Equal(t, "Custom Title", blocks[0].Text)
// Body blocks should have per-alert color
require.Equal(t, colorRed, blocks[1].Color) // firing
require.Equal(t, colorGreen, blocks[2].Color) // resolved
})
}
func TestMSTeamsV2ReadingURLFromFile(t *testing.T) {
ctx, u, fn := test.GetContextWithCancelingURL()
defer fn()
@@ -327,16 +209,14 @@ func TestMSTeamsV2ReadingURLFromFile(t *testing.T) {
_, err = f.WriteString(u.String() + "\n")
require.NoError(t, err, "writing to temp file failed")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.MSTeamsV2Config{
WebhookURLFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
`{{ template "msteamsv2.default.titleLink" . }}`,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)

View File

@@ -15,10 +15,7 @@ import (
"os"
"strings"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
@@ -37,27 +34,25 @@ const maxMessageLenRunes = 130
// Notifier implements a Notifier for OpsGenie notifications.
type Notifier struct {
conf *config.OpsGenieConfig
tmpl *template.Template
logger *slog.Logger
client *http.Client
retrier *notify.Retrier
templater alertmanagertypes.Templater
conf *config.OpsGenieConfig
tmpl *template.Template
logger *slog.Logger
client *http.Client
retrier *notify.Retrier
}
// New returns a new OpsGenie notifier.
func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
func New(c *config.OpsGenieConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
if err != nil {
return nil, err
}
return &Notifier{
conf: c,
tmpl: t,
logger: l,
client: client,
retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
templater: templater,
conf: c,
tmpl: t,
logger: l,
client: client,
retrier: &notify.Retrier{RetryCodes: []int{http.StatusTooManyRequests}},
}, nil
}
@@ -128,55 +123,6 @@ func safeSplit(s, sep string) []string {
return b
}
// prepareContent expands alert templates and returns the OpsGenie-ready title
// (truncated to the 130-rune limit) and HTML description. Custom bodies are
// rendered to HTML and stitched together with <hr> dividers; default bodies
// are joined with newlines (OpsGenie's legacy plain-text description).
func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert) (string, string, error) {
customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
TitleTemplate: customTitle,
BodyTemplate: customBody,
DefaultTitleTemplate: n.conf.Message,
DefaultBodyTemplate: n.conf.Description,
}, alerts)
if err != nil {
return "", "", err
}
var description string
if result.IsDefaultBody {
description = strings.Join(result.Body, "\n")
} else {
var b strings.Builder
first := true
for _, part := range result.Body {
if part == "" {
continue
}
rendered, renderErr := markdownrenderer.RenderHTML(part)
if renderErr != nil {
return "", "", renderErr
}
if !first {
b.WriteString("<hr>")
}
b.WriteString("<div>")
b.WriteString(rendered)
b.WriteString("</div>")
first = false
}
description = b.String()
}
title, truncated := notify.TruncateInRunes(result.Title, maxMessageLenRunes)
if truncated {
n.logger.WarnContext(ctx, "Truncated message", slog.Int("max_runes", maxMessageLenRunes))
}
return title, description, nil
}
// Create requests for a list of alerts.
func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*http.Request, bool, error) {
key, err := notify.ExtractGroupKey(ctx)
@@ -222,10 +168,9 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h
}
requests = append(requests, req.WithContext(ctx))
default:
message, description, err := n.prepareContent(ctx, as)
if err != nil {
n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
return nil, false, err
message, truncated := notify.TruncateInRunes(tmpl(n.conf.Message), maxMessageLenRunes)
if truncated {
logger.WarnContext(ctx, "Truncated message", slog.Any("alert", key), slog.Int("max_runes", maxMessageLenRunes))
}
createEndpointURL := n.conf.APIURL.Copy()
@@ -264,7 +209,7 @@ func (n *Notifier) createRequests(ctx context.Context, as ...*types.Alert) ([]*h
msg := &opsGenieCreateMessage{
Alias: alias,
Message: message,
Description: description,
Description: tmpl(n.conf.Description),
Details: details,
Source: tmpl(n.conf.Source),
Responders: responders,

View File

@@ -8,16 +8,12 @@ import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"os"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
@@ -26,23 +22,16 @@ import (
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/notify/test"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
)
func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
}
func TestOpsGenieRetry(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.OpsGenieConfig{
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -58,16 +47,14 @@ func TestOpsGenieRedactedURL(t *testing.T) {
defer fn()
key := "key"
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.OpsGenieConfig{
APIURL: &config.URL{URL: u},
APIKey: config.Secret(key),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -85,16 +72,14 @@ func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
_, err = f.WriteString(key)
require.NoError(t, err, "writing to temp file failed")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.OpsGenieConfig{
APIURL: &config.URL{URL: u},
APIKeyFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -217,7 +202,7 @@ func TestOpsGenie(t *testing.T) {
},
} {
t.Run(tc.title, func(t *testing.T) {
notifier, err := New(tc.cfg, tmpl, logger, newTestTemplater(tmpl))
notifier, err := New(tc.cfg, tmpl, logger)
require.NoError(t, err)
ctx := context.Background()
@@ -293,7 +278,7 @@ func TestOpsGenieWithUpdate(t *testing.T) {
APIURL: &config.URL{URL: u},
HTTPConfig: &commoncfg.HTTPClientConfig{},
}
notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
alert := &types.Alert{
Alert: model.Alert{
StartsAt: time.Now(),
@@ -336,7 +321,7 @@ func TestOpsGenieApiKeyFile(t *testing.T) {
APIURL: &config.URL{URL: u},
HTTPConfig: &commoncfg.HTTPClientConfig{},
}
notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
notifierWithUpdate, err := New(&opsGenieConfigWithUpdate, tmpl, promslog.NewNopLogger())
require.NoError(t, err)
requests, _, err := notifierWithUpdate.createRequests(ctx)
@@ -344,99 +329,6 @@ func TestOpsGenieApiKeyFile(t *testing.T) {
require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
}
func TestPrepareContent(t *testing.T) {
t.Run("default template", func(t *testing.T) {
tmpl := test.CreateTmpl(t)
logger := promslog.NewNopLogger()
notifier := &Notifier{
conf: &config.OpsGenieConfig{
Message: `{{ .CommonLabels.Message }}`,
Description: `{{ .CommonLabels.Description }}`,
},
tmpl: tmpl,
logger: logger,
templater: newTestTemplater(tmpl),
}
ctx := context.Background()
ctx = notify.WithGroupKey(ctx, "1")
alert := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{
"Message": "Firing alert: test",
"Description": "Check runbook for more details",
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
}
alerts := []*types.Alert{alert}
title, desc, prepErr := notifier.prepareContent(ctx, alerts)
require.NoError(t, prepErr)
require.Equal(t, "Firing alert: test", title)
require.Equal(t, "Check runbook for more details", desc)
})
t.Run("custom template", func(t *testing.T) {
tmpl := test.CreateTmpl(t)
logger := promslog.NewNopLogger()
notifier := &Notifier{
conf: &config.OpsGenieConfig{
Message: `{{ .CommonLabels.Message }}`,
Description: `{{ .CommonLabels.Description }}`,
},
tmpl: tmpl,
logger: logger,
templater: newTestTemplater(tmpl),
}
ctx := context.Background()
ctx = notify.WithGroupKey(ctx, "1")
alert1 := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{
"service": "payment",
"namespace": "potter-the-harry",
},
Annotations: model.LabelSet{
ruletypes.AnnotationTitleTemplate: "High request throughput for $service",
ruletypes.AnnotationBodyTemplate: "Alert firing in NS: $labels.namespace",
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
}
alert2 := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{
"service": "payment",
"namespace": "smart-the-rat",
},
Annotations: model.LabelSet{
ruletypes.AnnotationTitleTemplate: "High request throughput for $service",
ruletypes.AnnotationBodyTemplate: "Alert firing in NS: $labels.namespace",
},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
}
alerts := []*types.Alert{alert1, alert2}
title, desc, err := notifier.prepareContent(ctx, alerts)
require.NoError(t, err)
require.Equal(t, "High request throughput for payment", title)
// Each alert body wrapped in <div>, separated by <hr>
require.Equal(t, "<div><p>Alert firing in NS: potter-the-harry</p>\n</div><hr><div><p>Alert firing in NS: smart-the-rat</p>\n</div>", desc)
})
}
func readBody(t *testing.T, r *http.Request) string {
t.Helper()
body, err := io.ReadAll(r.Body)

View File

@@ -15,9 +15,7 @@ import (
"os"
"strings"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/alecthomas/units"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
@@ -42,22 +40,21 @@ const (
// Notifier implements a Notifier for PagerDuty notifications.
type Notifier struct {
conf *config.PagerdutyConfig
tmpl *template.Template
logger *slog.Logger
apiV1 string // for tests.
client *http.Client
retrier *notify.Retrier
templater alertmanagertypes.Templater
conf *config.PagerdutyConfig
tmpl *template.Template
logger *slog.Logger
apiV1 string // for tests.
client *http.Client
retrier *notify.Retrier
}
// New returns a new PagerDuty notifier.
func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
func New(c *config.PagerdutyConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
if err != nil {
return nil, err
}
n := &Notifier{conf: c, tmpl: t, logger: l, client: client, templater: templater}
n := &Notifier{conf: c, tmpl: t, logger: l, client: client}
if c.ServiceKey != "" || c.ServiceKeyFile != "" {
n.apiV1 = "https://events.pagerduty.com/generic/2010-04-15/create_event.json"
// Retrying can solve the issue on 403 (rate limiting) and 5xx response codes.
@@ -146,12 +143,11 @@ func (n *Notifier) notifyV1(
key notify.Key,
data *template.Data,
details map[string]any,
title string,
) (bool, error) {
var tmplErr error
tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
description, truncated := notify.TruncateInRunes(title, maxV1DescriptionLenRunes)
description, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV1DescriptionLenRunes)
if truncated {
n.logger.WarnContext(ctx, "Truncated description", slog.Any("key", key), slog.Int("max_runes", maxV1DescriptionLenRunes))
}
@@ -207,7 +203,6 @@ func (n *Notifier) notifyV2(
key notify.Key,
data *template.Data,
details map[string]any,
title string,
) (bool, error) {
var tmplErr error
tmpl := notify.TmplText(n.tmpl, data, &tmplErr)
@@ -216,7 +211,7 @@ func (n *Notifier) notifyV2(
n.conf.Severity = "error"
}
summary, truncated := notify.TruncateInRunes(title, maxV2SummaryLenRunes)
summary, truncated := notify.TruncateInRunes(tmpl(n.conf.Description), maxV2SummaryLenRunes)
if truncated {
n.logger.WarnContext(ctx, "Truncated summary", slog.Any("key", key), slog.Int("max_runes", maxV2SummaryLenRunes))
}
@@ -299,22 +294,6 @@ func (n *Notifier) notifyV2(
return retry, err
}
// prepareTitle expands the notification title. PagerDuty has no body surface
// we care about — the description/summary field is what users see as the
// incident headline, so we feed the configured Description as the default
// title template and ignore any custom body_template entirely.
func (n *Notifier) prepareTitle(ctx context.Context, alerts []*types.Alert) (string, error) {
customTitle, _ := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
TitleTemplate: customTitle,
DefaultTitleTemplate: n.conf.Description,
}, alerts)
if err != nil {
return "", err
}
return result.Title, nil
}
// Notify implements the Notifier interface.
func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
key, err := notify.ExtractGroupKey(ctx)
@@ -323,12 +302,6 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
}
logger := n.logger.With(slog.Any("group_key", key))
title, err := n.prepareTitle(ctx, as)
if err != nil {
n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
return false, err
}
var (
alerts = types.Alerts(as...)
data = notify.GetTemplateData(ctx, n.tmpl, as, logger)
@@ -356,7 +329,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
if n.apiV1 != "" {
nf = n.notifyV1
}
retry, err := nf(ctx, eventType, key, data, details, title)
retry, err := nf(ctx, eventType, key, data, details)
if err != nil {
if ctx.Err() != nil {
err = errors.WrapInternalf(err, errors.CodeInternal, "failed to notify PagerDuty: %v", context.Cause(ctx))

View File

@@ -9,7 +9,6 @@ import (
"context"
"encoding/json"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"net/url"
@@ -18,10 +17,7 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
@@ -34,20 +30,14 @@ import (
"github.com/prometheus/alertmanager/types"
)
func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
}
func TestPagerDutyRetryV1(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
ServiceKey: config.Secret("01234567890123456789012345678901"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -59,15 +49,13 @@ func TestPagerDutyRetryV1(t *testing.T) {
}
func TestPagerDutyRetryV2(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
RoutingKey: config.Secret("01234567890123456789012345678901"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -83,15 +71,13 @@ func TestPagerDutyRedactedURLV1(t *testing.T) {
defer fn()
key := "01234567890123456789012345678901"
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
ServiceKey: config.Secret(key),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
notifier.apiV1 = u.String()
@@ -104,16 +90,14 @@ func TestPagerDutyRedactedURLV2(t *testing.T) {
defer fn()
key := "01234567890123456789012345678901"
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
URL: &config.URL{URL: u},
RoutingKey: config.Secret(key),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -130,15 +114,13 @@ func TestPagerDutyV1ServiceKeyFromFile(t *testing.T) {
ctx, u, fn := test.GetContextWithCancelingURL()
defer fn()
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
ServiceKeyFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
notifier.apiV1 = u.String()
@@ -156,16 +138,14 @@ func TestPagerDutyV2RoutingKeyFromFile(t *testing.T) {
ctx, u, fn := test.GetContextWithCancelingURL()
defer fn()
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
URL: &config.URL{URL: u},
RoutingKeyFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -322,8 +302,7 @@ func TestPagerDutyTemplating(t *testing.T) {
t.Run(tc.title, func(t *testing.T) {
tc.cfg.URL = &config.URL{URL: u}
tc.cfg.HTTPConfig = &commoncfg.HTTPClientConfig{}
tmpl := test.CreateTmpl(t)
pd, err := New(tc.cfg, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
pd, err := New(tc.cfg, test.CreateTmpl(t), promslog.NewNopLogger())
require.NoError(t, err)
if pd.apiV1 != "" {
pd.apiV1 = u.String()
@@ -413,15 +392,13 @@ func TestEventSizeEnforcement(t *testing.T) {
Details: bigDetailsV1,
}
tmpl := test.CreateTmpl(t)
notifierV1, err := New(
&config.PagerdutyConfig{
ServiceKey: config.Secret("01234567890123456789012345678901"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -443,9 +420,8 @@ func TestEventSizeEnforcement(t *testing.T) {
RoutingKey: config.Secret("01234567890123456789012345678901"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -560,8 +536,7 @@ func TestPagerDutyEmptySrcHref(t *testing.T) {
Links: links,
}
pdTmpl := test.CreateTmpl(t)
pagerDuty, err := New(&pagerDutyConfig, pdTmpl, promslog.NewNopLogger(), newTestTemplater(pdTmpl))
pagerDuty, err := New(&pagerDutyConfig, test.CreateTmpl(t), promslog.NewNopLogger())
require.NoError(t, err)
ctx := context.Background()
@@ -628,8 +603,7 @@ func TestPagerDutyTimeout(t *testing.T) {
Timeout: tt.timeout,
}
tmpl := test.CreateTmpl(t)
pd, err := New(&cfg, tmpl, promslog.NewNopLogger(), newTestTemplater(tmpl))
pd, err := New(&cfg, test.CreateTmpl(t), promslog.NewNopLogger())
require.NoError(t, err)
ctx := context.Background()
@@ -907,79 +881,3 @@ func TestRenderDetails(t *testing.T) {
})
}
}
func TestPrepareContent(t *testing.T) {
prepareContext := func() context.Context {
ctx := context.Background()
ctx = notify.WithGroupKey(ctx, "1")
ctx = notify.WithReceiverName(ctx, "test-receiver")
ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "HighCPU for Payment service"})
return ctx
}
t.Run("default template uses go text template config for title", func(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
RoutingKey: config.Secret("01234567890123456789012345678901"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
Description: `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
},
tmpl,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
ctx := prepareContext()
alerts := []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{"alertname": "HighCPU for Payment service"},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
},
},
}
title, err := notifier.prepareTitle(ctx, alerts)
require.NoError(t, err)
require.Equal(t, "HighCPU for Payment service (FIRING)", title)
})
t.Run("custom template uses $variable annotation for title", func(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.PagerdutyConfig{
RoutingKey: config.Secret("01234567890123456789012345678901"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
ctx := prepareContext()
alerts := []*types.Alert{
{
Alert: model.Alert{
Labels: model.LabelSet{
"alertname": "HighCPU",
"service": "api-server",
},
Annotations: model.LabelSet{
ruletypes.AnnotationTitleTemplate: "$rule.name on $service is in $alert.status state",
},
StartsAt: time.Now().Add(-time.Hour),
EndsAt: time.Now(),
},
},
}
title, err := notifier.prepareTitle(ctx, alerts)
require.NoError(t, err)
require.Equal(t, "HighCPU on api-server is in resolved state", title)
})
}

View File

@@ -26,7 +26,7 @@ var customNotifierIntegrations = []string{
msteamsv2.Integration,
}
func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger, templater alertmanagertypes.Templater) ([]notify.Integration, error) {
func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error) {
upstreamIntegrations, err := receiver.BuildReceiverIntegrations(nc, tmpl, logger)
if err != nil {
return nil, err
@@ -53,25 +53,23 @@ func NewReceiverIntegrations(nc alertmanagertypes.Receiver, tmpl *template.Templ
}
for i, c := range nc.WebhookConfigs {
add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l, templater) })
add(webhook.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return webhook.New(c, tmpl, l) })
}
for i, c := range nc.EmailConfigs {
add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
return email.New(c, tmpl, l, templater), nil
})
add(email.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return email.New(c, tmpl, l), nil })
}
for i, c := range nc.PagerdutyConfigs {
add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l, templater) })
add(pagerduty.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return pagerduty.New(c, tmpl, l) })
}
for i, c := range nc.OpsGenieConfigs {
add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l, templater) })
add(opsgenie.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return opsgenie.New(c, tmpl, l) })
}
for i, c := range nc.SlackConfigs {
add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l, templater) })
add(slack.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) { return slack.New(c, tmpl, l) })
}
for i, c := range nc.MSTeamsV2Configs {
add(msteamsv2.Integration, i, c, func(l *slog.Logger) (notify.Notifier, error) {
return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l, templater)
return msteamsv2.New(c, tmpl, `{{ template "msteamsv2.default.titleLink" . }}`, l)
})
}

View File

@@ -14,11 +14,7 @@ import (
"os"
"strings"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/templating/markdownrenderer"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/alertmanager/config"
@@ -29,8 +25,6 @@ import (
const (
Integration = "slack"
colorRed = "#FF0000"
colorGreen = "#00FF00"
)
// https://api.slack.com/reference/messaging/attachments#legacy_fields - 1024, no units given, assuming runes or characters.
@@ -38,18 +32,17 @@ const maxTitleLenRunes = 1024
// Notifier implements a Notifier for Slack notifications.
type Notifier struct {
conf *config.SlackConfig
tmpl *template.Template
logger *slog.Logger
client *http.Client
retrier *notify.Retrier
templater alertmanagertypes.Templater
conf *config.SlackConfig
tmpl *template.Template
logger *slog.Logger
client *http.Client
retrier *notify.Retrier
postJSONFunc func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error)
}
// New returns a new Slack notification handler.
func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
client, err := notify.NewClientWithTracing(*c.HTTPConfig, Integration, httpOpts...)
if err != nil {
return nil, err
@@ -61,7 +54,6 @@ func New(c *config.SlackConfig, t *template.Template, l *slog.Logger, templater
logger: l,
client: client,
retrier: &notify.Retrier{},
templater: templater,
postJSONFunc: notify.PostJSON,
}, nil
}
@@ -89,10 +81,9 @@ type attachment struct {
Actions []config.SlackAction `json:"actions,omitempty"`
ImageURL string `json:"image_url,omitempty"`
ThumbURL string `json:"thumb_url,omitempty"`
Footer string `json:"footer,omitempty"`
Footer string `json:"footer"`
Color string `json:"color,omitempty"`
MrkdwnIn []string `json:"mrkdwn_in,omitempty"`
Blocks []any `json:"blocks,omitempty"`
}
// Notify implements the Notifier interface.
@@ -109,15 +100,79 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
data = notify.GetTemplateData(ctx, n.tmpl, as, logger)
tmplText = notify.TmplText(n.tmpl, data, &err)
)
var markdownIn []string
attachments, err := n.prepareContent(ctx, as, tmplText)
if err != nil {
n.logger.ErrorContext(ctx, "failed to prepare notification content", errors.Attr(err))
return false, err
if len(n.conf.MrkdwnIn) == 0 {
markdownIn = []string{"fallback", "pretext", "text"}
} else {
markdownIn = n.conf.MrkdwnIn
}
if len(attachments) > 0 {
n.addFieldsAndActions(&attachments[0], tmplText)
title, truncated := notify.TruncateInRunes(tmplText(n.conf.Title), maxTitleLenRunes)
if truncated {
logger.WarnContext(ctx, "Truncated title", slog.Int("max_runes", maxTitleLenRunes))
}
att := &attachment{
Title: title,
TitleLink: tmplText(n.conf.TitleLink),
Pretext: tmplText(n.conf.Pretext),
Text: tmplText(n.conf.Text),
Fallback: tmplText(n.conf.Fallback),
CallbackID: tmplText(n.conf.CallbackID),
ImageURL: tmplText(n.conf.ImageURL),
ThumbURL: tmplText(n.conf.ThumbURL),
Footer: tmplText(n.conf.Footer),
Color: tmplText(n.conf.Color),
MrkdwnIn: markdownIn,
}
numFields := len(n.conf.Fields)
if numFields > 0 {
fields := make([]config.SlackField, numFields)
for index, field := range n.conf.Fields {
// Check if short was defined for the field otherwise fallback to the global setting
var short bool
if field.Short != nil {
short = *field.Short
} else {
short = n.conf.ShortFields
}
// Rebuild the field by executing any templates and setting the new value for short
fields[index] = config.SlackField{
Title: tmplText(field.Title),
Value: tmplText(field.Value),
Short: &short,
}
}
att.Fields = fields
}
numActions := len(n.conf.Actions)
if numActions > 0 {
actions := make([]config.SlackAction, numActions)
for index, action := range n.conf.Actions {
slackAction := config.SlackAction{
Type: tmplText(action.Type),
Text: tmplText(action.Text),
URL: tmplText(action.URL),
Style: tmplText(action.Style),
Name: tmplText(action.Name),
Value: tmplText(action.Value),
}
if action.ConfirmField != nil {
slackAction.ConfirmField = &config.SlackConfirmationField{
Title: tmplText(action.ConfirmField.Title),
Text: tmplText(action.ConfirmField.Text),
OkText: tmplText(action.ConfirmField.OkText),
DismissText: tmplText(action.ConfirmField.DismissText),
}
}
actions[index] = slackAction
}
att.Actions = actions
}
req := &request{
@@ -127,7 +182,7 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
IconURL: tmplText(n.conf.IconURL),
LinkNames: n.conf.LinkNames,
Text: tmplText(n.conf.MessageText),
Attachments: attachments,
Attachments: []attachment{*att},
}
if err != nil {
return false, err
@@ -183,150 +238,6 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
return retry, nil
}
// prepareContent expands alert templates and returns the Slack attachment(s)
// ready to send. When alerts carry a custom body template, one title-only
// attachment plus one body attachment per alert is returned so that each alert
// can get its own firing/resolved color and per-alert action buttons.
func (n *Notifier) prepareContent(ctx context.Context, alerts []*types.Alert, tmplText func(string) string) ([]attachment, error) {
customTitle, customBody := alertmanagertemplate.ExtractTemplatesFromAnnotations(alerts)
result, err := n.templater.Expand(ctx, alertmanagertypes.ExpandRequest{
TitleTemplate: customTitle,
BodyTemplate: customBody,
DefaultTitleTemplate: n.conf.Title,
DefaultBodyTemplate: n.conf.Text,
}, alerts)
if err != nil {
return nil, err
}
title, truncated := notify.TruncateInRunes(result.Title, maxTitleLenRunes)
if truncated {
n.logger.WarnContext(ctx, "Truncated title", slog.Int("max_runes", maxTitleLenRunes))
}
if result.IsDefaultBody {
var markdownIn []string
if len(n.conf.MrkdwnIn) == 0 {
markdownIn = []string{"fallback", "pretext", "text"}
} else {
markdownIn = n.conf.MrkdwnIn
}
return []attachment{
{
Title: title,
TitleLink: tmplText(n.conf.TitleLink),
Pretext: tmplText(n.conf.Pretext),
Text: result.Body[0],
Fallback: tmplText(n.conf.Fallback),
CallbackID: tmplText(n.conf.CallbackID),
ImageURL: tmplText(n.conf.ImageURL),
ThumbURL: tmplText(n.conf.ThumbURL),
Footer: tmplText(n.conf.Footer),
Color: tmplText(n.conf.Color),
MrkdwnIn: markdownIn,
},
}, nil
}
// Custom template path: one title attachment + one attachment per
// non-empty alert body. result.Body is positionally aligned with alerts,
// so we index alerts[i] directly and skip empty entries.
attachments := make([]attachment, 0, 1+len(result.Body))
attachments = append(attachments, attachment{
Title: title,
TitleLink: tmplText(n.conf.TitleLink),
})
for i, body := range result.Body {
if body == "" || i >= len(alerts) {
continue
}
// Custom bodies are authored in markdown; render each non-empty body to
// Slack's mrkdwn flavour. Default bodies skip this because the Text
// template is already channel-ready.
rendered, renderErr := markdownrenderer.RenderSlackMrkdwn(body)
if renderErr != nil {
return nil, renderErr
}
color := colorRed
if alerts[i].Resolved() {
color = colorGreen
}
attachments = append(attachments, attachment{
Text: rendered,
Color: color,
MrkdwnIn: []string{"text"},
Actions: buildRelatedLinkActions(alerts[i]),
})
}
return attachments, nil
}
// buildRelatedLinkActions returns the "View Related Logs/Traces" action
// buttons for an alert, or nil when no related-link annotations are present.
func buildRelatedLinkActions(alert *types.Alert) []config.SlackAction {
var actions []config.SlackAction
if link := alert.Annotations[ruletypes.AnnotationRelatedLogs]; link != "" {
actions = append(actions, config.SlackAction{Type: "button", Text: "View Related Logs", URL: string(link)})
}
if link := alert.Annotations[ruletypes.AnnotationRelatedTraces]; link != "" {
actions = append(actions, config.SlackAction{Type: "button", Text: "View Related Traces", URL: string(link)})
}
return actions
}
// addFieldsAndActions populates fields and actions on the attachment from the Slack config.
func (n *Notifier) addFieldsAndActions(att *attachment, tmplText func(string) string) {
numFields := len(n.conf.Fields)
if numFields > 0 {
fields := make([]config.SlackField, numFields)
for index, field := range n.conf.Fields {
var short bool
if field.Short != nil {
short = *field.Short
} else {
short = n.conf.ShortFields
}
fields[index] = config.SlackField{
Title: tmplText(field.Title),
Value: tmplText(field.Value),
Short: &short,
}
}
att.Fields = fields
}
numActions := len(n.conf.Actions)
if numActions > 0 {
actions := make([]config.SlackAction, numActions)
for index, action := range n.conf.Actions {
slackAction := config.SlackAction{
Type: tmplText(action.Type),
Text: tmplText(action.Text),
URL: tmplText(action.URL),
Style: tmplText(action.Style),
Name: tmplText(action.Name),
Value: tmplText(action.Value),
}
if action.ConfirmField != nil {
slackAction.ConfirmField = &config.SlackConfirmationField{
Title: tmplText(action.ConfirmField.Title),
Text: tmplText(action.ConfirmField.Text),
OkText: tmplText(action.ConfirmField.OkText),
DismissText: tmplText(action.ConfirmField.DismissText),
}
}
actions[index] = slackAction
}
att.Actions = actions
}
}
// checkResponseError parses out the error message from Slack API response.
func checkResponseError(resp *http.Response) (bool, error) {
body, err := io.ReadAll(resp.Body)

View File

@@ -17,9 +17,6 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/common/model"
"github.com/prometheus/common/promslog"
@@ -32,19 +29,13 @@ import (
"github.com/prometheus/alertmanager/types"
)
func newTestTemplater(tmpl *template.Template) alertmanagertypes.Templater {
return alertmanagertemplate.New(tmpl, slog.New(slog.DiscardHandler))
}
func TestSlackRetry(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.SlackConfig{
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -58,15 +49,13 @@ func TestSlackRedactedURL(t *testing.T) {
ctx, u, fn := test.GetContextWithCancelingURL()
defer fn()
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.SlackConfig{
APIURL: &config.SecretURL{URL: u},
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -82,15 +71,13 @@ func TestGettingSlackURLFromFile(t *testing.T) {
_, err = f.WriteString(u.String())
require.NoError(t, err, "writing to temp file failed")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.SlackConfig{
APIURLFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -106,15 +93,13 @@ func TestTrimmingSlackURLFromFile(t *testing.T) {
_, err = f.WriteString(u.String() + "\n\n")
require.NoError(t, err, "writing to temp file failed")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.SlackConfig{
APIURLFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -199,7 +184,6 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
apiurl, _ := url.Parse("https://slack.com/post.Message")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.SlackConfig{
NotifierConfig: config.NotifierConfig{},
@@ -207,9 +191,8 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
APIURL: &config.SecretURL{URL: apiurl},
Channel: "channelname",
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
@@ -259,7 +242,6 @@ func TestSlackTimeout(t *testing.T) {
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
u, _ := url.Parse("https://slack.com/post.Message")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.SlackConfig{
NotifierConfig: config.NotifierConfig{},
@@ -268,9 +250,8 @@ func TestSlackTimeout(t *testing.T) {
Channel: "channelname",
Timeout: tt.timeout,
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
newTestTemplater(tmpl),
)
require.NoError(t, err)
notifier.postJSONFunc = func(ctx context.Context, client *http.Client, url string, body io.Reader) (*http.Response, error) {
@@ -301,225 +282,6 @@ func TestSlackTimeout(t *testing.T) {
}
}
// setupTestContext creates a context with group key, receiver name, and group labels
// required by the notification processor.
func setupTestContext() context.Context {
ctx := context.Background()
ctx = notify.WithGroupKey(ctx, "test-group")
ctx = notify.WithReceiverName(ctx, "slack")
ctx = notify.WithGroupLabels(ctx, model.LabelSet{
"alertname": "TestAlert",
"severity": "critical",
})
return ctx
}
func TestPrepareContent(t *testing.T) {
t.Run("default template uses go text template config for title and body", func(t *testing.T) {
// When alerts have no custom annotation templates (title_template / body_template),
tmpl := test.CreateTmpl(t)
templater := newTestTemplater(tmpl)
notifier := &Notifier{
conf: &config.SlackConfig{
Title: `{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})`,
Text: `{{ range .Alerts }}Alert: {{ .Labels.alertname }} - severity {{ .Labels.severity }}{{ end }}`,
Color: `{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}`,
TitleLink: "https://alertmanager.signoz.com",
},
tmpl: tmpl,
logger: slog.New(slog.DiscardHandler),
templater: templater,
}
ctx := setupTestContext()
alerts := []*types.Alert{
{Alert: model.Alert{
Labels: model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
}},
}
// Build tmplText the same way Notify does
var err error
data := notify.GetTemplateData(ctx, tmpl, alerts, slog.New(slog.DiscardHandler))
tmplText := notify.TmplText(tmpl, data, &err)
atts, attErr := notifier.prepareContent(ctx, alerts, tmplText)
require.NoError(t, attErr)
require.NoError(t, err)
require.Len(t, atts, 1)
require.Equal(t, "HighCPU (FIRING)", atts[0].Title)
require.Equal(t, "Alert: HighCPU - severity critical", atts[0].Text)
// Color is templated — firing alert should be "danger"
require.Equal(t, "danger", atts[0].Color)
// No BlockKit blocks for default template
require.Nil(t, atts[0].Blocks)
// Default markdownIn when config has none
require.Equal(t, []string{"fallback", "pretext", "text"}, atts[0].MrkdwnIn)
})
t.Run("custom template produces 1+N attachments with per-alert color", func(t *testing.T) {
// When alerts carry custom $variable annotation templates (title_template / body_template)
tmpl := test.CreateTmpl(t)
templater := newTestTemplater(tmpl)
notifier := &Notifier{
conf: &config.SlackConfig{
Title: "default title fallback",
Text: "default text fallback",
TitleLink: "https://alertmanager.signoz.com",
},
tmpl: tmpl,
logger: slog.New(slog.DiscardHandler),
templater: templater,
}
tmplText := func(s string) string { return s }
bodyTemplate := `## $rule.name
**Service:** *$labels.service*
**Instance:** *$labels.instance*
**Region:** *$labels.region*
**Method:** *$labels.http_method*
---
| Metric | Value |
|--------|-------|
| **Current** | *$value* |
| **Threshold** | *$threshold.value* |
**Status:** $alert.status | **Severity:** $labels.severity`
titleTemplate := "[$alert.status] $rule.name — $labels.service"
ctx := setupTestContext()
firingAlert := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical", "service": "api-server", "instance": "i-0abc123", "region": "us-east-1", "http_method": "GET"},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
Annotations: model.LabelSet{
ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
ruletypes.AnnotationBodyTemplate: model.LabelValue(bodyTemplate),
"value": "100",
"threshold.value": "200",
},
},
}
resolvedAlert := &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical", "service": "api-server", "instance": "i-0abc123", "region": "us-east-1", "http_method": "GET"},
StartsAt: time.Now().Add(-2 * time.Hour),
EndsAt: time.Now().Add(-time.Hour),
Annotations: model.LabelSet{
ruletypes.AnnotationTitleTemplate: model.LabelValue(titleTemplate),
ruletypes.AnnotationBodyTemplate: model.LabelValue(bodyTemplate),
"value": "50",
"threshold.value": "200",
},
},
}
atts, err := notifier.prepareContent(ctx, []*types.Alert{firingAlert, resolvedAlert}, tmplText)
require.NoError(t, err)
// 1 title attachment + 2 body attachments (one per alert)
require.Len(t, atts, 3)
// First attachment: title-only, no color, no blocks
require.Equal(t, "[firing] HighCPU — api-server", atts[0].Title)
require.Empty(t, atts[0].Color)
require.Nil(t, atts[0].Blocks)
require.Equal(t, "https://alertmanager.signoz.com", atts[0].TitleLink)
expectedFiringBody := "*HighCPU*\n\n" +
"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
"---\n\n" +
"```\nMetric | Value\n----------|------\nCurrent | 100 \nThreshold | 200 \n```\n\n" +
"*Status:* firing | *Severity:* critical\n\n"
expectedResolvedBody := "*HighCPU*\n\n" +
"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
"---\n\n" +
"```\nMetric | Value\n----------|------\nCurrent | 50 \nThreshold | 200 \n```\n\n" +
"*Status:* resolved | *Severity:* critical\n\n"
// Second attachment: firing alert body rendered as slack mrkdwn text, red color
require.Nil(t, atts[1].Blocks)
require.Equal(t, "#FF0000", atts[1].Color)
require.Equal(t, []string{"text"}, atts[1].MrkdwnIn)
require.Equal(t, expectedFiringBody, atts[1].Text)
// Third attachment: resolved alert body rendered as slack mrkdwn text, green color
require.Nil(t, atts[2].Blocks)
require.Equal(t, "#00FF00", atts[2].Color)
require.Equal(t, []string{"text"}, atts[2].MrkdwnIn)
require.Equal(t, expectedResolvedBody, atts[2].Text)
})
t.Run("default template with fields and actions", func(t *testing.T) {
// Verifies that addFieldsAndActions (called from Notify after prepareContent)
// correctly populates fields and actions on the attachment from config.
tmpl := test.CreateTmpl(t)
templater := newTestTemplater(tmpl)
short := true
notifier := &Notifier{
conf: &config.SlackConfig{
Title: `{{ .CommonLabels.alertname }}`,
Text: "alert text",
Color: "warning",
Fields: []*config.SlackField{
{Title: "Severity", Value: "critical", Short: &short},
{Title: "Service", Value: "api-server", Short: &short},
},
Actions: []*config.SlackAction{
{Type: "button", Text: "View Alert", URL: "https://alertmanager.signoz.com"},
},
TitleLink: "https://alertmanager.signoz.com",
},
tmpl: tmpl,
logger: slog.New(slog.DiscardHandler),
templater: templater,
}
tmplText := func(s string) string { return s }
ctx := setupTestContext()
alerts := []*types.Alert{
{Alert: model.Alert{
Labels: model.LabelSet{ruletypes.LabelAlertName: "TestAlert"},
StartsAt: time.Now(),
EndsAt: time.Now().Add(time.Hour),
}},
}
atts, err := notifier.prepareContent(ctx, alerts, tmplText)
require.NoError(t, err)
require.Len(t, atts, 1)
// prepareContent does not populate fields/actions — that's done by
// addFieldsAndActions which is called from Notify.
require.Nil(t, atts[0].Fields)
require.Nil(t, atts[0].Actions)
// Simulate what Notify does after prepareContent
notifier.addFieldsAndActions(&atts[0], tmplText)
// Verify fields
require.Len(t, atts[0].Fields, 2)
require.Equal(t, "Severity", atts[0].Fields[0].Title)
require.Equal(t, "critical", atts[0].Fields[0].Value)
require.True(t, *atts[0].Fields[0].Short)
require.Equal(t, "Service", atts[0].Fields[1].Title)
require.Equal(t, "api-server", atts[0].Fields[1].Value)
// Verify actions
require.Len(t, atts[0].Actions, 1)
require.Equal(t, "button", atts[0].Actions[0].Type)
require.Equal(t, "View Alert", atts[0].Actions[0].Text)
require.Equal(t, "https://alertmanager.signoz.com", atts[0].Actions[0].URL)
})
}
func TestSlackMessageField(t *testing.T) {
// 1. Setup a fake Slack server
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -567,7 +329,7 @@ func TestSlackMessageField(t *testing.T) {
tmpl.ExternalURL = u
logger := slog.New(slog.DiscardHandler)
notifier, err := New(conf, tmpl, logger, newTestTemplater(tmpl))
notifier, err := New(conf, tmpl, logger)
if err != nil {
t.Fatal(err)
}

View File

@@ -14,7 +14,6 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
commoncfg "github.com/prometheus/common/config"
"github.com/prometheus/alertmanager/config"
@@ -29,16 +28,15 @@ const (
// Notifier implements a Notifier for generic webhooks.
type Notifier struct {
conf *config.WebhookConfig
tmpl *template.Template
logger *slog.Logger
client *http.Client
retrier *notify.Retrier
templater alertmanagertypes.Templater
conf *config.WebhookConfig
tmpl *template.Template
logger *slog.Logger
client *http.Client
retrier *notify.Retrier
}
// New returns a new Webhook.
func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, templater alertmanagertypes.Templater, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, httpOpts ...commoncfg.HTTPClientOption) (*Notifier, error) {
client, err := notify.NewClientWithTracing(*conf.HTTPConfig, Integration, httpOpts...)
if err != nil {
return nil, err
@@ -50,8 +48,7 @@ func New(conf *config.WebhookConfig, t *template.Template, l *slog.Logger, templ
client: client,
// Webhooks are assumed to respond with 2xx response codes on a successful
// request and 5xx response codes are assumed to be recoverable.
retrier: &notify.Retrier{},
templater: templater,
retrier: &notify.Retrier{},
}, nil
}

View File

@@ -9,7 +9,6 @@ import (
"context"
"fmt"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"os"
@@ -21,7 +20,6 @@ import (
"github.com/prometheus/common/promslog"
"github.com/stretchr/testify/require"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/prometheus/alertmanager/config"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/notify/test"
@@ -29,15 +27,13 @@ import (
)
func TestWebhookRetry(t *testing.T) {
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.WebhookConfig{
URL: config.SecretTemplateURL("http://example.com"),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
alertmanagertemplate.New(tmpl, slog.Default()),
)
if err != nil {
require.NoError(t, err)
@@ -100,16 +96,13 @@ func TestWebhookRedactedURL(t *testing.T) {
defer fn()
secret := "secret"
tmpl := test.CreateTmpl(t)
templater := alertmanagertemplate.New(tmpl, slog.Default())
notifier, err := New(
&config.WebhookConfig{
URL: config.SecretTemplateURL(u.String()),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
templater,
)
require.NoError(t, err)
@@ -125,15 +118,13 @@ func TestWebhookReadingURLFromFile(t *testing.T) {
_, err = f.WriteString(u.String() + "\n")
require.NoError(t, err, "writing to temp file failed")
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.WebhookConfig{
URLFile: f.Name(),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
alertmanagertemplate.New(tmpl, slog.Default()),
)
require.NoError(t, err)
@@ -187,15 +178,13 @@ func TestWebhookURLTemplating(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
calledURL = "" // Reset for each test
tmpl := test.CreateTmpl(t)
notifier, err := New(
&config.WebhookConfig{
URL: config.SecretTemplateURL(tc.url),
HTTPConfig: &commoncfg.HTTPClientConfig{},
},
tmpl,
test.CreateTmpl(t),
promslog.NewNopLogger(),
alertmanagertemplate.New(tmpl, slog.Default()),
)
require.NoError(t, err)

View File

@@ -28,13 +28,6 @@ type Config struct {
// Configuration for the notification log.
NFLog NFLogConfig `mapstructure:"nflog"`
// Templates is the list of globs from which SigNoz's alertmanager notification
// templates are loaded (e.g. the email.signoz.html layout). This mirrors the
// upstream alertmanager `templates` config option (https://github.com/prometheus/alertmanager/blob/3b06b97af4d146e141af92885a185891eb79a5b0/config/config.go#L412).
// The upstream default templates (default.tmpl, email.tmpl) are always loaded
// from the embedded alertmanager assets, so only SigNoz's own templates are listed here.
Templates []string `mapstructure:"templates"`
}
type AlertsConfig struct {
@@ -107,6 +100,5 @@ func NewConfig() Config {
MaintenanceInterval: 15 * time.Minute,
Retention: 120 * time.Hour,
},
Templates: []string{"/root/templates/alertmanager/*.gotmpl"},
}
}

View File

@@ -10,7 +10,6 @@ import (
"github.com/prometheus/alertmanager/types"
"golang.org/x/sync/errgroup"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/prometheus/alertmanager/dispatch"
"github.com/prometheus/alertmanager/featurecontrol"
"github.com/prometheus/alertmanager/inhibit"
@@ -24,8 +23,8 @@ import (
"github.com/prometheus/common/model"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify"
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
)
@@ -66,7 +65,6 @@ type Server struct {
muter *MaintenanceMuter
marker *types.MemMarker
tmpl *template.Template
templater alertmanagertypes.Templater
wg sync.WaitGroup
stopc chan struct{}
notificationManager nfmanager.NotificationManager
@@ -244,21 +242,13 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
config := alertmanagerConfig.AlertmanagerConfig()
var err error
// Load SigNoz's alertmanager notification templates from the configured
// globs. The upstream default templates (default.tmpl, email.tmpl) are
// always loaded from the embedded alertmanager assets inside FromGlobs, so
// only SigNoz's own templates (e.g. the email.signoz.html layout) are listed
// here. The upstream config.Templates field is not used: SigNoz never
// populates it (there is no per-org template configuration).
server.tmpl, err = alertmanagertypes.FromGlobs(server.srvConfig.Templates)
server.tmpl, err = alertmanagertypes.FromGlobs(config.Templates)
if err != nil {
return err
}
server.tmpl.ExternalURL = server.srvConfig.ExternalURL
server.templater = alertmanagertemplate.New(server.tmpl, server.logger)
// Build the routing tree and record which receivers are used.
routes := dispatch.NewRoute(config.Route, nil)
activeReceivers := make(map[string]struct{})
@@ -275,7 +265,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
server.logger.InfoContext(ctx, "skipping creation of receiver not referenced by any route", slog.String("receiver", rcv.Name))
continue
}
integrations, err := alertmanagernotify.NewReceiverIntegrations(rcv, server.tmpl, server.logger, server.templater)
integrations, err := alertmanagernotify.NewReceiverIntegrations(rcv, server.tmpl, server.logger)
if err != nil {
return err
}
@@ -352,7 +342,7 @@ func (server *Server) SetConfig(ctx context.Context, alertmanagerConfig *alertma
func (server *Server) TestReceiver(ctx context.Context, receiver alertmanagertypes.Receiver) error {
testAlert := alertmanagertypes.NewTestAlert(receiver, time.Now(), time.Now())
return alertmanagertypes.TestReceiver(ctx, receiver, alertmanagernotify.NewReceiverIntegrations, server.alertmanagerConfig, server.tmpl, server.logger, server.templater, testAlert.Labels, testAlert)
return alertmanagertypes.TestReceiver(ctx, receiver, alertmanagernotify.NewReceiverIntegrations, server.alertmanagerConfig, server.tmpl, server.logger, testAlert.Labels, testAlert)
}
func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmanagertypes.PostableAlert][]string, config *alertmanagertypes.NotificationConfig) error {
@@ -435,7 +425,6 @@ func (server *Server) TestAlert(ctx context.Context, receiversMap map[*alertmana
server.alertmanagerConfig,
server.tmpl,
server.logger,
server.templater,
group.groupLabels,
group.alerts...,
)

View File

@@ -15,6 +15,13 @@ import (
"github.com/prometheus/common/model"
)
// Templater expands user-authored title and body templates against a group
// of alerts and returns channel-ready strings along with the aggregate data
// a caller might reuse (e.g. to render an email layout around the body).
type Templater interface {
Expand(ctx context.Context, req alertmanagertypes.ExpandRequest, alerts []*types.Alert) (*alertmanagertypes.ExpandResult, error)
}
type templater struct {
tmpl *template.Template
logger *slog.Logger
@@ -22,7 +29,7 @@ type templater struct {
// New returns a Templater bound to the given Prometheus alertmanager
// template and logger.
func New(tmpl *template.Template, logger *slog.Logger) alertmanagertypes.Templater {
func New(tmpl *template.Template, logger *slog.Logger) Templater {
return &templater{tmpl: tmpl, logger: logger}
}
@@ -130,9 +137,6 @@ func (at *templater) expandTitle(
}
// expandBody expands the body template for each individual alert. Returns nil if the template is empty.
// Non-nil results are positionally aligned with ntd.Alerts: sb[i] corresponds to alerts[i], and
// entries for alerts whose template expands to empty are kept as "" so callers can index per-alert
// metadata (related links, firing/resolved color) by the same index.
func (at *templater) expandBody(
bodyTemplate string,
ntd *alertmanagertypes.NotificationTemplateData,
@@ -140,7 +144,7 @@ func (at *templater) expandBody(
if bodyTemplate == "" {
return nil, nil, nil
}
sb := make([]string, len(ntd.Alerts))
var sb []string
missingVars := make(map[string]bool)
for i := range ntd.Alerts {
processRes, err := preProcessTemplateAndData(bodyTemplate, &ntd.Alerts[i])
@@ -151,10 +155,13 @@ func (at *templater) expandBody(
if err != nil {
return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute custom body template: %s", err.Error())
}
// add unknown variables and templated text to the result
for k := range processRes.UnknownVars {
missingVars[k] = true
}
sb[i] = strings.TrimSpace(part)
if strings.TrimSpace(part) != "" {
sb = append(sb, strings.TrimSpace(part))
}
}
return sb, missingVars, nil
}
@@ -182,20 +189,17 @@ func (at *templater) buildNotificationTemplateData(
externalURL = at.tmpl.ExternalURL.String()
}
// Raw (including private `_*`) kv first so buildRuleInfo can read the
// private rule annotations. The filtered copies are what ends up
// on the template-visible surfaces.
rawCommonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
commonLabels := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
// aggregate labels and annotations from all alerts
labels := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
annotations := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
// Strip private annotations from template-visible surfaces; the structured
// fields on AlertInfo/RuleInfo already hold anything a template needs from
// them.
commonAnnotations := alertmanagertypes.FilterPublicAnnotations(rawCommonAnnotations)
// Strip private annotations from surfaces visible to templates or
// notifications; the structured fields on AlertInfo/RuleInfo already hold
// anything a template needs from them.
commonAnnotations = alertmanagertypes.FilterPublicAnnotations(commonAnnotations)
annotations = alertmanagertypes.FilterPublicAnnotations(annotations)
// build the alert data slice
@@ -229,7 +233,7 @@ func (at *templater) buildNotificationTemplateData(
TotalFiring: firing,
TotalResolved: resolved,
},
Rule: buildRuleInfo(commonLabels, rawCommonAnnotations),
Rule: buildRuleInfo(commonLabels, commonAnnotations),
GroupLabels: gl,
CommonLabels: commonLabels,
CommonAnnotations: commonAnnotations,

View File

@@ -19,7 +19,7 @@ import (
// testSetup returns an AlertTemplater and a context pre-populated with group key,
// receiver name, and group labels for use in tests.
func testSetup(t *testing.T) (alertmanagertypes.Templater, context.Context) {
func testSetup(t *testing.T) (Templater, context.Context) {
t.Helper()
tmpl := test.CreateTmpl(t)
ctx := context.Background()

View File

@@ -29,5 +29,24 @@ func (provider *provider) addTraceDetailRoutes(router *mux.Router) error {
return err
}
if err := router.Handle("/api/v4/traces/{traceID}/waterfall", handler.New(
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.GetWaterfallV4),
handler.OpenAPIDef{
ID: "GetWaterfallV4",
Tags: []string{"tracedetail"},
Summary: "Get waterfall view for a trace (OOM-safe)",
Description: "Two-step fetch: minimal fields for all spans to build the tree, full fields only for the visible window. Aggregations are not included in the response.",
Request: new(spantypes.PostableWaterfall),
RequestContentType: "application/json",
Response: new(spantypes.GettableWaterfallTrace),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
},
)).Methods(http.MethodPost).GetError(); err != nil {
return err
}
return nil
}

View File

@@ -65,7 +65,7 @@ func newConfig() factory.Config {
return &Config{
Enabled: false,
Templates: Templates{
Directory: "/root/templates/email",
Directory: "/root/templates",
Format: Format{
Header: Header{
Enabled: false,

View File

@@ -38,3 +38,24 @@ func (h *handler) GetWaterfall(rw http.ResponseWriter, r *http.Request) {
render.Success(rw, http.StatusOK, result)
}
func (h *handler) GetWaterfallV4(rw http.ResponseWriter, r *http.Request) {
req := new(spantypes.PostableWaterfall)
if err := binding.JSON.BindBody(r.Body, req); err != nil {
render.Error(rw, err)
return
}
if err := req.Validate(); err != nil {
render.Error(rw, err)
return
}
result, err := h.module.GetWaterfallV4(r.Context(), mux.Vars(r)["traceID"], req)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, result)
}

View File

@@ -45,7 +45,7 @@ func (m *module) GetWaterfall(ctx context.Context, traceID string, req *spantype
return spantypes.NewGettableWaterfallTrace(waterfallTrace, selectedSpans, uncollapsedSpans, selectedAllSpans, aggregationResults), nil
}
// getTraceData returns the waterfall cache for the given traceID with fallback on DB.
// getTraceData fetches all spans for a trace and builds the WaterfallTrace.
func (m *module) getTraceData(ctx context.Context, traceID string) (*spantypes.WaterfallTrace, error) {
summary, err := m.store.GetTraceSummary(ctx, traceID)
if err != nil {
@@ -61,6 +61,86 @@ func (m *module) getTraceData(ctx context.Context, traceID string) (*spantypes.W
return nil, spantypes.ErrTraceNotFound
}
traceData := spantypes.NewWaterfallTraceFromSpans(spanItems)
return traceData, nil
nodes := make([]*spantypes.WaterfallSpan, len(spanItems))
for i := range spanItems {
nodes[i] = spanItems[i].ToWaterfallSpan()
}
return spantypes.NewWaterfallTraceFromSpans(nodes), nil
}
// GetWaterfallV4 is the OOM-safe V4 waterfall.
// For large traces (NumSpans > effectiveLimit) it uses a two-step fetch:
// minimal fields for all spans to build the tree, then full fields for the
// visible window only. Aggregations are not returned.
func (m *module) GetWaterfallV4(ctx context.Context, traceID string, req *spantypes.PostableWaterfall) (*spantypes.GettableWaterfallTrace, error) {
summary, err := m.store.GetTraceSummary(ctx, traceID)
if err != nil {
return nil, err
}
effectiveLimit := min(req.Limit, m.config.Waterfall.MaxLimitToSelectAllSpans)
if summary.NumSpans > uint64(effectiveLimit) {
return m.getWindowedWaterfall(ctx, traceID, req, summary, effectiveLimit)
}
return m.getFullWaterfall(ctx, traceID, summary)
}
func (m *module) getFullWaterfall(ctx context.Context, traceID string, summary *spantypes.TraceSummary) (*spantypes.GettableWaterfallTrace, error) {
spanItems, err := m.store.GetTraceSpans(ctx, traceID, summary)
if err != nil {
return nil, err
}
if len(spanItems) == 0 {
return nil, spantypes.ErrTraceNotFound
}
nodes := make([]*spantypes.WaterfallSpan, len(spanItems))
for i := range spanItems {
nodes[i] = spanItems[i].ToWaterfallSpan()
}
waterfallTrace := spantypes.NewWaterfallTraceFromSpans(nodes)
selectedSpans := waterfallTrace.GetAllSpans()
return spantypes.NewGettableWaterfallTrace(waterfallTrace, selectedSpans, nil, true, nil), nil
}
// getWindowedWaterfall builds the waterfall tree with minimal data and then returns only a window of full spans.
func (m *module) getWindowedWaterfall(ctx context.Context, traceID string, req *spantypes.PostableWaterfall, summary *spantypes.TraceSummary, effectiveLimit uint) (*spantypes.GettableWaterfallTrace, error) {
// Step 1: minimal fetch → build full tree → select visible window
minimalSpans, err := m.store.GetMinimalSpans(ctx, traceID, summary)
if err != nil {
return nil, err
}
if len(minimalSpans) == 0 {
return nil, spantypes.ErrTraceNotFound
}
nodes := make([]*spantypes.WaterfallSpan, len(minimalSpans))
for i := range minimalSpans {
nodes[i] = minimalSpans[i].ToWaterfallSpan()
}
waterfallTrace := spantypes.NewWaterfallTraceFromSpans(nodes)
selectedSpans, uncollapsedSpans := waterfallTrace.GetSelectedSpans(
req.UncollapsedSpans,
req.SelectedSpanID,
m.config.Waterfall.SpanPageSize,
m.config.Waterfall.MaxDepthToAutoExpand,
)
// Step 2: full fetch for the selected window only
spanIDs := make([]string, len(selectedSpans))
for i, s := range selectedSpans {
spanIDs[i] = s.SpanID
}
fullSpans, err := m.store.GetTraceSpansByIDs(ctx, traceID, summary, spanIDs)
if err != nil {
return nil, err
}
spantypes.EnrichSelectedSpans(selectedSpans, fullSpans)
return spantypes.NewGettableWaterfallTrace(
waterfallTrace, selectedSpans, uncollapsedSpans, false, nil,
), nil
}

View File

@@ -12,6 +12,9 @@ import (
"github.com/SigNoz/signoz/pkg/types/spantypes"
)
// The $$$$ becomes $$ since go-sqlbuilder escapes $ sign.
const serviceNameCol = "resource_string_service$$$$name"
type traceStore struct {
telemetryStore telemetrystore.TelemetryStore
}
@@ -69,3 +72,64 @@ func (s *traceStore) GetTraceSpans(ctx context.Context, traceID string, summary
}
return spanItems, nil
}
func (s *traceStore) GetMinimalSpans(ctx context.Context, traceID string, summary *spantypes.TraceSummary) ([]spantypes.MinimalSpan, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select(
"DISTINCT ON (span_id) span_id",
"parent_span_id", "timestamp", "duration_nano", "has_error",
serviceNameCol,
)
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
sb.Where(
sb.E("trace_id", traceID),
sb.GE("ts_bucket_start", summary.Start.Unix()-1800),
sb.LE("ts_bucket_start", summary.End.Unix()),
)
sb.OrderByAsc("timestamp")
sb.OrderByAsc("name")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
var spans []spantypes.MinimalSpan
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &spans, query, args...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying minimal spans")
}
return spans, nil
}
func (s *traceStore) GetTraceSpansByIDs(ctx context.Context, traceID string, summary *spantypes.TraceSummary, spanIDs []string) ([]spantypes.StorableSpan, error) {
if len(spanIDs) == 0 {
return []spantypes.StorableSpan{}, nil
}
sb := sqlbuilder.NewSelectBuilder()
sb.Select(
"DISTINCT ON (span_id) timestamp",
"duration_nano", "span_id", "trace_id", "has_error", "kind",
serviceNameCol, "name", "links as references",
"attributes_string", "attributes_number", "attributes_bool", "resources_string",
"events", "status_message", "status_code_string", "kind_string", "parent_span_id",
"flags", "is_remote", "trace_state", "status_code",
"db_name", "db_operation", "http_method", "http_url", "http_host",
"external_http_method", "external_http_url", "response_status_code",
)
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
ids := make([]any, len(spanIDs))
for i, id := range spanIDs {
ids[i] = id
}
sb.Where(
sb.E("trace_id", traceID),
sb.In("span_id", ids...),
sb.GE("ts_bucket_start", summary.Start.Unix()-1800),
sb.LE("ts_bucket_start", summary.End.Unix()),
)
sb.OrderByAsc("timestamp")
sb.OrderByAsc("name")
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
var spans []spantypes.StorableSpan
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &spans, query, args...); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying trace spans by IDs")
}
return spans, nil
}

View File

@@ -10,9 +10,11 @@ import (
// Handler exposes HTTP handlers for trace detail APIs.
type Handler interface {
GetWaterfall(http.ResponseWriter, *http.Request)
GetWaterfallV4(http.ResponseWriter, *http.Request)
}
// Module defines the business logic for trace detail operations.
type Module interface {
GetWaterfall(ctx context.Context, traceID string, req *spantypes.PostableWaterfall) (*spantypes.GettableWaterfallTrace, error)
GetWaterfallV4(ctx context.Context, traceID string, req *spantypes.PostableWaterfall) (*spantypes.GettableWaterfallTrace, error)
}

View File

@@ -352,13 +352,13 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
link := r.prepareLinksToTraces(ctx, ts, smpl.Metric)
if link != "" && r.hostFromSource() != "" {
r.logger.InfoContext(ctx, "adding traces link to annotations", slog.String("annotation.link", fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)))
annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRelatedTraces, Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)})
annotations = append(annotations, ruletypes.Label{Name: "related_traces", Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)})
}
case ruletypes.AlertTypeLogs:
link := r.prepareLinksToLogs(ctx, ts, smpl.Metric)
if link != "" && r.hostFromSource() != "" {
r.logger.InfoContext(ctx, "adding logs link to annotations", slog.String("annotation.link", fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)))
annotations = append(annotations, ruletypes.Label{Name: ruletypes.AnnotationRelatedLogs, Value: fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)})
annotations = append(annotations, ruletypes.Label{Name: "related_logs", Value: fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)})
}
}

View File

@@ -869,7 +869,7 @@ func TestThresholdRuleTracesLink(t *testing.T) {
assert.Equal(t, c.expectAlerts, alertsFound, "case %d", idx)
for _, item := range rule.Active {
for name, value := range item.Annotations.Map() {
if name == ruletypes.AnnotationRelatedTraces {
if name == "related_traces" {
assert.NotEmpty(t, value, "case %d", idx)
assert.Contains(t, value, "GET")
}
@@ -986,7 +986,7 @@ func TestThresholdRuleLogsLink(t *testing.T) {
assert.Equal(t, c.expectAlerts, alertsFound, "case %d", idx)
for _, item := range rule.Active {
for name, value := range item.Annotations.Map() {
if name == ruletypes.AnnotationRelatedLogs {
if name == "related_logs" {
assert.NotEmpty(t, value, "case %d", idx)
assert.Contains(t, value, "testcontainer")
}

View File

@@ -1,20 +0,0 @@
package alertmanagertypes
import (
"context"
"log/slog"
"github.com/prometheus/alertmanager/notify"
"github.com/prometheus/alertmanager/template"
"github.com/prometheus/alertmanager/types"
)
// Templater expands user-authored title and body templates against a group
// of alerts. Implemented by pkg/alertmanager/alertmanagertemplate.
type Templater interface {
Expand(ctx context.Context, req ExpandRequest, alerts []*types.Alert) (*ExpandResult, error)
}
// ReceiverIntegrationsFunc constructs the notify.Integration list for a
// configured receiver.
type ReceiverIntegrationsFunc = func(nc Receiver, tmpl *template.Template, logger *slog.Logger, templater Templater) ([]notify.Integration, error)

View File

@@ -19,7 +19,8 @@ import (
type (
// Receiver is the type for the receiver configuration.
Receiver = config.Receiver
Receiver = config.Receiver
ReceiverIntegrationsFunc = func(nc Receiver, tmpl *template.Template, logger *slog.Logger) ([]notify.Integration, error)
)
// Creates a new receiver from a string. The input is initialized with the default values from the upstream alertmanager.
@@ -50,7 +51,7 @@ func NewReceiver(input string) (Receiver, error) {
return receiverWithDefaults, nil
}
func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFunc ReceiverIntegrationsFunc, config *Config, tmpl *template.Template, logger *slog.Logger, templater Templater, lSet model.LabelSet, alert ...*Alert) error {
func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFunc ReceiverIntegrationsFunc, config *Config, tmpl *template.Template, logger *slog.Logger, lSet model.LabelSet, alert ...*Alert) error {
ctx = notify.WithGroupKey(ctx, fmt.Sprintf("%s-%s-%d", receiver.Name, lSet.Fingerprint(), time.Now().Unix()))
ctx = notify.WithGroupLabels(ctx, lSet)
ctx = notify.WithReceiverName(ctx, receiver.Name)
@@ -72,7 +73,7 @@ func TestReceiver(ctx context.Context, receiver Receiver, receiverIntegrationsFu
return err
}
integrations, err := receiverIntegrationsFunc(receiver, tmpl, logger, templater)
integrations, err := receiverIntegrationsFunc(receiver, tmpl, logger)
if err != nil {
return err
}

View File

@@ -77,28 +77,6 @@ func (c CompareOperator) Normalize() CompareOperator {
}
}
// Literal returns the canonical literal (string) form of the operator.
func (c CompareOperator) Literal() string {
switch c.Normalize() {
case ValueIsAbove:
return ValueIsAboveLiteral.StringValue()
case ValueIsBelow:
return ValueIsBelowLiteral.StringValue()
case ValueIsEq:
return ValueIsEqLiteral.StringValue()
case ValueIsNotEq:
return ValueIsNotEqLiteral.StringValue()
case ValueAboveOrEq:
return ValueAboveOrEqLiteral.StringValue()
case ValueBelowOrEq:
return ValueBelowOrEqLiteral.StringValue()
case ValueOutsideBounds:
return ValueOutsideBoundsLiteral.StringValue()
default:
return c.StringValue()
}
}
func (c CompareOperator) Validate() error {
switch c {
case ValueIsAbove,

View File

@@ -56,24 +56,6 @@ func (m MatchType) Normalize() MatchType {
}
}
// Literal returns the canonical literal (string) form of the match type.
func (m MatchType) Literal() string {
switch m.Normalize() {
case AtleastOnce:
return AtleastOnceLiteral.StringValue()
case AllTheTimes:
return AllTheTimesLiteral.StringValue()
case OnAverage:
return OnAverageLiteral.StringValue()
case InTotal:
return InTotalLiteral.StringValue()
case Last:
return LastLiteral.StringValue()
default:
return m.StringValue()
}
}
func (m MatchType) Validate() error {
switch m {
case

View File

@@ -24,10 +24,6 @@ type Sample struct {
RecoveryTarget *float64
TargetUnit string
// CompareOperator and MatchType carry the threshold evaluation context
CompareOperator CompareOperator
MatchType MatchType
}
func (s Sample) String() string {

View File

@@ -188,8 +188,6 @@ func (r BasicRuleThresholds) Eval(s *qbtypes.TimeSeries, unit string, evalData E
smpl.RecoveryTarget = threshold.RecoveryTarget
}
smpl.TargetUnit = threshold.TargetUnit
smpl.CompareOperator = threshold.CompareOperator
smpl.MatchType = threshold.MatchType
resultVector = append(resultVector, smpl)
continue
} else if evalData.SendUnmatched {
@@ -199,12 +197,10 @@ func (r BasicRuleThresholds) Eval(s *qbtypes.TimeSeries, unit string, evalData E
}
// prepare the sample with the first point of the series
smpl := Sample{
Point: Point{T: series.Values[0].Timestamp, V: series.Values[0].Value},
Metric: PrepareSampleLabelsForRule(series.Labels, threshold.Name),
Target: *threshold.TargetValue,
TargetUnit: threshold.TargetUnit,
CompareOperator: threshold.CompareOperator,
MatchType: threshold.MatchType,
Point: Point{T: series.Values[0].Timestamp, V: series.Values[0].Value},
Metric: PrepareSampleLabelsForRule(series.Labels, threshold.Name),
Target: *threshold.TargetValue,
TargetUnit: threshold.TargetUnit,
}
if threshold.RecoveryTarget != nil {
smpl.RecoveryTarget = threshold.RecoveryTarget
@@ -226,8 +222,6 @@ func (r BasicRuleThresholds) Eval(s *qbtypes.TimeSeries, unit string, evalData E
smpl.Target = *threshold.TargetValue
smpl.RecoveryTarget = threshold.RecoveryTarget
smpl.TargetUnit = threshold.TargetUnit
smpl.CompareOperator = threshold.CompareOperator
smpl.MatchType = threshold.MatchType
// IsRecovering to notify that metrics is in recovery stage
smpl.IsRecovering = true
resultVector = append(resultVector, smpl)

View File

@@ -26,4 +26,6 @@ type SpanMapperStore interface {
type TraceStore interface {
GetTraceSummary(ctx context.Context, traceID string) (*TraceSummary, error)
GetTraceSpans(ctx context.Context, traceID string, summary *TraceSummary) ([]StorableSpan, error)
GetMinimalSpans(ctx context.Context, traceID string, summary *TraceSummary) ([]MinimalSpan, error)
GetTraceSpansByIDs(ctx context.Context, traceID string, summary *TraceSummary, spanIDs []string) ([]StorableSpan, error)
}

View File

@@ -132,6 +132,31 @@ type StorableSpan struct {
ResponseStatusCode string `ch:"response_status_code"`
}
// MinimalSpan with only the fields needed to build the parent-child tree.
type MinimalSpan struct {
SpanID string `ch:"span_id"`
ParentSpanID string `ch:"parent_span_id"`
StartTime time.Time `ch:"timestamp"`
DurationNano uint64 `ch:"duration_nano"`
HasError bool `ch:"has_error"`
ServiceName string `ch:"resource_string_service$$name"`
}
func (item *MinimalSpan) ToWaterfallSpan() *WaterfallSpan {
return &WaterfallSpan{
SpanID: item.SpanID,
ParentSpanID: item.ParentSpanID,
TimeUnix: uint64(item.StartTime.UnixNano()),
DurationNano: item.DurationNano,
HasError: item.HasError,
ServiceName: item.ServiceName,
Resource: map[string]string{"service.name": item.ServiceName},
Children: make([]*WaterfallSpan, 0),
Attributes: make(map[string]any),
Events: make([]Event, 0),
}
}
// NewMissingWaterfallSpan creates a synthetic placeholder span for a parent that has no recorded data.
func NewMissingWaterfallSpan(spanID, traceID string, timeUnixNano, durationNano uint64) *WaterfallSpan {
return &WaterfallSpan{
@@ -297,6 +322,24 @@ func (item *StorableSpan) ToWaterfallSpan() *WaterfallSpan {
}
}
func EnrichSelectedSpans(window []*WaterfallSpan, fullSpans []StorableSpan) {
fullByID := make(map[string]*StorableSpan, len(fullSpans))
for i := range fullSpans {
fullByID[fullSpans[i].SpanID] = &fullSpans[i]
}
for i, ws := range window {
full, ok := fullByID[ws.SpanID]
if !ok {
continue // synthesized MissingSpan — keep empty shell
}
newWS := full.ToWaterfallSpan()
newWS.Level = ws.Level
newWS.HasChildren = ws.HasChildren
newWS.SubTreeNodeCount = ws.SubTreeNodeCount
window[i] = newWS
}
}
// getSpanIndex returns the index of matched span and -1 for no match.
func getSpanIndex(spans []*WaterfallSpan, targetSpanID string) int {
for i, s := range spans {

View File

@@ -62,26 +62,24 @@ func NewWaterfallTrace(
}
}
func NewWaterfallTraceFromSpans(spans []StorableSpan) *WaterfallTrace {
// NewWaterfallTraceFromSpans requires WaterfallSpan nodes with only below fields:
// SpanID, ParentSpanID, TimeUnix, DurationNano, HasError, and ServiceName.
func NewWaterfallTraceFromSpans(nodes []*WaterfallSpan) *WaterfallTrace {
var (
startTime, endTime, totalErrorSpans uint64
spanIDToSpanNodeMap = make(map[string]*WaterfallSpan, len(spans))
spanIDToSpanNodeMap = make(map[string]*WaterfallSpan, len(nodes))
traceRoots []*WaterfallSpan
hasMissingSpans bool
)
for _, item := range spans {
span := item.ToWaterfallSpan()
startTimeUnixNano := uint64(item.StartTime.UnixNano())
if startTime == 0 || startTimeUnixNano < startTime {
startTime = startTimeUnixNano
for _, span := range nodes {
if startTime == 0 || span.TimeUnix < startTime {
startTime = span.TimeUnix
}
endTime = max(endTime, startTimeUnixNano+span.DurationNano)
endTime = max(endTime, span.TimeUnix+span.DurationNano)
if span.HasError {
totalErrorSpans++
}
spanIDToSpanNodeMap[span.SpanID] = span
}
@@ -116,7 +114,7 @@ func NewWaterfallTraceFromSpans(spans []StorableSpan) *WaterfallTrace {
return NewWaterfallTrace(
startTime,
endTime,
uint64(len(spans)),
uint64(len(nodes)),
totalErrorSpans,
spanIDToSpanNodeMap,
traceRoots,

View File

@@ -1,122 +0,0 @@
{{ define "email.signoz.html" }}
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>{{.Title}}</title>
<style>
code {
background: #f0f0f0;
padding: 2px 6px;
border-radius: 3px;
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
font-size: 13px;
}
pre {
background: #f0f0f0;
padding: 12px 16px;
border-radius: 6px;
font-size: 13px;
overflow-x: auto;
white-space: pre;
}
pre code {
background: none;
padding: 0;
border-radius: 0;
font-size: inherit;
}
table:not([role="presentation"]) {
width: 100%;
border-collapse: collapse;
font-size: 14px;
}
table:not([role="presentation"]) th {
font-weight: 600;
text-align: left;
padding: 8px 12px;
border-bottom: 2px solid #d0d0d0;
}
table:not([role="presentation"]) td {
padding: 8px 12px;
border-bottom: 1px solid #e8e8e8;
}
table:not([role="presentation"]) tr:last-child td {
border-bottom: none;
}
</style>
</head>
<body style="margin:0;padding:0;font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,'Helvetica Neue',Arial,sans-serif;line-height:1.6;color:#333;background:#fff">
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0" style="background:#fff">
<tr>
<td align="center" style="padding:0">
<table role="presentation" width="600" cellspacing="0" cellpadding="0" border="0" style="max-width:600px;width:100%;border:1px solid #e2e2e2;border-radius:12px;overflow:hidden">
<tr>
<td align="center" style="padding:20px 20px 12px">
<h2 style="margin:0 0 8px;font-size:20px;color:#333">{{.Title}}</h2>
<p style="margin:0;font-size:14px;color:#666">
Status: <strong>{{.Alert.Status}}</strong>
{{if .Alert.TotalFiring}} | Firing: <strong style="color:#e53e3e">{{.Alert.TotalFiring}}</strong>{{end}}
{{if .Alert.TotalResolved}} | Resolved: <strong style="color:#38a169">{{.Alert.TotalResolved}}</strong>{{end}}
</p>
</td>
</tr>
<tr>
<td style="padding:0 20px">
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
<tr><td style="border-top:1px solid #e2e2e2;font-size:0;line-height:0" height="1">&nbsp;</td></tr>
</table>
</td>
</tr>
{{range .Bodies}}
<tr>
<td style="padding:8px 20px">
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
<tr>
<td style="padding:16px;background:#fafafa;border:1px solid #e8e8e8;border-radius:6px">
{{.}}
</td>
</tr>
</table>
</td>
</tr>
{{end}}
{{if .NotificationTemplateData.ExternalURL}}
<tr>
<td style="padding:16px 20px">
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0">
<tr>
<td align="center">
<a href="{{.NotificationTemplateData.ExternalURL}}" target="_blank" style="display:inline-block;padding:12px 32px;font-size:14px;font-weight:600;color:#fff;background:#4E74F8;text-decoration:none;border-radius:4px">
View in SigNoz
</a>
</td>
</tr>
</table>
</td>
</tr>
{{end}}
<tr>
<td align="center" style="padding:8px 16px 16px">
<p style="margin:0;font-size:12px;color:#999;line-height:1.5">
Sent by SigNoz AlertManager
</p>
</td>
</tr>
</table>
</td>
</tr>
</table>
</body>
</html>
{{ end }}

View File

@@ -28,7 +28,6 @@ pytest_plugins = [
"fixtures.serviceaccount",
"fixtures.role",
"fixtures.seed_golden_dataset",
"fixtures.maildev",
]

View File

@@ -4,7 +4,6 @@ import time
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from urllib.parse import urlparse
import pytest
import requests
@@ -14,7 +13,6 @@ from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.fs import get_testdata_file_path
from fixtures.logger import setup_logger
from fixtures.logs import Logs
from fixtures.maildev import verify_email_received
from fixtures.metrics import Metrics
from fixtures.traces import Traces
@@ -220,118 +218,3 @@ def update_rule_channel_name(rule_data: dict, channel_name: str):
# loop over all the sepcs and update the channels
for spec in thresholds["spec"]:
spec["channels"] = [channel_name]
def _is_json_subset(subset, superset) -> bool:
"""Check if subset is contained within superset recursively.
- For dicts: all keys in subset must exist in superset with matching values
- For lists: all items in subset must be present in superset
- For scalars: exact equality
"""
if isinstance(subset, dict):
if not isinstance(superset, dict):
return False
return all(key in superset and _is_json_subset(value, superset[key]) for key, value in subset.items())
if isinstance(subset, list):
if not isinstance(superset, list):
return False
return all(any(_is_json_subset(sub_item, sup_item) for sup_item in superset) for sub_item in subset)
return subset == superset
def verify_webhook_notification_expectation(
notification_channel: types.TestContainerDocker,
validation_data: dict,
) -> bool:
"""Check if wiremock received a request at the given path
whose JSON body is a superset of the expected json_body."""
path = validation_data["path"]
json_body = validation_data["json_body"]
url = notification_channel.host_configs["8080"].get("__admin/requests/find")
res = requests.post(url, json={"method": "POST", "url": path}, timeout=5)
assert res.status_code == HTTPStatus.OK, f"Failed to find requests for path {path}, status code: {res.status_code}, response: {res.text}"
for req in res.json()["requests"]:
body = json.loads(base64.b64decode(req["bodyAsBase64"]).decode("utf-8"))
# logger.info("Webhook request body: %s", json.dumps(body, indent=2))
if _is_json_subset(json_body, body):
return True
return False
def _check_notification_validation(
validation: types.NotificationValidation,
notification_channel: types.TestContainerDocker,
maildev: types.TestContainerDocker,
) -> bool:
"""Dispatch a single validation check to the appropriate verifier."""
if validation.destination_type == "webhook":
return verify_webhook_notification_expectation(notification_channel, validation.validation_data)
if validation.destination_type == "email":
return verify_email_received(maildev, validation.validation_data)
raise ValueError(f"Invalid destination type: {validation.destination_type}")
def verify_notification_expectation(
notification_channel: types.TestContainerDocker,
maildev: types.TestContainerDocker,
expected_notification: types.AMNotificationExpectation,
) -> bool:
"""Poll for expected notifications across webhook and email channels."""
time_to_wait = datetime.now() + timedelta(seconds=expected_notification.wait_time_seconds)
while datetime.now() < time_to_wait:
all_found = all(_check_notification_validation(v, notification_channel, maildev) for v in expected_notification.notification_validations)
if expected_notification.should_notify and all_found:
logger.info("All expected notifications found")
return True
time.sleep(1)
# Timeout reached
if not expected_notification.should_notify:
# Verify no notifications were received
for validation in expected_notification.notification_validations:
found = _check_notification_validation(validation, notification_channel, maildev)
assert not found, f"Expected no notification but found one for {validation.destination_type} with data {validation.validation_data}"
logger.info("No notifications found, as expected")
return True
# Expected notifications but didn't get them all — report missing
missing = [v for v in expected_notification.notification_validations if not _check_notification_validation(v, notification_channel, maildev)]
assert len(missing) == 0, f"Expected all notifications to be found but missing: {missing}"
return True
def update_raw_channel_config(
channel_config: dict,
channel_name: str,
notification_channel: types.TestContainerDocker,
) -> dict:
"""
Updates the channel config to point to the given wiremock
notification_channel container to receive notifications.
"""
config = channel_config.copy()
config["name"] = channel_name
url_field_map = {
"slack_configs": "api_url",
"msteamsv2_configs": "webhook_url",
"webhook_configs": "url",
"pagerduty_configs": "url",
"opsgenie_configs": "api_url",
}
for config_key, url_field in url_field_map.items():
if config_key in config:
for entry in config[config_key]:
if url_field in entry:
original_url = entry[url_field]
path = urlparse(original_url).path
entry[url_field] = notification_channel.container_configs["8080"].get(path)
return config

View File

@@ -124,19 +124,14 @@ def gateway(
@pytest.fixture(name="make_http_mocks", scope="function")
def make_http_mocks(
request: pytest.FixtureRequest,
) -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
def make_http_mocks() -> Callable[[types.TestContainerDocker, list[Mapping]], None]:
def _make_http_mocks(container: types.TestContainerDocker, mappings: list[Mapping]) -> None:
Config.base_url = container.host_configs["8080"].get("/__admin")
for mapping in mappings:
Mappings.create_mapping(mapping=mapping)
def cleanup():
Mappings.delete_all_mappings()
Requests.reset_request_journal()
yield _make_http_mocks
request.addfinalizer(cleanup)
return _make_http_mocks
Mappings.delete_all_mappings()
Requests.reset_request_journal()

View File

@@ -1,122 +0,0 @@
import json
from http import HTTPStatus
import docker
import docker.errors
import pytest
import requests
from testcontainers.core.container import DockerContainer, Network
from fixtures import reuse, types
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
@pytest.fixture(name="maildev", scope="package")
def maildev(network: Network, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.TestContainerDocker:
"""
Package-scoped fixture for MailDev container.
Provides SMTP (port 1025) and HTTP API (port 1080) for email testing.
"""
def create() -> types.TestContainerDocker:
container = DockerContainer(image="maildev/maildev:2.2.1")
container.with_exposed_ports(1025, 1080)
container.with_network(network=network)
container.start()
return types.TestContainerDocker(
id=container.get_wrapped_container().id,
host_configs={
"1025": types.TestContainerUrlConfig(
scheme="smtp",
address=container.get_container_host_ip(),
port=container.get_exposed_port(1025),
),
"1080": types.TestContainerUrlConfig(
scheme="http",
address=container.get_container_host_ip(),
port=container.get_exposed_port(1080),
),
},
container_configs={
"1025": types.TestContainerUrlConfig(
scheme="smtp",
address=container.get_wrapped_container().name,
port=1025,
),
"1080": types.TestContainerUrlConfig(
scheme="http",
address=container.get_wrapped_container().name,
port=1080,
),
},
)
def delete(container: types.TestContainerDocker):
client = docker.from_env()
try:
client.containers.get(container_id=container.id).stop()
client.containers.get(container_id=container.id).remove(v=True)
except docker.errors.NotFound:
logger.info(
"Skipping removal of MailDev, MailDev(%s) not found. Maybe it was manually removed?",
{"id": container.id},
)
def restore(cache: dict) -> types.TestContainerDocker:
return types.TestContainerDocker.from_cache(cache)
return reuse.wrap(
request,
pytestconfig,
"maildev",
lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
create,
delete,
restore,
)
def get_all_mails(_maildev: types.TestContainerDocker) -> list[dict]:
"""
Fetches all emails from the MailDev HTTP API.
Returns list of dicts with keys: subject, html, text.
"""
url = _maildev.host_configs["1080"].get("/email")
response = requests.get(url, timeout=5)
assert response.status_code == HTTPStatus.OK, f"Failed to fetch emails from MailDev, status code: {response.status_code}, response: {response.text}"
emails = response.json()
# logger.info("Emails: %s", json.dumps(emails, indent=2))
return [
{
"subject": email.get("subject", ""),
"html": email.get("html", ""),
"text": email.get("text", ""),
}
for email in emails
]
def verify_email_received(_maildev: types.TestContainerDocker, filters: dict) -> bool:
"""
Checks if any email in MailDev matches all the given filters.
Filters are matched with exact equality against the email fields (subject, html, text).
Returns True if at least one matching email is found.
"""
emails = get_all_mails(_maildev)
for email in emails:
logger.info("Email: %s", json.dumps(email, indent=2))
if all(key in email and filter_value == email[key] for key, filter_value in filters.items()):
return True
return False
def delete_all_mails(_maildev: types.TestContainerDocker) -> None:
"""
Deletes all emails from the MailDev inbox.
"""
url = _maildev.host_configs["1080"].get("/email/all")
response = requests.delete(url, timeout=5)
assert response.status_code == HTTPStatus.OK, f"Failed to delete emails from MailDev, status code: {response.status_code}, response: {response.text}"

View File

@@ -1,4 +1,3 @@
# pylint: disable=line-too-long
from collections.abc import Callable
from http import HTTPStatus
@@ -16,87 +15,6 @@ from fixtures.logger import setup_logger
logger = setup_logger(__name__)
"""
Default notification channel configs shared across alertmanager tests.
"""
slack_default_config = {
# channel name configured on runtime
"slack_configs": [
{
"api_url": "services/TEAM_ID/BOT_ID/TOKEN_ID", # base_url configured on runtime
"title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
"text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
}
],
}
# MSTeams default config
msteams_default_config = {
"msteamsv2_configs": [
{
"webhook_url": "msteams/webhook_url", # base_url configured on runtime
"title": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}',
"text": '{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n {{ end -}}\r\n {{ end -}}\r\n{{ end }}',
}
],
}
# pagerduty default config
pagerduty_default_config = {
"pagerduty_configs": [
{
"routing_key": "PagerDutyRoutingKey",
"url": "v2/enqueue", # base_url configured on runtime
"client": "SigNoz Alert Manager",
"client_url": "https://enter-signoz-host-n-port-here/alerts",
"description": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n\t{{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n\t {{" "}}(\n\t {{- with .CommonLabels.Remove .GroupLabels.Names }}\n\t\t{{- range $index, $label := .SortedPairs -}}\n\t\t {{ if $index }}, {{ end }}\n\t\t {{- $label.Name }}="{{ $label.Value -}}"\n\t\t{{- end }}\n\t {{- end -}}\n\t )\n\t{{- end }}',
"details": {
"firing": '{{ template "pagerduty.default.instances" .Alerts.Firing }}',
"num_firing": "{{ .Alerts.Firing | len }}",
"num_resolved": "{{ .Alerts.Resolved | len }}",
"resolved": '{{ template "pagerduty.default.instances" .Alerts.Resolved }}',
},
"source": "SigNoz Alert Manager",
"severity": "{{ (index .Alerts 0).Labels.severity }}",
}
],
}
# opsgenie default config
opsgenie_default_config = {
"opsgenie_configs": [
{
"api_key": "OpsGenieAPIKey",
"api_url": "/", # base_url configured on runtime
"description": '{{ if gt (len .Alerts.Firing) 0 -}}\r\n\tAlerts Firing:\r\n\t{{ range .Alerts.Firing }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }} Annotations:\r\n\t{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end }} Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}\r\n{{ if gt (len .Alerts.Resolved) 0 -}}\r\n\tAlerts Resolved:\r\n\t{{ range .Alerts.Resolved }}\r\n\t - Message: {{ .Annotations.description }}\r\n\tLabels:\r\n\t{{ range .Labels.SortedPairs -}}\r\n\t\t{{- if ne .Name "ruleId" }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end -}}\r\n\t{{- end }} Annotations:\r\n\t{{ range .Annotations.SortedPairs }} - {{ .Name }} = {{ .Value }}\r\n\t{{ end }} Source: {{ .GeneratorURL }}\r\n\t{{ end }}\r\n{{- end }}',
"priority": '{{ if eq (index .Alerts 0).Labels.severity "critical" }}P1{{ else if eq (index .Alerts 0).Labels.severity "warning" }}P2{{ else if eq (index .Alerts 0).Labels.severity "info" }}P3{{ else }}P4{{ end }}',
"message": "{{ .CommonLabels.alertname }}",
"details": {},
}
]
}
# webhook default config
webhook_default_config = {
"webhook_configs": [
{
"url": "webhook/webhook_url", # base_url configured on runtime
}
],
}
# email default config
email_default_config = {
"email_configs": [
{
"to": "test@example.com",
"html": '<html><body>{{ range .Alerts -}}\r\n *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}\r\n\r\n *Summary:* {{ .Annotations.summary }}\r\n *Description:* {{ .Annotations.description }}\r\n *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}\r\n *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}\r\n\r\n *Details:*\r\n {{ range .Labels.SortedPairs -}}\r\n {{- if ne .Name "ruleId" -}}\r\n \u2022 *{{ .Name }}:* {{ .Value }}\r\n {{ end -}}\r\n {{ end -}}\r\n{{ end }}</body></html>',
"headers": {
"Subject": '[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}\n {{- if gt (len .CommonLabels) (len .GroupLabels) -}}\n {{" "}}(\n {{- with .CommonLabels.Remove .GroupLabels.Names }}\n {{- range $index, $label := .SortedPairs -}}\n {{ if $index }}, {{ end }}\n {{- $label.Name }}="{{ $label.Value -}}"\n {{- end }}\n {{- end -}}\n )\n {{- end }}'
},
}
],
}
@pytest.fixture(name="notification_channel", scope="package")
def notification_channel(
network: Network,
@@ -149,27 +67,6 @@ def notification_channel(
)
@pytest.fixture(name="create_notification_channel", scope="function")
def create_notification_channel(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> Callable[[dict], str]:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
def _create_notification_channel(channel_config: dict) -> str:
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/channels"),
json=channel_config,
headers={"Authorization": f"Bearer {admin_token}"},
timeout=5,
)
assert response.status_code == HTTPStatus.CREATED, f"Failed to create channel, Response: {response.text} Response status: {response.status_code}"
return response.json()["data"]["id"]
return _create_notification_channel
@pytest.fixture(name="create_webhook_notification_channel", scope="function")
def create_webhook_notification_channel(
signoz: types.SigNoz,

View File

@@ -192,40 +192,3 @@ class AlertTestCase:
alert_data: list[AlertData]
# list of alert expectations for the test case
alert_expectation: AlertExpectation
@dataclass(frozen=True)
class NotificationValidation:
# destination type of the notification, either webhook or email
# slack, msteams, pagerduty, opsgenie, webhook channels send notifications through webhook
# email channels send notifications through email
destination_type: Literal["webhook", "email"]
# validation data for validating the received notification payload
validation_data: dict[str, any]
@dataclass(frozen=True)
class AMNotificationExpectation:
# whether we expect any notifications to be fired or not, false when testing downtime scenarios
# or don't expect any notifications to be fired in given time period
should_notify: bool
# seconds to wait for the notifications to be fired, if no
# notifications are fired in the expected time, the test will fail
wait_time_seconds: int
# list of notifications to expect, as a single rule can trigger multiple notifications
# spanning across different notifiers
notification_validations: list[NotificationValidation]
@dataclass(frozen=True)
class AlertManagerNotificationTestCase:
# name of the test case
name: str
# path to the rule file in testdata directory
rule_path: str
# list of alert data that will be inserted into the database for the rule to be triggered
alert_data: list[AlertData]
# configuration for the notification channel
channel_config: dict[str, any]
# notification expectations for the test case
notification_expectation: AMNotificationExpectation

View File

@@ -39,7 +39,5 @@ def test_teardown(
idp: types.TestContainerIDP, # pylint: disable=unused-argument
create_user_admin: types.Operation, # pylint: disable=unused-argument
migrator: types.Operation, # pylint: disable=unused-argument
maildev: types.TestContainerDocker, # pylint: disable=unused-argument
notification_channel: types.TestContainerDocker, # pylint: disable=unused-argument
) -> None:
pass

View File

@@ -1,20 +0,0 @@
{ "timestamp": "2026-01-29T10:00:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "User login successful", "severity_text": "INFO" }
{ "timestamp": "2026-01-29T10:00:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:01:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:01:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Database connection established", "severity_text": "INFO" }
{ "timestamp": "2026-01-29T10:02:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:02:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:03:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "API request received", "severity_text": "INFO" }
{ "timestamp": "2026-01-29T10:03:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:04:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:04:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:05:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:05:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:06:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:06:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: insufficient funds", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:07:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:07:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:08:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "Response sent to client", "severity_text": "INFO" }
{ "timestamp": "2026-01-29T10:08:30.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: invalid token", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:09:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: card declined", "severity_text": "ERROR" }
{ "timestamp": "2026-01-29T10:10:00.000000Z", "resources": { "service.name": "payment-service" }, "attributes": { "code.file": "payment_handler.py" }, "body": "payment failure: gateway timeout", "severity_text": "ERROR" }

View File

@@ -1,71 +0,0 @@
{
"alert": "content_templating_logs",
"ruleType": "threshold_rule",
"alertType": "LOGS_BASED_ALERT",
"condition": {
"thresholds": {
"kind": "basic",
"spec": [
{
"name": "critical",
"target": 0,
"matchType": "1",
"op": "1",
"channels": [
"test channel"
]
}
]
},
"compositeQuery": {
"queryType": "builder",
"panelType": "graph",
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "logs",
"filter": {
"expression": "body CONTAINS 'payment failure'"
},
"aggregations": [
{
"expression": "count()"
}
],
"groupBy": [
{"name": "service.name", "fieldContext": "resource"}
]
}
}
]
},
"selectedQueryName": "A"
},
"evaluation": {
"kind": "rolling",
"spec": {
"evalWindow": "5m0s",
"frequency": "15s"
}
},
"labels": {},
"annotations": {
"description": "Payment failure spike detected on $service_name",
"summary": "Payment failures elevated on $service_name",
"_title_template": "[$alert.status] Payment failure spike in $labels.service.name",
"_body_template": "**Severity:** $rule.severity\n**Status:** $alert.status\n\n**Service:** $labels.service.name\n\n**Condition ($labels.threshold.name):**\n- **Current:** $alert.value\n- **Threshold:** $rule.threshold.op $rule.threshold.value\n\n**Description:** Payment failures observed at $alert.value over the evaluation window, crossing the $labels.threshold.name threshold of $rule.threshold.op $rule.threshold.value on $labels.service.name. Investigate downstream payment processor health.\n\n**Runbook:** https://signoz.io/docs/runbooks/payment-failure-spike"
},
"notificationSettings": {
"groupBy": [],
"usePolicy": false,
"renotify": {
"enabled": false,
"interval": "30m",
"alertStates": []
}
},
"version": "v5",
"schemaVersion": "v2alpha1"
}

View File

@@ -1,12 +0,0 @@
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:01:00+00:00","value":80,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:02:00+00:00","value":95,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:03:00+00:00","value":110,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:04:00+00:00","value":120,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:05:00+00:00","value":125,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:06:00+00:00","value":130,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:07:00+00:00","value":135,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:08:00+00:00","value":140,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:09:00+00:00","value":145,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:10:00+00:00","value":150,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:11:00+00:00","value":155,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}
{"metric_name":"container_memory_bytes_content_templating","labels":{"namespace":"production","pod":"checkout-7d9c8b5f4-x2k9p","container":"checkout","node":"ip-10-0-1-23","severity":"critical","service":"checkout"},"timestamp":"2026-01-29T10:12:00+00:00","value":160,"temporality":"Unspecified","type_":"Gauge","is_monotonic":false,"flags":0,"description":"","unit":"bytes","env":"default","resource_attrs":{},"scope_attrs":{}}

View File

@@ -1,74 +0,0 @@
{
"alert": "content_templating_metrics",
"ruleType": "threshold_rule",
"alertType": "METRIC_BASED_ALERT",
"condition": {
"thresholds": {
"kind": "basic",
"spec": [
{
"name": "critical",
"target": 100,
"matchType": "1",
"op": "1",
"channels": [
"test channel"
]
}
]
},
"compositeQuery": {
"queryType": "builder",
"panelType": "graph",
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "metrics",
"aggregations": [
{
"metricName": "container_memory_bytes_content_templating",
"timeAggregation": "avg",
"spaceAggregation": "max"
}
],
"groupBy": [
{"name": "namespace", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "pod", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "container", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "node", "fieldContext": "attribute", "fieldDataType": "string"},
{"name": "severity", "fieldContext": "attribute", "fieldDataType": "string"}
]
}
}
]
},
"selectedQueryName": "A"
},
"evaluation": {
"kind": "rolling",
"spec": {
"evalWindow": "5m0s",
"frequency": "15s"
}
},
"labels": {},
"annotations": {
"description": "Container $container in pod $pod ($namespace) exceeded memory threshold",
"summary": "High container memory in $namespace/$pod",
"_title_template": "[$alert.status] High container memory in $labels.namespace/$labels.pod",
"_body_template": "**Severity:** $rule.severity\n**Status:** $alert.status\n\n**Pod Details:**\n- **Namespace:** $labels.namespace\n- **Pod:** $labels.pod\n- **Container:** $labels.container\n- **Node:** $labels.node\n\n**Condition ($labels.threshold.name):**\n- **Current:** $alert.value\n- **Threshold:** $rule.threshold.op $rule.threshold.value\n\n**Description:** Container $labels.container in pod $labels.pod ($labels.namespace) has memory usage at $alert.value, which crossed the $labels.threshold.name threshold of $rule.threshold.op $rule.threshold.value. Immediate investigation is recommended to prevent OOMKill.\n\n**Runbook:** https://signoz.io/docs/runbooks/container-memory-near-limit"
},
"notificationSettings": {
"groupBy": [],
"usePolicy": false,
"renotify": {
"enabled": false,
"interval": "30m",
"alertStates": []
}
},
"version": "v5",
"schemaVersion": "v2alpha1"
}

View File

@@ -1,20 +0,0 @@
{ "timestamp": "2026-01-29T10:00:00.000000Z", "duration": "PT1.2S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a1", "span_id": "c1b2c3d4e5f6a7b8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:00:30.000000Z", "duration": "PT1.4S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a2", "span_id": "c2b3c4d5e6f7a8b9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:01:00.000000Z", "duration": "PT1.6S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a3", "span_id": "c3b4c5d6e7f8a9b0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:01:30.000000Z", "duration": "PT1.8S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a4", "span_id": "c4b5c6d7e8f9a0b1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:02:00.000000Z", "duration": "PT2.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a5", "span_id": "c5b6c7d8e9f0a1b2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:02:30.000000Z", "duration": "PT2.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a6", "span_id": "c6b7c8d9e0f1a2b3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:03:00.000000Z", "duration": "PT2.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a7", "span_id": "c7b8c9d0e1f2a3b4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:03:30.000000Z", "duration": "PT2.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a8", "span_id": "c8b9c0d1e2f3a4b5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:04:00.000000Z", "duration": "PT2.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6a9", "span_id": "c9b0c1d2e3f4a5b6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:04:30.000000Z", "duration": "PT3.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b1", "span_id": "d1c2d3e4f5a6b7c8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:05:00.000000Z", "duration": "PT3.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b2", "span_id": "d2c3d4e5f6a7b8c9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:05:30.000000Z", "duration": "PT3.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b3", "span_id": "d3c4d5e6f7a8b9c0", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:06:00.000000Z", "duration": "PT3.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b4", "span_id": "d4c5d6e7f8a9b0c1", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:06:30.000000Z", "duration": "PT3.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b5", "span_id": "d5c6d7e8f9a0b1c2", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:07:00.000000Z", "duration": "PT4.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b6", "span_id": "d6c7d8e9f0a1b2c3", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:07:30.000000Z", "duration": "PT4.3S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b7", "span_id": "d7c8d9e0f1a2b3c4", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:08:00.000000Z", "duration": "PT4.5S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b8", "span_id": "d8c9d0e1f2a3b4c5", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:08:30.000000Z", "duration": "PT4.7S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6b9", "span_id": "d9c0d1e2f3a4b5c6", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:09:00.000000Z", "duration": "PT4.9S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c1", "span_id": "e1d2e3f4a5b6c7d8", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }
{ "timestamp": "2026-01-29T10:10:00.000000Z", "duration": "PT5.1S", "trace_id": "591f6d3d6b0a1f9e8a71b2c3d4e5f6c2", "span_id": "e2d3e4f5a6b7c8d9", "parent_span_id": "", "name": "POST /checkout", "kind": 2, "status_code": 1, "status_message": "", "resources": { "deployment.environment": "production", "service.name": "checkout-service", "os.type": "linux", "host.name": "ip-10-0-1-23" }, "attributes": { "net.transport": "IP.TCP", "http.scheme": "http", "http.user_agent": "Integration Test", "http.request.method": "POST", "http.response.status_code": "200", "http.request.path": "/checkout" } }

View File

@@ -1,73 +0,0 @@
{
"alert": "content_templating_traces",
"ruleType": "threshold_rule",
"alertType": "TRACES_BASED_ALERT",
"condition": {
"thresholds": {
"kind": "basic",
"spec": [
{
"name": "critical",
"target": 1,
"matchType": "1",
"op": "1",
"channels": [
"test channel"
],
"targetUnit": "s"
}
]
},
"compositeQuery": {
"queryType": "builder",
"unit": "ns",
"panelType": "graph",
"queries": [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "traces",
"filter": {
"expression": "http.request.path = '/checkout'"
},
"aggregations": [
{
"expression": "p90(duration_nano)"
}
],
"groupBy": [
{"name": "service.name", "fieldContext": "resource"}
]
}
}
]
},
"selectedQueryName": "A"
},
"evaluation": {
"kind": "rolling",
"spec": {
"evalWindow": "5m0s",
"frequency": "15s"
}
},
"labels": {},
"annotations": {
"description": "p90 latency high on $service_name",
"summary": "p90 latency exceeded threshold on $service_name",
"_title_template": "[$alert.status] p90 latency high on $labels.service_name",
"_body_template": "**Severity:** $rule.severity\n**Status:** $alert.status\n\n**Service:** $labels.service_name\n\n**Condition ($labels.threshold.name):**\n- **Current:** $alert.value\n- **Threshold:** $rule.threshold.op $rule.threshold.value\n\n**Description:** p90 request latency on $labels.service_name reached $alert.value, crossing the $labels.threshold.name threshold of $rule.threshold.op $rule.threshold.value. Investigate downstream dependencies and recent deploys.\n\n**Runbook:** https://signoz.io/docs/runbooks/high-latency"
},
"notificationSettings": {
"groupBy": [],
"usePolicy": false,
"renotify": {
"enabled": false,
"interval": "30m",
"alertStates": []
}
},
"version": "v5",
"schemaVersion": "v2alpha1"
}

View File

@@ -1,411 +0,0 @@
import json
import uuid
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
import pytest
from wiremock.client import HttpMethods, Mapping, MappingRequest, MappingResponse
from fixtures import types
from fixtures.alerts import (
get_testdata_file_path,
update_raw_channel_config,
update_rule_channel_name,
verify_notification_expectation,
)
from fixtures.logger import setup_logger
from fixtures.maildev import delete_all_mails
from fixtures.notification_channel import (
email_default_config,
msteams_default_config,
opsgenie_default_config,
pagerduty_default_config,
slack_default_config,
webhook_default_config,
)
# tests to verify the notifiers sending out the notifications with expected content
NOTIFIERS_TEST = [
types.AlertManagerNotificationTestCase(
name="slack_notifier_default_templating",
rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
),
],
channel_config=slack_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
# extra wait for alertmanager server setup
wait_time_seconds=60,
notification_validations=[
types.NotificationValidation(
destination_type="webhook",
validation_data={
"path": "/services/TEAM_ID/BOT_ID/TOKEN_ID",
"json_body": {
"username": "Alertmanager",
"attachments": [
{
"title": '[FIRING:1] threshold_above_at_least_once for (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
"text": "*Alert:* threshold_above_at_least_once - critical\r\n\r\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n *RelatedLogs:* \r\n *RelatedTraces:* \r\n\r\n *Details:*\r\n • *alertname:* threshold_above_at_least_once\r\n • *severity:* critical\r\n • *threshold.name:* critical\r\n ",
"color": "danger",
"mrkdwn_in": ["fallback", "pretext", "text"],
}
],
},
},
),
],
),
),
types.AlertManagerNotificationTestCase(
name="msteams_notifier_default_templating",
rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
),
],
channel_config=msteams_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
wait_time_seconds=60,
notification_validations=[
types.NotificationValidation(
destination_type="webhook",
validation_data={
"path": "/msteams/webhook_url",
"json_body": {
"type": "message",
"attachments": [
{
"contentType": "application/vnd.microsoft.card.adaptive",
"content": {
"$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
"type": "AdaptiveCard",
"version": "1.2",
"body": [
{
"type": "TextBlock",
"text": '[FIRING:1] threshold_above_at_least_once for (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
"weight": "Bolder",
"size": "Medium",
"wrap": True,
"style": "heading",
"color": "Attention",
},
{
"type": "TextBlock",
"text": "Alerts",
"weight": "Bolder",
"size": "Medium",
"wrap": True,
"color": "Attention",
},
{
"type": "TextBlock",
"text": "Labels",
"weight": "Bolder",
"size": "Medium",
},
{
"type": "FactSet",
"text": "",
"facts": [
{
"title": "threshold.name",
"value": "critical",
}
],
},
{
"type": "TextBlock",
"text": "Annotations",
"weight": "Bolder",
"size": "Medium",
},
{
"type": "FactSet",
"text": "",
"facts": [
{
"title": "threshold.value",
"value": "10",
},
{
"title": "compare_op",
"value": "above",
},
{
"title": "match_type",
"value": "at_least_once",
},
{"title": "value", "value": "15"},
{
"title": "description",
"value": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
},
],
},
],
"msteams": {"width": "full"},
"actions": [
{
"type": "Action.OpenUrl",
"title": "View Alert",
}
],
},
}
],
},
},
),
],
),
),
types.AlertManagerNotificationTestCase(
name="pagerduty_notifier_default_templating",
rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
),
],
channel_config=pagerduty_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
wait_time_seconds=60,
notification_validations=[
types.NotificationValidation(
destination_type="webhook",
validation_data={
"path": "/v2/enqueue",
"json_body": {
"routing_key": "PagerDutyRoutingKey",
"event_action": "trigger",
"payload": {
"summary": '[FIRING:1] threshold_above_at_least_once for (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
"source": "SigNoz Alert Manager",
"severity": "critical",
"custom_details": {
"firing": {
"Annotations": [
"compare_op = above",
{"description = This alert is fired when the defined metric (current value": "15) crosses the threshold (10)"},
"match_type = at_least_once",
"threshold.value = 10",
"value = 15",
],
"Labels": [
"alertname = threshold_above_at_least_once",
"severity = critical",
"threshold.name = critical",
],
}
},
},
"client": "SigNoz Alert Manager",
"client_url": "https://enter-signoz-host-n-port-here/alerts",
},
},
),
],
),
),
types.AlertManagerNotificationTestCase(
name="opsgenie_notifier_default_templating",
rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
),
],
channel_config=opsgenie_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
wait_time_seconds=60,
notification_validations=[
types.NotificationValidation(
destination_type="webhook",
validation_data={
"path": "/v2/alerts",
"json_body": {
"message": "threshold_above_at_least_once",
"description": "Alerts Firing:\r\n\t\r\n\t - Message: This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\tLabels:\r\n\t - alertname = threshold_above_at_least_once\r\n\t - severity = critical\r\n\t - threshold.name = critical\r\n\t Annotations:\r\n\t - compare_op = above\r\n\t - description = This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\t - match_type = at_least_once\r\n\t - summary = This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\r\n\t - threshold.value = 10\r\n\t - value = 15\r\n\t Source: \r\n\t\r\n",
"details": {
"alertname": "threshold_above_at_least_once",
"severity": "critical",
"threshold.name": "critical",
},
"priority": "P1",
},
},
),
],
),
),
types.AlertManagerNotificationTestCase(
name="webhook_notifier_default_templating",
rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
),
],
channel_config=webhook_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
wait_time_seconds=60,
notification_validations=[
types.NotificationValidation(
destination_type="webhook",
validation_data={
"path": "/webhook/webhook_url",
"json_body": {
"status": "firing",
"alerts": [
{
"status": "firing",
"labels": {
"alertname": "threshold_above_at_least_once",
"severity": "critical",
"threshold.name": "critical",
},
"annotations": {
"compare_op": "above",
"description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
"match_type": "at_least_once",
"summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
"threshold.value": "10",
"value": "15",
},
}
],
"commonLabels": {
"alertname": "threshold_above_at_least_once",
"severity": "critical",
"threshold.name": "critical",
},
"commonAnnotations": {
"compare_op": "above",
"description": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
"match_type": "at_least_once",
"summary": "This alert is fired when the defined metric (current value: 15) crosses the threshold (10)",
"threshold.value": "10",
"value": "15",
},
},
},
),
],
),
),
types.AlertManagerNotificationTestCase(
name="email_notifier_default_templating",
rule_path="alerts/test_scenarios/threshold_above_at_least_once/rule.json",
alert_data=[
types.AlertData(
type="metrics",
data_path="alerts/test_scenarios/threshold_above_at_least_once/alert_data.jsonl",
),
],
channel_config=email_default_config,
notification_expectation=types.AMNotificationExpectation(
should_notify=True,
wait_time_seconds=60,
notification_validations=[
types.NotificationValidation(
destination_type="email",
validation_data={
"subject": '[FIRING:1] threshold_above_at_least_once for (alertname="threshold_above_at_least_once", severity="critical", threshold.name="critical")',
"html": "<html><body>*Alert:* threshold_above_at_least_once - critical\n\n *Summary:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\n *Description:* This alert is fired when the defined metric (current value: 15) crosses the threshold (10)\n *RelatedLogs:* \n *RelatedTraces:* \n\n *Details:*\n \u2022 *alertname:* threshold_above_at_least_once\n \u2022 *severity:* critical\n \u2022 *threshold.name:* critical\n </body></html>",
},
),
],
),
),
]
logger = setup_logger(__name__)
@pytest.mark.parametrize(
"notifier_test_case",
NOTIFIERS_TEST,
ids=lambda notifier_test_case: notifier_test_case.name,
)
def test_notifier_templating(
# wiremock container for webhook notifications
notification_channel: types.TestContainerDocker,
# function to create wiremock mocks
make_http_mocks: Callable[[types.TestContainerDocker, list[Mapping]], None],
create_notification_channel: Callable[[dict], str],
# function to create alert rule
create_alert_rule: Callable[[dict], str],
# Alert data insertion related fixture
insert_alert_data: Callable[[list[types.AlertData], datetime], None],
# Mail dev container for email verification
maildev: types.TestContainerDocker,
# test case from parametrize
notifier_test_case: types.AlertManagerNotificationTestCase,
):
# generate unique channel name
channel_name = str(uuid.uuid4())
# update channel config: set name and rewrite URLs to wiremock
channel_config = update_raw_channel_config(notifier_test_case.channel_config, channel_name, notification_channel)
logger.info("Channel config: %s", {"channel_config": channel_config})
# setup wiremock mocks for webhook-based notification validations
webhook_validations = [v for v in notifier_test_case.notification_expectation.notification_validations if v.destination_type == "webhook"]
if len(webhook_validations) > 0:
mock_mappings = [
Mapping(
request=MappingRequest(method=HttpMethods.POST, url=v.validation_data["path"]),
response=MappingResponse(status=200, json_body={}),
persistent=False,
)
for v in webhook_validations
]
make_http_mocks(notification_channel, mock_mappings)
logger.info("Mock mappings created")
# clear mails if any destination is email
if any(v.destination_type == "email" for v in notifier_test_case.notification_expectation.notification_validations):
delete_all_mails(maildev)
logger.info("Mails deleted")
# create notification channel
create_notification_channel(channel_config)
logger.info("Channel created with name: %s", {"channel_name": channel_name})
# insert alert data
insert_alert_data(
notifier_test_case.alert_data,
base_time=datetime.now(tz=UTC) - timedelta(minutes=5),
)
# create alert rule
rule_path = get_testdata_file_path(notifier_test_case.rule_path)
with open(rule_path, encoding="utf-8") as f:
rule_data = json.loads(f.read())
update_rule_channel_name(rule_data, channel_name)
rule_id = create_alert_rule(rule_data)
logger.info("rule created: %s", {"rule_id": rule_id, "rule_name": rule_data["alert"]})
# verify notification expectations
verify_notification_expectation(
notification_channel,
maildev,
notifier_test_case.notification_expectation,
)

File diff suppressed because one or more lines are too long

View File

@@ -1,42 +0,0 @@
import pytest
from testcontainers.core.container import Network
from fixtures import types
from fixtures.signoz import create_signoz
@pytest.fixture(name="signoz", scope="package")
def signoz( # pylint: disable=too-many-arguments,too-many-positional-arguments
network: Network,
zeus: types.TestContainerDocker,
gateway: types.TestContainerDocker,
sqlstore: types.TestContainerSQL,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
maildev: types.TestContainerDocker,
notification_channel: types.TestContainerDocker,
) -> types.SigNoz:
"""
Package-scoped fixture for setting up SigNoz.
Overrides SMTP, PagerDuty, and OpsGenie URLs to point to test containers.
"""
return create_signoz(
network=network,
zeus=zeus,
gateway=gateway,
sqlstore=sqlstore,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
env_overrides={
# SMTP config for email notifications via maildev
"SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__SMARTHOST": f"{maildev.container_configs['1025'].address}:{maildev.container_configs['1025'].port}",
"SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__REQUIRE__TLS": "false",
"SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_SMTP__FROM": "alertmanager@signoz.io",
# PagerDuty API URL -> wiremock (default: https://events.pagerduty.com/v2/enqueue)
"SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_PAGERDUTY__URL": notification_channel.container_configs["8080"].get("/v2/enqueue"),
# OpsGenie API URL -> wiremock (default: https://api.opsgenie.com/)
"SIGNOZ_ALERTMANAGER_SIGNOZ_GLOBAL_OPSGENIE__API__URL": notification_channel.container_configs["8080"].get("/"),
},
)

View File

@@ -75,7 +75,3 @@ ignore = [
[tool.ruff.format]
# Defaults align with black (double quotes, 4-space indent).
[tool.ruff.lint.per-file-ignores]
"integration/src/alertmanager/*" = ["E501"]
"fixtures/notification_channel.py" = ["E501"]