chore: update @signozhq packages and adjust styles in AuthHeader and AnnouncementTooltip components (#10989)

* chore: custom notifiers in alert manager

* chore: lint fixs

* chore: fix email linter

* chore: added tracing to msteamsv2 notifier

* feat: alert manager template to template title and notification body

* chore: updated test name + code for timeout errors

* chore: added utils for using variables with $ notation

* chore: exposed templates for alertmanager types

* feat: added preprocessor for alert templater

* chore: hooked preProcess function in expandTitle and body, added labels and annotations in alertdata

* chore: fix lint issues

* chore: added handling for missing variable used in template

* feat: converted alerttemplater to interface and updated tests

* refactor: added extractCommonKV instead of 2 different functions

* test: fix preprocessor test case

* feat: added support for  and  in templating

* chore: lint fix

* chore: renamed the interface

* chore: added test for missing function

* refactor: test case and sb related changed

* refactor: comments and test improvements

* chore: lint fix

* chore: updated comments

* chore: updated newline to markdown format

* chore: updated br with new line in test and logs added

* refactor: review comments

* refactor: lint fixes

* chore: updated licenses for notifiers

* chore: updated email notifier from upstream

* feat: return single templating result from  with flag for template type

* fix: variables with symbols in template

* chore: removed notifier test files

* refactor: changes as per internal review

* chore: lint issue

---------

Co-authored-by: Srikanth Chekuri <srikanth.chekuri92@gmail.com>

ee/query-service/app/server.go

@@ -10,6 +10,7 @@ import (
 	"go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux"
 	"go.opentelemetry.io/otel/propagation"
 
+	"github.com/SigNoz/signoz/pkg/cache/memorycache"
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/queryparser"
@@ -73,12 +74,25 @@ type Server struct {
 
 // NewServer creates and initializes Server
 func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
+	cacheForTraceDetail, err := memorycache.New(context.TODO(), signoz.Instrumentation.ToProviderSettings(), cache.Config{
+		Provider: "memory",
+		Memory: cache.Memory{
+			NumCounters: 10 * 10000,
+			MaxCost:     1 << 27, // 128 MB
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	reader := clickhouseReader.NewReader(
 		signoz.Instrumentation.Logger(),
 		signoz.SQLStore,
 		signoz.TelemetryStore,
 		signoz.Prometheus,
 		signoz.TelemetryStore.Cluster(),
+		config.Querier.FluxInterval,
+		cacheForTraceDetail,
 		signoz.Cache,
 		nil,
 	)

frontend/package.json

@@ -48,22 +48,22 @@
     "@radix-ui/react-tooltip": "1.0.7",
     "@sentry/react": "8.41.0",
     "@sentry/vite-plugin": "2.22.6",
-    "@signozhq/button": "0.0.2",
-    "@signozhq/calendar": "0.0.0",
-    "@signozhq/callout": "0.0.2",
-    "@signozhq/checkbox": "0.0.2",
-    "@signozhq/combobox": "0.0.2",
-    "@signozhq/command": "0.0.0",
+    "@signozhq/button": "0.0.5",
+    "@signozhq/calendar": "0.1.1",
+    "@signozhq/callout": "0.0.4",
+    "@signozhq/checkbox": "0.0.4",
+    "@signozhq/combobox": "0.0.4",
+    "@signozhq/command": "0.0.2",
     "@signozhq/design-tokens": "2.1.4",
-    "@signozhq/dialog": "^0.0.2",
-    "@signozhq/drawer": "0.0.4",
+    "@signozhq/dialog": "0.0.4",
+    "@signozhq/drawer": "0.0.6",
     "@signozhq/icons": "0.1.0",
-    "@signozhq/input": "0.0.2",
-    "@signozhq/popover": "0.0.0",
-    "@signozhq/radio-group": "0.0.2",
-    "@signozhq/resizable": "0.0.0",
+    "@signozhq/input": "0.0.4",
+    "@signozhq/popover": "0.1.2",
+    "@signozhq/radio-group": "0.0.4",
+    "@signozhq/resizable": "0.0.2",
     "@signozhq/table": "0.3.7",
-    "@signozhq/toggle-group": "0.0.1",
+    "@signozhq/toggle-group": "0.0.3",
     "@signozhq/ui": "0.0.5",
     "@tanstack/react-table": "8.21.3",
     "@tanstack/react-virtual": "3.13.22",

frontend/src/components/AuthPageContainer/AuthHeader.styles.scss

@@ -42,6 +42,7 @@
 	height: 32px;
 	padding: 10px 16px;
 	background: var(--l2-background);
+	color: var(--l2-foreground);
 	border: none;
 	border-radius: 2px;
 	cursor: pointer;
@@ -65,10 +66,3 @@
 		opacity: 0.8;
 	}
 }
-
-.lightMode {
-	.auth-header-help-button {
-		background: var(--l2-background);
-		border: 1px solid var(--l1-border);
-	}
-}

frontend/src/components/QuickFilters/QuickFiltersSettings/AnnouncementTooltip/AnnouncementTooltip.styles.scss

@@ -46,8 +46,8 @@
 	}
 
 	&__button {
-		background: var(--card);
-		color: var(--accent-primary);
+		background: var(--secondary-background);
+		color: var(--secondary-foreground);
 		border: none;
 		padding: 6px 12px;
 		border-radius: 4px;

frontend/yarn.lock

@@ -5503,20 +5503,21 @@
   resolved "https://registry.yarnpkg.com/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz#a90ab31d0cc1dfb54c66a69e515bf624fa7b2224"
   integrity sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==
 
-"@signozhq/button@0.0.1":
-  version "0.0.1"
-  resolved "https://registry.yarnpkg.com/@signozhq/button/-/button-0.0.1.tgz#7d3204454b0361bd3fdf91fa6604af01a481a9db"
-  integrity sha512-k5WFpckNXzwcTS82jU+65M3V1KdriopBObB1ls7W2OU0RKof6Gf+/9uqDXnuu+Y4Cxn2cPo8+6MfiQbS02LHeg==
+"@signozhq/button@0.0.5":
+  version "0.0.5"
+  resolved "https://registry.yarnpkg.com/@signozhq/button/-/button-0.0.5.tgz#e8220a6e9ed78552694f41700c277956f26232e1"
+  integrity sha512-fgobypuXv2kWGDkqXZoEjcySPHELzI/X515cdcR1hx4N9rizzOglLtYEjGTLR13iQrzLwSsNX8xxsv0/iSCjQg==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.1.0"
+    "@signozhq/icons" "^0.1.0"
     class-variance-authority "^0.7.0"
     clsx "^2.1.1"
     lucide-react "^0.445.0"
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/button@0.0.2", "@signozhq/button@^0.0.2":
+"@signozhq/button@^0.0.2":
   version "0.0.2"
   resolved "https://registry.yarnpkg.com/@signozhq/button/-/button-0.0.2.tgz#c13edef1e735134b784a41f874b60a14bc16993f"
   integrity sha512-434/gbTykC00LrnzFPp7c33QPWZkf9n+8+SToLZFTB0rzcaS/xoB4b7QKhvk+8xLCj4zpw6BxfeRAL+gSoOUJw==
@@ -5529,14 +5530,28 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/calendar@0.0.0":
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/@signozhq/calendar/-/calendar-0.0.0.tgz#93b2cec2586efee814df934f88a2193cec95bae9"
-  integrity sha512-lm7tzPEhaHNjrksvi2GPGH4suEe6x2DQJ2dpku+JmKyLGB5rg9saSAosvrZVKhXLoZuSSjlBSkz+oHYEKIdHfA==
+"@signozhq/button@workspace:*":
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/@signozhq/button/-/button-0.0.3.tgz#ce6c722b24859198f8c18a708b9d09249b184e3e"
+  integrity sha512-b0JGoP0AIoYep/ApQUOn9LiK8dUATfqikz7jAKe20dPc8SjwUtsLPOao9j6I+2W4qd0CZDffJADBVpF2SAIsPQ==
+  dependencies:
+    "@radix-ui/react-icons" "^1.3.0"
+    "@radix-ui/react-slot" "^1.1.0"
+    "@signozhq/icons" "^0.1.0"
+    class-variance-authority "^0.7.0"
+    clsx "^2.1.1"
+    lucide-react "^0.445.0"
+    tailwind-merge "^2.5.2"
+    tailwindcss-animate "^1.0.7"
+
+"@signozhq/calendar@0.1.1":
+  version "0.1.1"
+  resolved "https://registry.yarnpkg.com/@signozhq/calendar/-/calendar-0.1.1.tgz#8fb6432c4397ad2e8204b0ce9fb1b7aaa231e4cc"
+  integrity sha512-Mw5cVtqSI1F57YwG+ufuJKJs/KrkrTSeP6aVPcTvO3tHJ5H4aZ1oaeZtGzTEDMqUvusDlRdZPSHhklJiUg/ixQ==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.2.3"
-    "@signozhq/button" "0.0.1"
+    "@signozhq/button" "workspace:*"
     class-variance-authority "^0.7.0"
     clsx "^2.1.1"
     date-fns "^4.1.0"
@@ -5545,13 +5560,14 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/callout@0.0.2":
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/@signozhq/callout/-/callout-0.0.2.tgz#131ca15f89a8ee6729fecc4d322f11359c02e5cf"
-  integrity sha512-tmguHm+/JVRKjMElJOFyG7LJcdqCW1hHnFfp8ZkjQ+Gi7MfFt/r2foLZG2DNdOcfxSvhf2zhzr7D+epgvmbQ1A==
+"@signozhq/callout@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@signozhq/callout/-/callout-0.0.4.tgz#8d0c224cc4f64930a04bd0d075597358ae7ec1d8"
+  integrity sha512-g1NXkzAkuMdmH+z58t9CSL4+MZdWB5zPLyOHKeJYnQk1JXSYpEGK8CzSz4ZtVb4OrMS7mDkWxnqMK0EHxwVOWA==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.1.0"
+    "@signozhq/icons" "^0.1.0"
     class-variance-authority "^0.7.0"
     clsx "^2.1.1"
     lucide-react "^0.445.0"
@@ -5559,10 +5575,10 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/checkbox@0.0.2":
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/@signozhq/checkbox/-/checkbox-0.0.2.tgz#d11fb5eff3927c540937e3bd24351bfc1fdef9ec"
-  integrity sha512-odQdh839GaTy1kqC8yavUKrOYP5tiIppUIV7xGNyxs/KnLGDWLw3ZSdACRV1Z55CLddjQ6OWKiwyVV7t+sxEuw==
+"@signozhq/checkbox@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@signozhq/checkbox/-/checkbox-0.0.4.tgz#2cf83d7bfd4db4aaec815e5788061e3fe5f86483"
+  integrity sha512-X93EqHEy06pfpGcEJkJpNrxvkZryJaA+M0NQ09oTb1wzFweGMw+fR1Hgjl8lMqQPkHtz3MCP820uLhxvOJhktg==
   dependencies:
     "@radix-ui/react-checkbox" "^1.2.3"
     "@radix-ui/react-icons" "^1.3.0"
@@ -5573,10 +5589,24 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/combobox@0.0.2":
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/@signozhq/combobox/-/combobox-0.0.2.tgz#019cc6d619e4eb6d1061fdfa00d4bd99d6aa727f"
-  integrity sha512-QnGCNJAHd55Wqblw0CLOEOJoLFx8dgP+q/9hXbN5qil72DjRzxBgb5DAkIkon0owCmlagDQknFiOygYnzVJS8g==
+"@signozhq/checkbox@workspace:*":
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/@signozhq/checkbox/-/checkbox-0.0.3.tgz#9ccf8bd3b118405b2407c71db780089de050e651"
+  integrity sha512-x2uqV3GsLmnDz4Zd2oY9/sExSqTY9gw/tw5gVd9ZziSNWOhy9rBKI8xwVpaAYiaZZHH3NReEwASwjx7C4EKKiQ==
+  dependencies:
+    "@radix-ui/react-checkbox" "^1.2.3"
+    "@radix-ui/react-icons" "^1.3.0"
+    "@radix-ui/react-slot" "^1.1.0"
+    class-variance-authority "^0.7.0"
+    clsx "^2.1.1"
+    lucide-react "^0.445.0"
+    tailwind-merge "^2.5.2"
+    tailwindcss-animate "^1.0.7"
+
+"@signozhq/combobox@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@signozhq/combobox/-/combobox-0.0.4.tgz#7195416144ae881874e3744f7b71251fbacc4f3e"
+  integrity sha512-8mXlpkZ6066+JE+9EXmuR47ky+tJLwZ1W/9qXa69x5F1r7zXxuWvNQe/GYiGetpxohV/jO6QQDN1WpCV9hNjiw==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-popover" "^1.1.2"
@@ -5589,10 +5619,10 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/command@0.0.0":
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/@signozhq/command/-/command-0.0.0.tgz#bd1e1cac7346e862dd61df64b756302e89e1a322"
-  integrity sha512-AwRYxZTi4o8SBOL4hmgcgbhCKXl2Qb/TUSLbSYEMFdiQSl5VYA8XZJv5fSYVMJkAIlOaHzFzR04XNEU7lZcBpw==
+"@signozhq/command@0.0.2":
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/@signozhq/command/-/command-0.0.2.tgz#9d72f0d8d0945773461350f8d311072b2b4f96c6"
+  integrity sha512-MFMDtm6qC/aG84k+q9XVCkRR46kNQYVP0qP59Xg34gx5baD76iivu13rZLc27MEhOucHacP2UODRJ4uMGZt/Mw==
   dependencies:
     "@radix-ui/react-dialog" "^1.1.11"
     "@radix-ui/react-icons" "^1.3.0"
@@ -5609,24 +5639,25 @@
   resolved "https://registry.yarnpkg.com/@signozhq/design-tokens/-/design-tokens-2.1.4.tgz#f209da6fbd2ac97ab4434b71472f741009306550"
   integrity sha512-Ny7/VA5YGFFmZx58jMh7ATFyu7VePaJ4ySmj/DopP1hilmfdxQsKWnpqKaZJWRXrbNkc0gmq3cR7q7Z8nnN7ZQ==
 
-"@signozhq/dialog@^0.0.2":
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/@signozhq/dialog/-/dialog-0.0.2.tgz#55bd8e693f76325fda9aabe3197350e0adc163c4"
-  integrity sha512-YT5t3oZpGkAuWptTqhCgVtLjxsRQrEIrQHFoXpP9elM1+O4TS9WHr+07BLQutOVg6u9n9pCvW3OYf0SCETkDVQ==
+"@signozhq/dialog@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@signozhq/dialog/-/dialog-0.0.4.tgz#54f385aa7cc17c6281653437e448f81dee0190ff"
+  integrity sha512-j/RPhx98sCTyfg1VlSxbHfLzG/cVKCdd1JZrVkfzs4WK1ijM7Scpl4MUnFxn0ShCFrJ8trbR6I5NioKUIojK2g==
   dependencies:
     "@radix-ui/react-dialog" "^1.1.11"
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.1.0"
+    "@signozhq/checkbox" "workspace:*"
     class-variance-authority "^0.7.0"
     clsx "^2.1.1"
     lucide-react "^0.445.0"
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/drawer@0.0.4":
-  version "0.0.4"
-  resolved "https://registry.yarnpkg.com/@signozhq/drawer/-/drawer-0.0.4.tgz#7c6e6779602113f55df8a55076e68b9cc13c7d79"
-  integrity sha512-m/shStl5yVPjHjrhDAh3EeKqqTtMmZUBVlgJPUGgoNV3sFsuN6JNaaAtEJI8cQBWkbEEiHLWKVkL/vhbQ7YrUg==
+"@signozhq/drawer@0.0.6":
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/@signozhq/drawer/-/drawer-0.0.6.tgz#ea280d71af6bc665679c7da26e0703a7bf96aa0e"
+  integrity sha512-xoDHdsVZuj4rHK5gTnM4px7E2rv/6Jgqm81uxs1CfheOQsEAnPuiq3Dpdrdsf6XxCeORwnpcGUR5PEJhnAsjmA==
   dependencies:
     "@radix-ui/react-dialog" "^1.1.11"
     "@radix-ui/react-icons" "^1.3.0"
@@ -5647,23 +5678,24 @@
     "@commitlint/cli" "^17.6.7"
     "@commitlint/config-conventional" "^17.6.7"
 
-"@signozhq/input@0.0.2":
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/@signozhq/input/-/input-0.0.2.tgz#b2fea8c0979a53984ebcd5e3c3c50b38082eb1b1"
-  integrity sha512-Iti9GkvexSsULX1pQsN6FT6Gw96YWilts72wITZd5fzgZq1yKqaDtQl98/QNuyoS3I3WEh+hVF4EIeCCe7oRsQ==
+"@signozhq/input@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@signozhq/input/-/input-0.0.4.tgz#f07dddb26ac4dda1cc8e07bfe605e8b34299955e"
+  integrity sha512-S6tIcrkRZAsmwA80eAJCwZlgHLuioUHMTeszx4PRf/DUqVQ8cjIjmTfHDwzO0zIbMcVHpxqxVNpYArIqUJtgZQ==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.1.0"
+    "@signozhq/button" "workspace:*"
     class-variance-authority "^0.7.0"
     clsx "^2.1.1"
     lucide-react "^0.445.0"
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/popover@0.0.0":
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/@signozhq/popover/-/popover-0.0.0.tgz#675baf1c18ca0180369b4df0700c24e2c55ad758"
-  integrity sha512-XW0MhzxWzZNQWjVeb+BFjiOIbBbYCT+9MCUOIW8kiL0axFaaimnk0QPi1rk09u136MMGByI6fYuCJ5Qa07l1dA==
+"@signozhq/popover@0.1.2":
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/@signozhq/popover/-/popover-0.1.2.tgz#73b418be584e8a671ff4189ed47a540ed73bbde6"
+  integrity sha512-6TVMVjWuO7XcKfFMrndcmDdg4JsGvRLe0SV43CRPNV6OkL5uFwUHFJZK2BU3v8S9xMoOf3LsdpfxPuCeK8QKCA==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-popover" "^1.1.15"
@@ -5675,10 +5707,10 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/radio-group@0.0.2":
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/@signozhq/radio-group/-/radio-group-0.0.2.tgz#4b13567bfee2645226f2cf41f261bbb288e1be4b"
-  integrity sha512-ahykmA5hPujOC964CFveMlQ12tWSyut2CUiFRqT1QxRkOLS2R44Qn2hh2psqJJ18JMX/24ZYCAIh9Bdd5XW+7g==
+"@signozhq/radio-group@0.0.4":
+  version "0.0.4"
+  resolved "https://registry.yarnpkg.com/@signozhq/radio-group/-/radio-group-0.0.4.tgz#2fbd28bf48bb761063d642d3a169c3e0c098639d"
+  integrity sha512-1t1idP+TsYV52GoQM/5KaxUQtA9/CAxAcVOvT6sBXAbyR12azUaR7GL4S0VxRRfOlA+pZ9bT9TqlfQT7LWUTDg==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-radio-group" "^1.3.4"
@@ -5689,10 +5721,10 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/resizable@0.0.0":
-  version "0.0.0"
-  resolved "https://registry.yarnpkg.com/@signozhq/resizable/-/resizable-0.0.0.tgz#a517818b9f9bcdaeafc55ae134be86522bc90e9f"
-  integrity sha512-yAkJdMgTkh8kv42ZuabwTZguxalwYqIp4b44YdSrw6jRUSq9tscUBXVllNN79T71lPUtc5AV13uQ4Qm5AcfVbQ==
+"@signozhq/resizable@0.0.2":
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/@signozhq/resizable/-/resizable-0.0.2.tgz#76b9212c5f1b1e982013bec1b618e967a752e705"
+  integrity sha512-xMdCDHOUDssPknFYRmwNTlQIy583wMpJSx6aHmjeYEcVfhDfDBrW+KEko/ODFoM8dOB0CpMtDjlLU14BchsNnA==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.1.0"
@@ -5721,10 +5753,10 @@
     tailwind-merge "^2.5.2"
     tailwindcss-animate "^1.0.7"
 
-"@signozhq/toggle-group@0.0.1":
-  version "0.0.1"
-  resolved "https://registry.yarnpkg.com/@signozhq/toggle-group/-/toggle-group-0.0.1.tgz#c82ff1da34e77b24da53c2d595ad6b4a0d1b1de4"
-  integrity sha512-871bQayL5MaqsuNOFHKexidu9W2Hlg1y4xmH8C5mGmlfZ4bd0ovJ9OweQrM6Puys3jeMwi69xmJuesYCfKQc1g==
+"@signozhq/toggle-group@0.0.3":
+  version "0.0.3"
+  resolved "https://registry.yarnpkg.com/@signozhq/toggle-group/-/toggle-group-0.0.3.tgz#6a38312b46198c555ca4c287d4818135a4a9dabe"
+  integrity sha512-qWoxC9jxW/otq1AdD/9ATGEJi4KyTfQSY8BSXlqu+4z4iEVjW4eeWSiyaoHcnB9UaypHDMFDMR+gPkyavtMY2Q==
   dependencies:
     "@radix-ui/react-icons" "^1.3.0"
     "@radix-ui/react-slot" "^1.1.0"

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate.go

@@ -0,0 +1,360 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"sort"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+)
+
+// Templater expands user-authored title and body templates against a group
+// of alerts and returns channel-ready strings along with the aggregate data
+// a caller might reuse (e.g. to render an email layout around the body).
+type Templater interface {
+	Expand(ctx context.Context, req alertmanagertypes.ExpandRequest, alerts []*types.Alert) (*alertmanagertypes.ExpandResult, error)
+}
+
+type templater struct {
+	tmpl   *template.Template
+	logger *slog.Logger
+}
+
+// New returns a Templater bound to the given Prometheus alertmanager
+// template and logger.
+func New(tmpl *template.Template, logger *slog.Logger) Templater {
+	return &templater{tmpl: tmpl, logger: logger}
+}
+
+func (at *templater) Expand(
+	ctx context.Context,
+	req alertmanagertypes.ExpandRequest,
+	alerts []*types.Alert,
+) (*alertmanagertypes.ExpandResult, error) {
+	ntd := at.buildNotificationTemplateData(ctx, alerts)
+	missingVars := make(map[string]bool)
+
+	title, titleMissingVars, err := at.expandTitle(req.TitleTemplate, ntd)
+	if err != nil {
+		return nil, err
+	}
+	// if title template results in empty string, use default template
+	// this happens for rules where custom title annotation was not set
+	if title == "" && req.DefaultTitleTemplate != "" {
+		title, err = at.expandDefaultTemplate(ctx, req.DefaultTitleTemplate, alerts)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		mergeMissingVars(missingVars, titleMissingVars)
+	}
+
+	isDefaultBody := false
+	body, bodyMissingVars, err := at.expandBody(req.BodyTemplate, ntd)
+	if err != nil {
+		return nil, err
+	}
+	// if body template results in nil, use default template
+	// this happens for rules where custom body annotation was not set
+	if body == nil {
+		isDefaultBody = true
+		defaultBody, err := at.expandDefaultTemplate(ctx, req.DefaultBodyTemplate, alerts)
+		if err != nil {
+			return nil, err
+		}
+		body = []string{defaultBody} // default template combines all alerts message into a single body
+	} else {
+		mergeMissingVars(missingVars, bodyMissingVars)
+	}
+
+	// convert the internal map to a sorted slice for returning missing variables
+	missingVarsList := make([]string, 0, len(missingVars))
+	for k := range missingVars {
+		missingVarsList = append(missingVarsList, k)
+	}
+	sort.Strings(missingVarsList)
+
+	return &alertmanagertypes.ExpandResult{
+		Title:            title,
+		Body:             body,
+		MissingVars:      missingVarsList,
+		IsDefaultBody:    isDefaultBody,
+		NotificationData: ntd,
+	}, nil
+}
+
+// expandDefaultTemplate uses go-template to expand the default template.
+func (at *templater) expandDefaultTemplate(
+	ctx context.Context,
+	tmplStr string,
+	alerts []*types.Alert,
+) (string, error) {
+	// if even the default template is empty, return empty string
+	// this is possible if user added channel with blank template
+	if tmplStr == "" {
+		at.logger.WarnContext(ctx, "default template is empty")
+		return "", nil
+	}
+	data := notify.GetTemplateData(ctx, at.tmpl, alerts, at.logger)
+	result, err := at.tmpl.ExecuteTextString(tmplStr, data)
+	if err != nil {
+		return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute default template: %s", err.Error())
+	}
+	return result, nil
+}
+
+// mergeMissingVars adds all keys from src into dst.
+func mergeMissingVars(dst, src map[string]bool) {
+	for k := range src {
+		dst[k] = true
+	}
+}
+
+// expandTitle expands the title template. Returns empty string if the template is empty.
+func (at *templater) expandTitle(
+	titleTemplate string,
+	ntd *alertmanagertypes.NotificationTemplateData,
+) (string, map[string]bool, error) {
+	if titleTemplate == "" {
+		return "", nil, nil
+	}
+	processRes, err := preProcessTemplateAndData(titleTemplate, ntd)
+	if err != nil {
+		return "", nil, err
+	}
+	result, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+	if err != nil {
+		return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute custom title template: %s", err.Error())
+	}
+	return strings.TrimSpace(result), processRes.UnknownVars, nil
+}
+
+// expandBody expands the body template for each individual alert. Returns nil if the template is empty.
+func (at *templater) expandBody(
+	bodyTemplate string,
+	ntd *alertmanagertypes.NotificationTemplateData,
+) ([]string, map[string]bool, error) {
+	if bodyTemplate == "" {
+		return nil, nil, nil
+	}
+	var sb []string
+	missingVars := make(map[string]bool)
+	for i := range ntd.Alerts {
+		processRes, err := preProcessTemplateAndData(bodyTemplate, &ntd.Alerts[i])
+		if err != nil {
+			return nil, nil, err
+		}
+		part, err := at.tmpl.ExecuteTextString(processRes.Template, processRes.Data)
+		if err != nil {
+			return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "failed to execute custom body template: %s", err.Error())
+		}
+		// add unknown variables and templated text to the result
+		for k := range processRes.UnknownVars {
+			missingVars[k] = true
+		}
+		if strings.TrimSpace(part) != "" {
+			sb = append(sb, strings.TrimSpace(part))
+		}
+	}
+	return sb, missingVars, nil
+}
+
+// buildNotificationTemplateData creates the NotificationTemplateData using
+// info from context and the raw alerts.
+func (at *templater) buildNotificationTemplateData(
+	ctx context.Context,
+	alerts []*types.Alert,
+) *alertmanagertypes.NotificationTemplateData {
+	// extract the required data from the context
+	receiver, ok := notify.ReceiverName(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing receiver name in context")
+	}
+
+	groupLabels, ok := notify.GroupLabels(ctx)
+	if !ok {
+		at.logger.WarnContext(ctx, "missing group labels in context")
+	}
+
+	// extract the external URL from the template
+	externalURL := ""
+	if at.tmpl.ExternalURL != nil {
+		externalURL = at.tmpl.ExternalURL.String()
+	}
+
+	commonAnnotations := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+	commonLabels := extractCommonKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+
+	// aggregate labels and annotations from all alerts
+	labels := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Labels })
+	annotations := aggregateKV(alerts, func(a *types.Alert) model.LabelSet { return a.Annotations })
+
+	// Strip private annotations from surfaces visible to templates or
+	// notifications; the structured fields on AlertInfo/RuleInfo already hold
+	// anything a template needs from them.
+	commonAnnotations = alertmanagertypes.FilterPublicAnnotations(commonAnnotations)
+	annotations = alertmanagertypes.FilterPublicAnnotations(annotations)
+
+	// build the alert data slice
+	alertDataSlice := make([]alertmanagertypes.AlertData, 0, len(alerts))
+	for _, a := range alerts {
+		ad := buildAlertData(a, receiver)
+		alertDataSlice = append(alertDataSlice, ad)
+	}
+
+	// count the number of firing and resolved alerts
+	var firing, resolved int
+	for _, ad := range alertDataSlice {
+		if ad.Alert.IsFiring {
+			firing++
+		} else if ad.Alert.IsResolved {
+			resolved++
+		}
+	}
+
+	// build the group labels
+	gl := make(template.KV, len(groupLabels))
+	for k, v := range groupLabels {
+		gl[string(k)] = string(v)
+	}
+
+	// build the notification template data
+	return &alertmanagertypes.NotificationTemplateData{
+		Alert: alertmanagertypes.NotificationAlert{
+			Receiver:      receiver,
+			Status:        string(types.Alerts(alerts...).Status()),
+			TotalFiring:   firing,
+			TotalResolved: resolved,
+		},
+		Rule:              buildRuleInfo(commonLabels, commonAnnotations),
+		GroupLabels:       gl,
+		CommonLabels:      commonLabels,
+		CommonAnnotations: commonAnnotations,
+		ExternalURL:       externalURL,
+		Labels:            labels,
+		Annotations:       annotations,
+		Alerts:            alertDataSlice,
+	}
+}
+
+// buildAlertData converts a single *types.Alert into an AlertData.
+func buildAlertData(a *types.Alert, receiver string) alertmanagertypes.AlertData {
+	labels := make(template.KV, len(a.Labels))
+	for k, v := range a.Labels {
+		labels[string(k)] = string(v)
+	}
+
+	annotations := make(template.KV, len(a.Annotations))
+	for k, v := range a.Annotations {
+		annotations[string(k)] = string(v)
+	}
+
+	return alertmanagertypes.AlertData{
+		Alert: alertmanagertypes.AlertInfo{
+			Status:        string(a.Status()),
+			Receiver:      receiver,
+			Value:         annotations[ruletypes.AnnotationValue],
+			StartsAt:      a.StartsAt,
+			EndsAt:        a.EndsAt,
+			GeneratorURL:  a.GeneratorURL,
+			Fingerprint:   a.Fingerprint().String(),
+			IsFiring:      a.Status() == model.AlertFiring,
+			IsResolved:    a.Status() == model.AlertResolved,
+			IsMissingData: labels[ruletypes.LabelNoData] == "true",
+			IsRecovering:  labels[ruletypes.LabelIsRecovering] == "true",
+		},
+		Rule:   buildRuleInfo(labels, annotations),
+		Log:    alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedLogs]},
+		Trace:  alertmanagertypes.LinkInfo{URL: annotations[ruletypes.AnnotationRelatedTraces]},
+		Labels: labels,
+		// Strip private annotations once the structured fields above have
+		// been populated from the raw map.
+		Annotations: alertmanagertypes.FilterPublicAnnotations(annotations),
+	}
+}
+
+// buildRuleInfo extracts the rule metadata from the well-known labels and
+// annotations that the rule manager attaches to every emitted alert.
+func buildRuleInfo(labels, annotations template.KV) alertmanagertypes.RuleInfo {
+	return alertmanagertypes.RuleInfo{
+		Name:      labels[ruletypes.LabelAlertName],
+		ID:        labels[ruletypes.LabelRuleID],
+		URL:       labels[ruletypes.LabelRuleSource],
+		Severity:  labels[ruletypes.LabelSeverityName],
+		MatchType: annotations[ruletypes.AnnotationMatchType],
+		Threshold: alertmanagertypes.Threshold{
+			Value: annotations[ruletypes.AnnotationThresholdValue],
+			Op:    annotations[ruletypes.AnnotationCompareOp],
+		},
+	}
+}
+
+// maxAggregatedValues caps the number of distinct label/annotation values
+// joined together when summarising across alerts. Beyond this, extras are
+// dropped rather than concatenated.
+const maxAggregatedValues = 5
+
+// aggregateKV merges label or annotation sets from a group of alerts into a
+// single KV. Per key, up to maxAggregatedValues distinct values are kept (in
+// order of first appearance) and joined with ", ". A lossy summary used for
+// grouped-notification display, not a true union.
+func aggregateKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	valuesPerKey := make(map[string][]string)
+	seenValues := make(map[string]map[string]bool)
+
+	for _, alert := range alerts {
+		for k, v := range extractFn(alert) {
+			key := string(k)
+			value := string(v)
+
+			if seenValues[key] == nil {
+				seenValues[key] = make(map[string]bool)
+			}
+			if !seenValues[key][value] && len(valuesPerKey[key]) < maxAggregatedValues {
+				seenValues[key][value] = true
+				valuesPerKey[key] = append(valuesPerKey[key], value)
+			}
+		}
+	}
+
+	result := make(template.KV, len(valuesPerKey))
+	for key, values := range valuesPerKey {
+		result[key] = strings.Join(values, ", ")
+	}
+	return result
+}
+
+// extractCommonKV returns the intersection of label or annotation pairs
+// across all alerts. A pair is included only if every alert carries the same
+// key with the same value.
+func extractCommonKV(alerts []*types.Alert, extractFn func(*types.Alert) model.LabelSet) template.KV {
+	if len(alerts) == 0 {
+		return template.KV{}
+	}
+
+	common := make(template.KV, len(extractFn(alerts[0])))
+	for k, v := range extractFn(alerts[0]) {
+		common[string(k)] = string(v)
+	}
+
+	for _, a := range alerts[1:] {
+		kv := extractFn(a)
+		for k := range common {
+			if string(kv[model.LabelName(k)]) != common[k] {
+				delete(common, k)
+			}
+		}
+		if len(common) == 0 {
+			break
+		}
+	}
+
+	return common
+}

pkg/alertmanager/alertmanagertemplate/alertmanagertemplate_test.go

@@ -0,0 +1,287 @@
+package alertmanagertemplate
+
+import (
+	"context"
+	"log/slog"
+	"sort"
+	"testing"
+	"time"
+
+	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// testSetup returns an AlertTemplater and a context pre-populated with group key,
+// receiver name, and group labels for use in tests.
+func testSetup(t *testing.T) (Templater, context.Context) {
+	t.Helper()
+	tmpl := test.CreateTmpl(t)
+	ctx := context.Background()
+	ctx = notify.WithGroupKey(ctx, "test-group")
+	ctx = notify.WithReceiverName(ctx, "slack")
+	ctx = notify.WithGroupLabels(ctx, model.LabelSet{"alertname": "TestAlert", "severity": "critical"})
+	logger := slog.New(slog.DiscardHandler)
+	return New(tmpl, logger), ctx
+}
+
+func createAlert(labels, annotations map[string]string, isFiring bool) *types.Alert {
+	ls := model.LabelSet{}
+	for k, v := range labels {
+		ls[model.LabelName(k)] = model.LabelValue(v)
+	}
+	ann := model.LabelSet{}
+	for k, v := range annotations {
+		ann[model.LabelName(k)] = model.LabelValue(v)
+	}
+	startsAt := time.Now()
+	var endsAt time.Time
+	if isFiring {
+		endsAt = startsAt.Add(time.Hour)
+	} else {
+		startsAt = startsAt.Add(-2 * time.Hour)
+		endsAt = startsAt.Add(-time.Hour)
+	}
+	return &types.Alert{Alert: model.Alert{Labels: ls, Annotations: ann, StartsAt: startsAt, EndsAt: endsAt}}
+}
+
+func TestExpandTemplates(t *testing.T) {
+	at, ctx := testSetup(t)
+
+	tests := []struct {
+		name              string
+		alerts            []*types.Alert
+		input             alertmanagertypes.ExpandRequest
+		wantTitle         string
+		wantBody          []string
+		wantMissingVars   []string
+		errorContains     string
+		wantIsDefaultBody bool
+	}{
+		{
+			// High request throughput on a service — service is a custom label.
+			// $labels.service extracts the label value; $annotations.description pulls the annotation.
+			name: "new template: high request throughput for a service",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{
+						ruletypes.LabelAlertName:    "HighRequestThroughput",
+						ruletypes.LabelSeverityName: "warning",
+						"service.name":              "payment-service",
+					},
+					map[string]string{"description": "Request rate exceeded 10k/s"},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "High request throughput for $service.name",
+				BodyTemplate: `The service $service.name is getting high request. Please investigate.
+Severity: $rule.severity
+Status: $alert.status
+Service: $service.name
+Description: $description`,
+			},
+			wantTitle: "High request throughput for payment-service",
+			wantBody: []string{`The service payment-service is getting high request. Please investigate.
+Severity: warning
+Status: firing
+Service: payment-service
+Description: Request rate exceeded 10k/s`},
+			wantIsDefaultBody: false,
+		},
+		{
+			// Disk usage alert using old Go template syntax throughout.
+			// No custom templates — both title and body use the default fallback path.
+			name: "old template: disk usage high on database host",
+			alerts: []*types.Alert{
+				createAlert(
+					map[string]string{ruletypes.LabelAlertName: "DiskUsageHigh",
+						ruletypes.LabelSeverityName: "critical",
+						"instance":                  "db-primary-01",
+					},
+					map[string]string{
+						"summary":        "Disk usage high on database host",
+						"description":    "Disk usage is high on the database host",
+						"related_logs":   "https://logs.example.com/search?q=DiskUsageHigh",
+						"related_traces": "https://traces.example.com/search?q=DiskUsageHigh",
+					},
+					true,
+				),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				DefaultTitleTemplate: `[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }}
+     {{- if gt (len .CommonLabels) (len .GroupLabels) -}}
+       {{" "}}(
+       {{- with .CommonLabels.Remove .GroupLabels.Names }}
+         {{- range $index, $label := .SortedPairs -}}
+           {{ if $index }}, {{ end }}
+           {{- $label.Name }}="{{ $label.Value -}}"
+         {{- end }}
+       {{- end -}}
+       )
+     {{- end }}`,
+				DefaultBodyTemplate: `{{ range .Alerts -}}
+     *Alert:* {{ .Labels.alertname }}{{ if .Labels.severity }} - {{ .Labels.severity }}{{ end }}
+
+     *Summary:* {{ .Annotations.summary }}
+     *Description:* {{ .Annotations.description }}
+     *RelatedLogs:* {{ if gt (len .Annotations.related_logs) 0 -}} View in <{{ .Annotations.related_logs }}|logs explorer> {{- end}}
+     *RelatedTraces:* {{ if gt (len .Annotations.related_traces) 0 -}} View in <{{ .Annotations.related_traces }}|traces explorer> {{- end}}
+
+     *Details:*
+       {{ range .Labels.SortedPairs }} • *{{ .Name }}:* {{ .Value }}
+       {{ end }}
+     {{ end }}`,
+			},
+			wantTitle: "[FIRING:1] DiskUsageHigh for  (instance=\"db-primary-01\")",
+			// Written with explicit \n so trailing whitespace inside the body
+			// (emitted by the un-trimmed "{{ end }}" in the default template)
+			// survives format-on-save.
+			wantBody: []string{"*Alert:* DiskUsageHigh - critical\n" +
+				"\n" +
+				"     *Summary:* Disk usage high on database host\n" +
+				"     *Description:* Disk usage is high on the database host\n" +
+				"     *RelatedLogs:* View in <https://logs.example.com/search?q=DiskUsageHigh|logs explorer>\n" +
+				"     *RelatedTraces:* View in <https://traces.example.com/search?q=DiskUsageHigh|traces explorer>\n" +
+				"\n" +
+				"     *Details:*\n" +
+				"        • *alertname:* DiskUsageHigh\n" +
+				"        • *instance:* db-primary-01\n" +
+				"        • *severity:* critical\n" +
+				"       \n" +
+				"     "},
+			wantIsDefaultBody: true,
+		},
+		{
+			// Pod crash loop on multiple pods — body is expanded once per alert
+			// and joined with "\n\n", with the pod name pulled from labels.
+			name: "new template: pod crash loop on multiple pods, body per-alert",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-1"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-2"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodCrashLoop", "pod": "api-worker-3"}, nil, true),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name: $alert.total_firing pods affected",
+				BodyTemplate:  "$labels.pod is crash looping",
+			},
+			wantTitle:         "PodCrashLoop: 3 pods affected",
+			wantBody:          []string{"api-worker-1 is crash looping", "api-worker-2 is crash looping", "api-worker-3 is crash looping"},
+			wantIsDefaultBody: false,
+		},
+		{
+			// Incident partially resolved — one service still down, one recovered.
+			// Title shows the aggregate counts; body shows per-service status.
+			name: "new template: service degradation with mixed firing and resolved alerts",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "auth-service"}, nil, true),
+				createAlert(map[string]string{ruletypes.LabelAlertName: "ServiceDown", "service": "payment-service"}, nil, false),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$alert.total_firing firing, $alert.total_resolved resolved",
+				BodyTemplate:  "$labels.service ($alert.status)",
+			},
+			wantTitle:         "1 firing, 1 resolved",
+			wantBody:          []string{"auth-service (firing)", "payment-service (resolved)"},
+			wantIsDefaultBody: false,
+		},
+		{
+			// $environment is not a known AlertData or NotificationTemplateData field,
+			// so it lands in MissingVars and renders as "<no value>" in the output.
+			name: "missing vars: unknown $environment variable in title",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "[$environment] $rule.name",
+			},
+			wantTitle:         "[<no value>] HighCPU",
+			wantMissingVars:   []string{"environment"},
+			wantIsDefaultBody: true,
+		},
+		{
+			// $runbook_url is not a known field — someone tried to embed a runbook link
+			// directly as a variable instead of via annotations.
+			name: "missing vars: unknown $runbook_url variable in body",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "PodOOMKilled", ruletypes.LabelSeverityName: "warning"}, nil, true),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				BodyTemplate: "$rule.name: see runbook at $runbook_url",
+			},
+			wantBody:        []string{"PodOOMKilled: see runbook at <no value>"},
+			wantMissingVars: []string{"runbook_url"},
+		},
+		{
+			// Both title and body use unknown variables; MissingVars is the union of both.
+			name: "missing vars: unknown variables in both title and body",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighMemory", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "[$environment] $rule.name and [{{ $service }}]",
+				BodyTemplate:  "$rule.name: see runbook at $runbook_url",
+			},
+			wantTitle:       "[<no value>] HighMemory and [<no value>]",
+			wantBody:        []string{"HighMemory: see runbook at <no value>"},
+			wantMissingVars: []string{"environment", "runbook_url", "service"},
+		},
+		{
+			// Custom title template that expands to only whitespace triggers the fallback,
+			// so the default title template is used instead.
+			name: "fallback: whitespace-only custom title falls back to default",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, false),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate:        "   ",
+				DefaultTitleTemplate: "{{ .CommonLabels.alertname }} ({{ .Status | toUpper }})",
+				DefaultBodyTemplate:  "Runbook: https://runbook.example.com",
+			},
+			wantTitle:         "HighCPU (RESOLVED)",
+			wantBody:          []string{"Runbook: https://runbook.example.com"},
+			wantIsDefaultBody: true,
+		},
+		{
+			name: "using non-existing function in template",
+			alerts: []*types.Alert{
+				createAlert(map[string]string{ruletypes.LabelAlertName: "HighCPU", ruletypes.LabelSeverityName: "critical"}, nil, true),
+			},
+			input: alertmanagertypes.ExpandRequest{
+				TitleTemplate: "$rule.name ({{$severity | toUpperAndTrim}}) for $alertname",
+			},
+			errorContains: "function \"toUpperAndTrim\" not defined",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := at.Expand(ctx, tc.input, tc.alerts)
+			if tc.errorContains != "" {
+				require.ErrorContains(t, err, tc.errorContains)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.wantTitle != "" {
+				require.Equal(t, tc.wantTitle, got.Title)
+			}
+			if tc.wantBody != nil {
+				require.Equal(t, tc.wantBody, got.Body)
+			}
+			require.Equal(t, tc.wantIsDefaultBody, got.IsDefaultBody)
+
+			if len(tc.wantMissingVars) == 0 {
+				require.Empty(t, got.MissingVars)
+			} else {
+				sort.Strings(tc.wantMissingVars)
+				require.Equal(t, tc.wantMissingVars, got.MissingVars)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/annotations.go

@@ -0,0 +1,33 @@
+package alertmanagertemplate
+
+import (
+	"github.com/SigNoz/signoz/pkg/types/ruletypes"
+	"github.com/prometheus/alertmanager/types"
+)
+
+// ExtractTemplatesFromAnnotations pulls the user-authored title and body
+// templates off the well-known annotation keys attached by the rule manager.
+// A template is returned only if every alert in the group carries the same
+// value under that key; otherwise the empty string is returned for that slot
+// (which causes Expand to fall back to the channel default).
+func ExtractTemplatesFromAnnotations(alerts []*types.Alert) (titleTemplate, bodyTemplate string) {
+	if len(alerts) == 0 {
+		return "", ""
+	}
+
+	title := string(alerts[0].Annotations[ruletypes.AnnotationTitleTemplate])
+	body := string(alerts[0].Annotations[ruletypes.AnnotationBodyTemplate])
+
+	for _, a := range alerts[1:] {
+		if title != "" && string(a.Annotations[ruletypes.AnnotationTitleTemplate]) != title {
+			title = ""
+		}
+		if body != "" && string(a.Annotations[ruletypes.AnnotationBodyTemplate]) != body {
+			body = ""
+		}
+		if title == "" && body == "" {
+			break
+		}
+	}
+	return title, body
+}

pkg/alertmanager/alertmanagertemplate/kvaggregate_test.go

@@ -0,0 +1,153 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+
+	"github.com/prometheus/alertmanager/template"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAggregateKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert",
+			alerts: []*types.Alert{
+				{
+					Alert: model.Alert{
+						Labels: model.LabelSet{
+							"env":     "production",
+							"service": "backend",
+						},
+					},
+				},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend",
+			},
+		},
+		{
+			name: "varying values with duplicates deduped",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "backend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "frontend"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "production", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"env":     "production",
+				"service": "backend, api, frontend",
+			},
+		},
+		{
+			name: "more than 5 unique values truncates to 5",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc1"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc2"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc3"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc4"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc5"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc6"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "svc7"}}},
+			},
+			extractFn: extractLabels,
+			expected: template.KV{
+				"service": "svc1, svc2, svc3, svc4, svc5",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := aggregateKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestExtractCommonKV(t *testing.T) {
+	extractLabels := func(a *types.Alert) model.LabelSet { return a.Labels }
+	extractAnnotations := func(a *types.Alert) model.LabelSet { return a.Annotations }
+
+	testCases := []struct {
+		name      string
+		alerts    []*types.Alert
+		extractFn func(*types.Alert) model.LabelSet
+		expected  template.KV
+	}{
+		{
+			name:      "empty alerts slice",
+			alerts:    []*types.Alert{},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "single alert returns all labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "service": "api"},
+		},
+		{
+			name: "multiple alerts with fully common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "region": "us-east"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod", "region": "us-east"},
+		},
+		{
+			name: "multiple alerts with partially common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"env": "prod", "service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{"env": "prod"},
+		},
+		{
+			name: "multiple alerts with no common labels",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "api"}}},
+				{Alert: model.Alert{Labels: model.LabelSet{"service": "worker"}}},
+			},
+			extractFn: extractLabels,
+			expected:  template.KV{},
+		},
+		{
+			name: "annotations extract common annotations",
+			alerts: []*types.Alert{
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://x"}}},
+				{Alert: model.Alert{Annotations: model.LabelSet{"summary": "high cpu", "runbook": "http://y"}}},
+			},
+			extractFn: extractAnnotations,
+			expected:  template.KV{"summary": "high cpu"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractCommonKV(tc.alerts, tc.extractFn)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/preprocessor.go

@@ -0,0 +1,318 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/go-viper/mapstructure/v2"
+)
+
+// fieldPath is a dotted mapstructure path into the templating data map,
+// e.g. "alert.is_firing" or "rule.threshold.value".
+type fieldPath string
+
+// extractFieldMappings flattens the struct hierarchy into a list of dotted
+// mapstructure paths that user templates can reference. It emits:
+//   - a leaf for every scalar field
+//   - a leaf for every map field (labels, annotations)
+//   - a mapping for each intermediate sub-struct itself, so {{ $alert := .alert }}
+//     bindings let action blocks write {{ if $alert.is_firing }}
+//
+// Slices and interfaces are not surfaced. Pointer fields are dereferenced.
+func extractFieldMappings(data any) []fieldPath {
+	val := reflect.ValueOf(data)
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		val = val.Elem()
+	}
+	if val.Kind() != reflect.Struct {
+		return nil
+	}
+	return collectFieldMappings(val, "")
+}
+
+func collectFieldMappings(val reflect.Value, prefix string) []fieldPath {
+	typ := val.Type()
+	var paths []fieldPath
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		if !field.IsExported() {
+			continue
+		}
+
+		tag := field.Tag.Get("mapstructure")
+		if tag == "" || tag == "-" {
+			continue
+		}
+		name := strings.Split(tag, ",")[0]
+		if name == "" {
+			continue
+		}
+
+		key := name
+		if prefix != "" {
+			key = prefix + "." + name
+		}
+
+		ft := field.Type
+		if ft.Kind() == reflect.Ptr {
+			ft = ft.Elem()
+		}
+
+		switch ft.Kind() {
+		case reflect.Slice, reflect.Interface:
+			continue
+		}
+
+		// Recurse into sub-structs (time.Time treated as a leaf).
+		if ft.Kind() == reflect.Struct && ft.String() != "time.Time" {
+			paths = append(paths, fieldPath(key))
+			fv := val.Field(i)
+			if fv.Kind() == reflect.Ptr {
+				if fv.IsNil() {
+					continue
+				}
+				fv = fv.Elem()
+			}
+			paths = append(paths, collectFieldMappings(fv, key)...)
+			continue
+		}
+
+		paths = append(paths, fieldPath(key))
+	}
+	return paths
+}
+
+// structRootSet returns the top-level mapstructure tag names whose field
+// type is a nested struct (excluding time.Time and map/slice/interface
+// fields). These are the paths the rewriter walks segment-by-segment; any
+// other dotted $-reference is treated as a flat key on the root map so that
+// flattened OTel-style label keys like "service.name" resolve naturally.
+func structRootSet(data any) map[string]bool {
+	val := reflect.ValueOf(data)
+	if val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		val = val.Elem()
+	}
+	if val.Kind() != reflect.Struct {
+		return nil
+	}
+
+	roots := make(map[string]bool)
+	typ := val.Type()
+	for i := 0; i < typ.NumField(); i++ {
+		field := typ.Field(i)
+		if !field.IsExported() {
+			continue
+		}
+		tag := field.Tag.Get("mapstructure")
+		if tag == "" || tag == "-" {
+			continue
+		}
+		name := strings.Split(tag, ",")[0]
+		if name == "" {
+			continue
+		}
+		ft := field.Type
+		if ft.Kind() == reflect.Ptr {
+			ft = ft.Elem()
+		}
+		if ft.Kind() == reflect.Struct && ft.String() != "time.Time" {
+			roots[name] = true
+		}
+	}
+	return roots
+}
+
+// buildDataMap converts the typed data struct to the map[string]any that the
+// template engine indexes into. Each label and annotation is additionally
+// exposed at the root under its raw key, so $service.name resolves a flat
+// OTel-style label as a single-key index on the root. Struct-path keys
+// already present at the root take precedence on collisions.
+func buildDataMap(data any) (map[string]any, error) {
+	var result map[string]any
+	if err := mapstructure.Decode(data, &result); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to build template data map")
+	}
+
+	flatten := func(labels, annotations map[string]string) {
+		for k, v := range labels {
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+		for k, v := range annotations {
+			if _, ok := result[k]; !ok {
+				result[k] = v
+			}
+		}
+	}
+
+	switch data := data.(type) {
+	case *alertmanagertypes.NotificationTemplateData:
+		flatten(data.Labels, data.Annotations)
+	case *alertmanagertypes.AlertData:
+		flatten(data.Labels, data.Annotations)
+	}
+	return result, nil
+}
+
+// renderPreamble serialises a map of binding name → RHS expression into
+// `{{ $name := expr }}` declarations. Dotted names are skipped: Go's
+// text/template parser rejects `{{ $a.b := ... }}`; dotted paths are resolved
+// at expansion time by the rewriter.
+func renderPreamble(bindings map[string]string) string {
+	if len(bindings) == 0 {
+		return ""
+	}
+	var sb strings.Builder
+	for name, expr := range bindings {
+		if strings.Contains(name, ".") {
+			continue
+		}
+		fmt.Fprintf(&sb, `{{ $%s := %s }}`, name, expr)
+	}
+	return sb.String()
+}
+
+// buildPreamble constructs the variable-definition preamble prepended to the
+// user template, covering:
+//   - known root-level struct paths ({{ $alert := .alert }})
+//   - "<no value>" stubs for $-refs whose first segment matches nothing, so
+//     action blocks like {{ if $custom_note }} don't error at parse time
+//
+// The set of unmatched names is returned separately so callers (preview API)
+// can surface warnings.
+func buildPreamble(tmpl string, data any) (string, map[string]bool, error) {
+	bindings := make(map[string]string)
+	// knownFirstSegments tracks every valid first segment of a $-ref, since
+	// extractUsedVariables only gives us first segments. A label key like
+	// "service.name" contributes "service" here, so $service.name isn't
+	// flagged as unknown even though "service" has no direct binding.
+	knownFirstSegments := make(map[string]bool)
+
+	for _, p := range extractFieldMappings(data) {
+		bindings[string(p)] = fmt.Sprintf(".%s", p)
+		knownFirstSegments[firstSegment(string(p))] = true
+	}
+	// Labels/annotations are flattened into the root map by buildDataMap, so
+	// a bare-accessible key (no dots) can be bound in the preamble — this is
+	// what makes {{ if $severity }} or {{ $severity | toUpper }} work in
+	// action blocks. Dotted label keys only contribute to knownFirstSegments:
+	// their action-block use would be a syntax error anyway ($a.b is not a
+	// valid Go template identifier).
+	for k := range dataLabelsAndAnnotations(data) {
+		knownFirstSegments[firstSegment(k)] = true
+		if !strings.Contains(k, ".") {
+			if _, ok := bindings[k]; !ok {
+				bindings[k] = fmt.Sprintf(".%s", k)
+			}
+		}
+	}
+
+	used, err := extractUsedVariables(tmpl)
+	if err != nil {
+		return "", nil, err
+	}
+
+	unknown := make(map[string]bool)
+	for name := range used {
+		if !knownFirstSegments[name] {
+			unknown[name] = true
+			bindings[name] = `"<no value>"`
+		}
+	}
+
+	return renderPreamble(bindings), unknown, nil
+}
+
+// firstSegment returns the portion of a dotted path before the first dot,
+// or the whole string if there is no dot.
+func firstSegment(path string) string {
+	if i := strings.IndexByte(path, '.'); i >= 0 {
+		return path[:i]
+	}
+	return path
+}
+
+// dataLabelsAndAnnotations returns the union of label and annotation keys on
+// the given data struct (if it carries them). Used for first-segment
+// recognition of $-refs that point into flat OTel-style label keys.
+func dataLabelsAndAnnotations(data any) map[string]struct{} {
+	keys := make(map[string]struct{})
+	switch d := data.(type) {
+	case *alertmanagertypes.NotificationTemplateData:
+		for k := range d.Labels {
+			keys[k] = struct{}{}
+		}
+		for k := range d.Annotations {
+			keys[k] = struct{}{}
+		}
+	case *alertmanagertypes.AlertData:
+		for k := range d.Labels {
+			keys[k] = struct{}{}
+		}
+		for k := range d.Annotations {
+			keys[k] = struct{}{}
+		}
+	}
+	return keys
+}
+
+// processingResult is the rewritten template and its backing data map,
+// ready to be passed to Go text/template's Execute.
+type processingResult struct {
+	// Template is the input template with $-refs rewritten to Go template
+	// syntax and an identifier preamble prepended.
+	Template string
+	// Data is the flattened map the template indexes into.
+	Data map[string]any
+	// UnknownVars are $-refs in the input that had no matching data path.
+	// They render as "<no value>" at execution; useful for preview warnings.
+	UnknownVars map[string]bool
+}
+
+// preProcessTemplateAndData prepares a user-authored template and its backing
+// struct for Go text/template execution.
+//
+//	Input (with data *AlertData):
+//	    "$rule.name fired with value $alert.value"
+//	Output:
+//	    "{{ $alert := .alert }}{{ $rule := .rule }}..."
+//	    "{{ index . \"rule\" \"name\" }} fired with value {{ index . \"alert\" \"value\" }}"
+func preProcessTemplateAndData(tmpl string, data any) (*processingResult, error) {
+	unknownVars := make(map[string]bool)
+	if tmpl == "" {
+		result, err := buildDataMap(data)
+		if err != nil {
+			return nil, err
+		}
+		return &processingResult{Data: result, UnknownVars: unknownVars}, nil
+	}
+
+	preamble, unknownVars, err := buildPreamble(tmpl, data)
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to build template preamble")
+	}
+
+	// Prepend the preamble so wrapDollarVariables can parse the AST without
+	// "undefined variable" errors for $-refs inside action blocks.
+	wrapped, err := wrapDollarVariables(preamble+tmpl, structRootSet(data))
+	if err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to rewrite template")
+	}
+
+	result, err := buildDataMap(data)
+	if err != nil {
+		return nil, err
+	}
+
+	return &processingResult{Template: wrapped, Data: result, UnknownVars: unknownVars}, nil
+}

pkg/alertmanager/alertmanagertemplate/preprocessor_test.go

@@ -0,0 +1,380 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/types/alertmanagertypes"
+	"github.com/prometheus/alertmanager/template"
+	"github.com/stretchr/testify/require"
+)
+
+func TestExtractFieldMappings(t *testing.T) {
+	// Flat struct: mapstructure-tagged leaves only. Slices and interfaces are
+	// dropped; maps (labels/annotations analogues) are kept as top-level leaves.
+	type Flat struct {
+		Name       string            `mapstructure:"name"`
+		Status     string            `mapstructure:"status"`
+		UserCount  int               `mapstructure:"user_count"`
+		IsActive   bool              `mapstructure:"is_active"`
+		CreatedAt  time.Time         `mapstructure:"created_at"`
+		Extra      map[string]string `mapstructure:"extra"`
+		Items      []string          `mapstructure:"items"` // slice skipped
+		unexported string            //nolint:unused // unexported skipped
+		NoTag      string            // no mapstructure tag skipped
+		SkippedTag string            `mapstructure:"-"` // explicit skip
+	}
+
+	// Nested struct: sub-struct paths are flattened into dotted mappings; the
+	// parent path itself is also emitted so templates can bind `$alert := .alert`.
+	type Inner struct {
+		Value string `mapstructure:"value"`
+		Op    string `mapstructure:"op"`
+	}
+	type Outer struct {
+		Alert struct {
+			Status   string `mapstructure:"status"`
+			IsFiring bool   `mapstructure:"is_firing"`
+		} `mapstructure:"alert"`
+		Rule struct {
+			Name      string `mapstructure:"name"`
+			Threshold Inner  `mapstructure:"threshold"`
+		} `mapstructure:"rule"`
+	}
+
+	testCases := []struct {
+		name     string
+		data     any
+		expected []fieldPath
+	}{
+		{
+			name: "flat struct surfaces only mapstructure-tagged scalars",
+			data: Flat{},
+			expected: []fieldPath{
+				"name", "status", "user_count", "is_active", "created_at", "extra",
+			},
+		},
+		{
+			name: "nested struct emits parent and dotted leaf paths",
+			data: Outer{},
+			expected: []fieldPath{
+				"alert",
+				"alert.status",
+				"alert.is_firing",
+				"rule",
+				"rule.name",
+				"rule.threshold",
+				"rule.threshold.value",
+				"rule.threshold.op",
+			},
+		},
+		{
+			name:     "nil data",
+			data:     nil,
+			expected: nil,
+		},
+		{
+			name:     "non-struct type",
+			data:     "string",
+			expected: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result := extractFieldMappings(tc.data)
+			require.Equal(t, tc.expected, result)
+		})
+	}
+}
+
+func TestBuildVariableDefinitions(t *testing.T) {
+	testCases := []struct {
+		name          string
+		tmpl          string
+		data          any
+		expectedVars  []string // substrings that must appear in result
+		forbiddenVars []string // substrings that must NOT appear (dotted identifiers)
+		expectError   bool
+	}{
+		{
+			name: "empty template still emits struct bindings for title data",
+			tmpl: "",
+			data: &alertmanagertypes.NotificationTemplateData{Alert: alertmanagertypes.NotificationAlert{Receiver: "slack"}},
+			expectedVars: []string{
+				"{{ $alert := .alert }}",
+				"{{ $rule := .rule }}",
+			},
+			// Dotted leaves are NOT emitted as preamble bindings — they're
+			// reached via {{ index . "alert" "status" }} at expansion time.
+			forbiddenVars: []string{
+				"$alert.status",
+				"$rule.threshold.value",
+			},
+		},
+		{
+			name: "mix of known and unknown vars in alert body",
+			tmpl: "$rule.name fired: $custom_label",
+			data: &alertmanagertypes.AlertData{Rule: alertmanagertypes.RuleInfo{Name: "test"}, Alert: alertmanagertypes.AlertInfo{Status: "firing"}},
+			expectedVars: []string{
+				"{{ $alert := .alert }}",
+				"{{ $rule := .rule }}",
+				`{{ $custom_label := "<no value>" }}`,
+			},
+		},
+		{
+			name: "known dotted variables do not get flagged as unknown",
+			tmpl: "$alert.is_firing and $rule.threshold.value",
+			data: &alertmanagertypes.AlertData{},
+			// $alert and $rule (first segments) are in mappings, so no unknown
+			// stubs; the dotted leaves are resolved by WrapDollarVariables.
+			expectedVars: []string{
+				"{{ $alert := .alert }}",
+				"{{ $rule := .rule }}",
+			},
+			forbiddenVars: []string{
+				`"<no value>"`,
+			},
+		},
+		{
+			// Label-derived $-refs aren't stubbed as unknown; their first
+			// segment is marked known so {{ $severity := ... }} stubs don't
+			// appear in the preamble. Resolution happens at expansion via the
+			// root-level flattening performed in buildDataMap.
+			name: "label first-segments suppress unknown-var stubs",
+			tmpl: "$severity for $service $cloud.region.instance",
+			data: &alertmanagertypes.NotificationTemplateData{Labels: template.KV{
+				"severity":              "critical",
+				"service":               "test",
+				"cloud.region.instance": "ap-south-1",
+			}},
+			forbiddenVars: []string{
+				`{{ $severity := "<no value>" }}`,
+				`{{ $service := "<no value>" }}`,
+				`{{ $cloud := "<no value>" }}`,
+			},
+		},
+		{
+			name: "same rule holds for AlertData labels",
+			tmpl: "$severity $service",
+			data: &alertmanagertypes.AlertData{Labels: template.KV{
+				"severity": "critical",
+				"service":  "test",
+			}},
+			forbiddenVars: []string{
+				`{{ $severity := "<no value>" }}`,
+				`{{ $service := "<no value>" }}`,
+			},
+		},
+		{
+			name:        "invalid template syntax returns error",
+			tmpl:        "{{invalid",
+			data:        &alertmanagertypes.NotificationTemplateData{},
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, _, err := buildPreamble(tc.tmpl, tc.data)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			for _, expected := range tc.expectedVars {
+				require.Contains(t, result, expected, "expected preamble substring missing")
+			}
+			for _, forbidden := range tc.forbiddenVars {
+				require.NotContains(t, result, forbidden, "unexpected preamble substring present")
+			}
+		})
+	}
+}
+
+func TestPreProcessTemplateAndData(t *testing.T) {
+	testCases := []struct {
+		name                     string
+		tmpl                     string
+		data                     any
+		expectedTemplateContains []string
+		expectedData             map[string]any
+		expectedUnknownVars      map[string]bool
+		expectError              bool
+	}{
+		{
+			name: "title template: struct-root walks and flat dotted label keys",
+			tmpl: "[$alert.status] $rule.name (ID: $rule.id) firing=$alert.total_firing severity=$severity method=$http.request.method",
+			data: &alertmanagertypes.NotificationTemplateData{
+				Alert: alertmanagertypes.NotificationAlert{
+					Receiver:    "pagerduty",
+					Status:      "firing",
+					TotalFiring: 3,
+				},
+				Rule: alertmanagertypes.RuleInfo{
+					Name: "HighMemory",
+					ID:   "rule-123",
+				},
+				Labels: template.KV{
+					"severity":            "critical",
+					"http.request.method": "GET",
+				},
+			},
+			expectedTemplateContains: []string{
+				"{{$alert := .alert}}",
+				"{{$rule := .rule}}",
+				`[{{ index . "alert" "status" }}] {{ index . "rule" "name" }} (ID: {{ index . "rule" "id" }})`,
+				`firing={{ index . "alert" "total_firing" }} severity={{ .severity }}`,
+				`method={{ index . "http.request.method" }}`,
+			},
+			expectedData: map[string]any{
+				"alert": map[string]any{
+					"receiver":       "pagerduty",
+					"status":         "firing",
+					"total_firing":   3,
+					"total_resolved": 0,
+				},
+				"severity":            "critical",
+				"http.request.method": "GET",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			name: "body template with nested threshold access and per-alert annotation",
+			tmpl: "$rule.name: value $alert.value $rule.threshold.op $rule.threshold.value ($alert.status) desc=$description",
+			data: &alertmanagertypes.AlertData{
+				Alert: alertmanagertypes.AlertInfo{
+					Status:   "firing",
+					Value:    "85%",
+					IsFiring: true,
+				},
+				Rule: alertmanagertypes.RuleInfo{
+					Name:      "DiskFull",
+					ID:        "disk-001",
+					Severity:  "warning",
+					Threshold: alertmanagertypes.Threshold{Value: "80%", Op: ">"},
+				},
+				Annotations: template.KV{
+					"description": "Disk full and cannot be written to",
+				},
+			},
+			expectedTemplateContains: []string{
+				"{{$alert := .alert}}",
+				"{{$rule := .rule}}",
+				// "description" is an annotation flattened to root; the preamble
+				// now binds it off the root rather than via .annotations lookup.
+				"{{$description := .description}}",
+				`{{ index . "rule" "name" }}: value {{ index . "alert" "value" }} {{ index . "rule" "threshold" "op" }} {{ index . "rule" "threshold" "value" }}`,
+			},
+			expectedData: map[string]any{
+				"description": "Disk full and cannot be written to",
+			},
+			expectedUnknownVars: map[string]bool{},
+		},
+		{
+			// Struct roots reserve their first-segment namespace: a label
+			// whose key starts with "alert." is shadowed by the Alert sub-map,
+			// and must be accessed via the explicit $labels.* prefix.
+			name: "label colliding with struct root is accessed via $labels.*",
+			tmpl: "$alert.status via $labels.alert.custom",
+			data: &alertmanagertypes.NotificationTemplateData{
+				Alert:  alertmanagertypes.NotificationAlert{Status: "firing"},
+				Labels: template.KV{"alert.custom": "x"},
+			},
+			expectedTemplateContains: []string{
+				`{{ index . "alert" "status" }}`,
+				`{{ index .labels "alert.custom" }}`,
+			},
+		},
+		{
+			// Same shadowing rule applies symmetrically to annotations.
+			name: "annotation colliding with struct root is accessed via $annotations.*",
+			tmpl: "$alert.status via $annotations.alert.meta",
+			data: &alertmanagertypes.NotificationTemplateData{
+				Alert:       alertmanagertypes.NotificationAlert{Status: "firing"},
+				Annotations: template.KV{"alert.meta": "x"},
+			},
+			expectedTemplateContains: []string{
+				`{{ index . "alert" "status" }}`,
+				`{{ index .annotations "alert.meta" }}`,
+			},
+		},
+		{
+			// When a label and an annotation share a key, the label wins at the
+			// root flattening layer. Users who want the annotation must address
+			// it explicitly via $annotations.<key>.
+			name: "label takes precedence over same-named annotation at root",
+			tmpl: "flat=$env labels_only=$labels.env annotations_only=$annotations.env",
+			data: &alertmanagertypes.NotificationTemplateData{
+				Labels:      template.KV{"env": "prod"},
+				Annotations: template.KV{"env": "staging"},
+			},
+			expectedTemplateContains: []string{
+				`flat={{ .env }}`,
+				`labels_only={{ index .labels "env" }}`,
+				`annotations_only={{ index .annotations "env" }}`,
+			},
+			expectedData: map[string]any{
+				"env": "prod",
+			},
+		},
+		{
+			name: "empty template returns flattened data",
+			tmpl: "",
+			data: &alertmanagertypes.NotificationTemplateData{Alert: alertmanagertypes.NotificationAlert{Receiver: "slack"}},
+		},
+		{
+			name:        "invalid template syntax",
+			tmpl:        "{{invalid",
+			data:        &alertmanagertypes.NotificationTemplateData{},
+			expectError: true,
+		},
+		{
+			name: "unknown dollar var in text renders empty",
+			tmpl: "alert $custom_note fired",
+			data: &alertmanagertypes.NotificationTemplateData{Rule: alertmanagertypes.RuleInfo{Name: "HighCPU"}},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				"alert {{ .custom_note }} fired",
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+		{
+			name: "unknown dollar var in action block renders empty",
+			tmpl: "alert {{ $custom_note }} fired",
+			data: &alertmanagertypes.NotificationTemplateData{Rule: alertmanagertypes.RuleInfo{Name: "HighCPU"}},
+			expectedTemplateContains: []string{
+				`{{$custom_note := "<no value>"}}`,
+				`alert {{$custom_note}} fired`,
+			},
+			expectedUnknownVars: map[string]bool{"custom_note": true},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := preProcessTemplateAndData(tc.tmpl, tc.data)
+
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			if tc.tmpl == "" {
+				require.Equal(t, "", result.Template)
+				return
+			}
+
+			for _, substr := range tc.expectedTemplateContains {
+				require.Contains(t, result.Template, substr)
+			}
+			for k, v := range tc.expectedData {
+				require.Equal(t, v, result.Data[k], "data[%q] mismatch", k)
+			}
+			if tc.expectedUnknownVars != nil {
+				require.Equal(t, tc.expectedUnknownVars, result.UnknownVars)
+			}
+		})
+	}
+}

pkg/alertmanager/alertmanagertemplate/rewrite.go

@@ -0,0 +1,155 @@
+package alertmanagertemplate
+
+import (
+	"fmt"
+	"reflect"
+	"regexp"
+	"strings"
+	"text/template/parse"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+)
+
+// bareVariableRegex matches $-references including dotted paths (e.g. $alert.is_firing).
+var bareVariableRegex = regexp.MustCompile(`\$(\w+(?:\.\w+)*)`)
+
+// bareVariableFirstSegRegex captures only the first segment of a $-reference.
+// $labels.severity yields "$labels"; $alert.is_firing yields "$alert".
+var bareVariableFirstSegRegex = regexp.MustCompile(`\$\w+`)
+
+// wrapDollarVariables rewrites bare $-references in a template's plain-text
+// regions into Go text/template syntax. References inside `{{ ... }}` action
+// blocks are left untouched — they're already template syntax. structRoots
+// is the set of top-level mapstructure names whose values are nested structs
+// (e.g. "alert", "rule"): $-refs whose first segment is in this set are
+// walked segment-by-segment; everything else dotted is treated as a flat key
+// on the root map so flattened OTel-style label keys resolve naturally.
+//
+// Examples (structRoots = {alert, rule}):
+//
+//	"$name is $status"       -> "{{ .name }} is {{ .status }}"
+//	"$labels.severity"       -> `{{ index .labels "severity" }}`
+//	"$labels.http.method"    -> `{{ index .labels "http.method" }}`
+//	"$annotations.summary"   -> `{{ index .annotations "summary" }}`
+//	"$alert.is_firing"       -> `{{ index . "alert" "is_firing" }}`
+//	"$rule.threshold.value"  -> `{{ index . "rule" "threshold" "value" }}`
+//	"$service.name"          -> `{{ index . "service.name" }}`
+//	"$name is {{ .Status }}" -> "{{ .name }} is {{ .Status }}"
+func wrapDollarVariables(src string, structRoots map[string]bool) (string, error) {
+	if src == "" {
+		return src, nil
+	}
+
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+
+	if _, err := tree.Parse(src, "{{", "}}", make(map[string]*parse.Tree), nil); err != nil {
+		return "", err
+	}
+
+	walkAndWrapTextNodes(tree.Root, structRoots)
+	return tree.Root.String(), nil
+}
+
+// walkAndWrapTextNodes descends the parse tree and rewrites $-references
+// found in TextNodes. If/Range bodies are recursed into. ActionNodes and
+// other `{{...}}` constructs are left alone because they're already template
+// syntax. WithNode is not yet supported — add when the editor grows a "with"
+// block.
+func walkAndWrapTextNodes(node parse.Node, structRoots map[string]bool) {
+	if reflect.ValueOf(node).IsNil() {
+		return
+	}
+
+	switch n := node.(type) {
+	case *parse.ListNode:
+		if n.Nodes != nil {
+			for _, child := range n.Nodes {
+				walkAndWrapTextNodes(child, structRoots)
+			}
+		}
+
+	case *parse.TextNode:
+		n.Text = bareVariableRegex.ReplaceAllFunc(n.Text, func(match []byte) []byte {
+			return rewriteDollarRef(match, structRoots)
+		})
+
+	case *parse.IfNode:
+		walkAndWrapTextNodes(n.List, structRoots)
+		walkAndWrapTextNodes(n.ElseList, structRoots)
+
+	case *parse.RangeNode:
+		walkAndWrapTextNodes(n.List, structRoots)
+		walkAndWrapTextNodes(n.ElseList, structRoots)
+	}
+}
+
+// rewriteDollarRef converts one $-reference (with the leading $) into the
+// corresponding Go template expression.
+//
+//   - labels./annotations. prefixes: treat the remainder as a single map key
+//     (OTel-style keys like "http.request.method" are flat keys, not paths).
+//   - Dotted path with a struct-root first segment: walk via chained index
+//     arguments. `index x a b c` is equivalent to x[a][b][c].
+//   - Other dotted path: treat as a single flat key on the root map, so a
+//     flattened OTel-style label key like "service.name" resolves.
+//   - Simple names: plain dot access on the root map.
+func rewriteDollarRef(match []byte, structRoots map[string]bool) []byte {
+	name := string(match[1:])
+
+	if !strings.Contains(name, ".") {
+		return fmt.Appendf(nil, `{{ .%s }}`, name)
+	}
+
+	if key, ok := strings.CutPrefix(name, "labels."); ok {
+		return fmt.Appendf(nil, `{{ index .labels %q }}`, key)
+	}
+	if key, ok := strings.CutPrefix(name, "annotations."); ok {
+		return fmt.Appendf(nil, `{{ index .annotations %q }}`, key)
+	}
+
+	// If the first segment is a known struct root, walk segments.
+	if dot := strings.IndexByte(name, '.'); dot >= 0 && structRoots[name[:dot]] {
+		parts := strings.Split(name, ".")
+		args := make([]string, len(parts))
+		for i, p := range parts {
+			args[i] = fmt.Sprintf("%q", p)
+		}
+		return fmt.Appendf(nil, `{{ index . %s }}`, strings.Join(args, " "))
+	}
+
+	// Otherwise treat the full dotted path as a single flat root key.
+	return fmt.Appendf(nil, `{{ index . %q }}`, name)
+}
+
+// extractUsedVariables returns the set of every base $-name referenced in the
+// template — text nodes, action blocks, branch conditions, loop declarations.
+// Names are first-segment only: $labels.severity contributes "labels".
+//
+// The template is validated during extraction (a synthesised preamble
+// pre-declares each name so the parser doesn't trip on "undefined variable"
+// for names that genuinely exist in the data), so a parse error here
+// indicates a genuine syntax problem rather than a missing binding.
+func extractUsedVariables(src string) (map[string]bool, error) {
+	if src == "" {
+		return map[string]bool{}, nil
+	}
+
+	used := make(map[string]bool)
+	for _, m := range bareVariableFirstSegRegex.FindAll([]byte(src), -1) {
+		used[string(m[1:])] = true
+	}
+
+	var preamble strings.Builder
+	for name := range used {
+		fmt.Fprintf(&preamble, `{{$%s := ""}}`, name)
+	}
+
+	tree := parse.New("template")
+	tree.Mode = parse.SkipFuncCheck
+	if _, err := tree.Parse(preamble.String()+src, "{{", "}}", make(map[string]*parse.Tree), nil); err != nil {
+		return nil, errors.WrapInvalidInputf(err, errors.CodeInternal, "failed to extract used variables")
+	}
+
+	return used, nil
+}

pkg/alertmanager/alertmanagertemplate/rewrite_test.go

@@ -0,0 +1,213 @@
+package alertmanagertemplate
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWrapBareVars(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:     "mixed variables with actions",
+			input:    "$name is {{.Status}}",
+			expected: "{{ .name }} is {{.Status}}",
+		},
+		{
+			name: "nested variables in range",
+			input: `{{range .items}}
+					$title
+					{{end}}`,
+			expected: `{{range .items}}
+					{{ .title }}
+					{{end}}`,
+		},
+		{
+			name:     "nested variables in if else",
+			input:    "{{if .ok}}$a{{else}}$b{{end}}",
+			expected: "{{if .ok}}{{ .a }}{{else}}{{ .b }}{{end}}",
+		},
+		// Labels prefix: index into .labels map
+		{
+			name:     "labels variables prefix simple",
+			input:    "$labels.service",
+			expected: `{{ index .labels "service" }}`,
+		},
+		{
+			name:     "labels variables prefix nested with multiple dots",
+			input:    "$labels.http.status",
+			expected: `{{ index .labels "http.status" }}`,
+		},
+		{
+			name:     "multiple labels variables simple and nested",
+			input:    "$labels.service and $labels.instance.id",
+			expected: `{{ index .labels "service" }} and {{ index .labels "instance.id" }}`,
+		},
+		// Annotations prefix: index into .annotations map
+		{
+			name:     "annotations variables prefix simple",
+			input:    "$annotations.summary",
+			expected: `{{ index .annotations "summary" }}`,
+		},
+		{
+			name:     "annotations variables prefix nested with multiple dots",
+			input:    "$annotations.alert.url",
+			expected: `{{ index .annotations "alert.url" }}`,
+		},
+		// Struct-root paths: walk segment-by-segment via chained index.
+		{
+			name:     "struct-root dotted path walks via chained index",
+			input:    "$alert.is_firing",
+			expected: `{{ index . "alert" "is_firing" }}`,
+		},
+		{
+			name:     "deeply nested struct-root path",
+			input:    "$rule.threshold.value",
+			expected: `{{ index . "rule" "threshold" "value" }}`,
+		},
+		// Non-struct-root dotted paths: treated as a single flat key on the
+		// root map, so flattened OTel-style label keys resolve naturally.
+		{
+			name:     "non-struct-root dotted path hits flat root key",
+			input:    "$service.name",
+			expected: `{{ index . "service.name" }}`,
+		},
+		// Hybrid: all types combined
+		{
+			name:     "hybrid - all variables types",
+			input:    "Alert: $alert_name Labels: $labels.severity Annotations: $annotations.desc Value: $alert.value Count: $error_count",
+			expected: `Alert: {{ .alert_name }} Labels: {{ index .labels "severity" }} Annotations: {{ index .annotations "desc" }} Value: {{ index . "alert" "value" }} Count: {{ .error_count }}`,
+		},
+		{
+			name:     "already wrapped should not be changed",
+			input:    "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+			expected: "{{$status := .status}}{{.name}} is {{$status | toUpper}}",
+		},
+		{
+			name:     "no variables should not be changed",
+			input:    "Hello world",
+			expected: "Hello world",
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: "",
+		},
+		{
+			name:     "deeply nested",
+			input:    "{{range .items}}{{if .ok}}$deep{{end}}{{end}}",
+			expected: "{{range .items}}{{if .ok}}{{ .deep }}{{end}}{{end}}",
+		},
+		{
+			name: "complex example",
+			input: `Hello $name, your score is $score.
+					{{if .isAdmin}}
+						Welcome back $name, you have {{.unreadCount}} messages.
+					{{end}}`,
+			expected: `Hello {{ .name }}, your score is {{ .score }}.
+					{{if .isAdmin}}
+						Welcome back {{ .name }}, you have {{.unreadCount}} messages.
+					{{end}}`,
+		},
+		{
+			name:     "with custom function",
+			input:    "$name triggered at {{urlescape .url}}",
+			expected: "{{ .name }} triggered at {{urlescape .url}}",
+		},
+		{
+			name:        "invalid template",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	// structRoots used across the test cases: "alert" and "rule" are walked,
+	// anything else dotted is treated as a flat root-map key.
+	structRoots := map[string]bool{"alert": true, "rule": true}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := wrapDollarVariables(tc.input, structRoots)
+
+			if tc.expectError {
+				require.Error(t, err, "should error on invalid template syntax")
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestExtractUsedVariables(t *testing.T) {
+	testCases := []struct {
+		name        string
+		input       string
+		expected    map[string]bool
+		expectError bool
+	}{
+		{
+			name:     "simple usage in text",
+			input:    "$name is $status",
+			expected: map[string]bool{"name": true, "status": true},
+		},
+		{
+			name:     "declared in action block",
+			input:    "{{ $name := .name }}",
+			expected: map[string]bool{"name": true},
+		},
+		{
+			name:     "range loop vars",
+			input:    "{{ range $i, $v := .items }}{{ end }}",
+			expected: map[string]bool{"i": true, "v": true},
+		},
+		{
+			name:     "mixed text and action",
+			input:    "$x and {{ $y }}",
+			expected: map[string]bool{"x": true, "y": true},
+		},
+		{
+			name:     "dotted path in text extracts base only",
+			input:    "$labels.severity",
+			expected: map[string]bool{"labels": true},
+		},
+		{
+			name:     "nested if else",
+			input:    "{{ if .ok }}{{ $a }}{{ else }}{{ $b }}{{ end }}",
+			expected: map[string]bool{"a": true, "b": true},
+		},
+		{
+			name:     "empty string",
+			input:    "",
+			expected: map[string]bool{},
+		},
+		{
+			name:     "no variables",
+			input:    "Hello world",
+			expected: map[string]bool{},
+		},
+		{
+			name:        "invalid template returns error",
+			input:       "{{invalid",
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			result, err := extractUsedVariables(tc.input)
+
+			if tc.expectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expected, result)
+			}
+		})
+	}
+}

pkg/query-service/app/clickhouseReader/reader.go

@@ -160,9 +160,11 @@ type ClickHouseReader struct {
 	traceResourceTableV3 string
 	traceSummaryTable    string
 
-	cache         cache.Cache
-	metadataDB    string
-	metadataTable string
+	fluxIntervalForTraceDetail time.Duration
+	cache                      cache.Cache
+	cacheForTraceDetail        cache.Cache
+	metadataDB                 string
+	metadataTable              string
 }
 
 // NewTraceReader returns a TraceReader for the database
@@ -172,6 +174,8 @@ func NewReader(
 	telemetryStore telemetrystore.TelemetryStore,
 	prometheus prometheus.Prometheus,
 	cluster string,
+	fluxIntervalForTraceDetail time.Duration,
+	cacheForTraceDetail cache.Cache,
 	cache cache.Cache,
 	options *Options,
 ) *ClickHouseReader {
@@ -185,43 +189,45 @@ func NewReader(
 	traceLocalTableName := options.primary.TraceLocalTableNameV3
 
 	return &ClickHouseReader{
-		db:                       telemetryStore.ClickhouseDB(),
-		logger:                   logger,
-		prometheus:               prometheus,
-		sqlDB:                    sqlDB,
-		TraceDB:                  options.primary.TraceDB,
-		operationsTable:          options.primary.OperationsTable,
-		indexTable:               options.primary.IndexTable,
-		errorTable:               options.primary.ErrorTable,
-		usageExplorerTable:       options.primary.UsageExplorerTable,
-		durationTable:            options.primary.DurationTable,
-		SpansTable:               options.primary.SpansTable,
-		spanAttributeTableV2:     options.primary.SpanAttributeTableV2,
-		spanAttributesKeysTable:  options.primary.SpanAttributeKeysTable,
-		dependencyGraphTable:     options.primary.DependencyGraphTable,
-		topLevelOperationsTable:  options.primary.TopLevelOperationsTable,
-		logsDB:                   options.primary.LogsDB,
-		logsTable:                options.primary.LogsTable,
-		logsLocalTable:           options.primary.LogsLocalTable,
-		logsAttributeKeys:        options.primary.LogsAttributeKeysTable,
-		logsResourceKeys:         options.primary.LogsResourceKeysTable,
-		logsTagAttributeTableV2:  options.primary.LogsTagAttributeTableV2,
-		liveTailRefreshSeconds:   options.primary.LiveTailRefreshSeconds,
-		cluster:                  cluster,
-		queryProgressTracker:     queryprogress.NewQueryProgressTracker(logger),
-		logsTableV2:              options.primary.LogsTableV2,
-		logsLocalTableV2:         options.primary.LogsLocalTableV2,
-		logsResourceTableV2:      options.primary.LogsResourceTableV2,
-		logsResourceLocalTableV2: options.primary.LogsResourceLocalTableV2,
-		logsTableName:            logsTableName,
-		logsLocalTableName:       logsLocalTableName,
-		traceLocalTableName:      traceLocalTableName,
-		traceTableName:           traceTableName,
-		traceResourceTableV3:     options.primary.TraceResourceTableV3,
-		traceSummaryTable:        options.primary.TraceSummaryTable,
-		cache:                    cache,
-		metadataDB:               options.primary.MetadataDB,
-		metadataTable:            options.primary.MetadataTable,
+		db:                         telemetryStore.ClickhouseDB(),
+		logger:                     logger,
+		prometheus:                 prometheus,
+		sqlDB:                      sqlDB,
+		TraceDB:                    options.primary.TraceDB,
+		operationsTable:            options.primary.OperationsTable,
+		indexTable:                 options.primary.IndexTable,
+		errorTable:                 options.primary.ErrorTable,
+		usageExplorerTable:         options.primary.UsageExplorerTable,
+		durationTable:              options.primary.DurationTable,
+		SpansTable:                 options.primary.SpansTable,
+		spanAttributeTableV2:       options.primary.SpanAttributeTableV2,
+		spanAttributesKeysTable:    options.primary.SpanAttributeKeysTable,
+		dependencyGraphTable:       options.primary.DependencyGraphTable,
+		topLevelOperationsTable:    options.primary.TopLevelOperationsTable,
+		logsDB:                     options.primary.LogsDB,
+		logsTable:                  options.primary.LogsTable,
+		logsLocalTable:             options.primary.LogsLocalTable,
+		logsAttributeKeys:          options.primary.LogsAttributeKeysTable,
+		logsResourceKeys:           options.primary.LogsResourceKeysTable,
+		logsTagAttributeTableV2:    options.primary.LogsTagAttributeTableV2,
+		liveTailRefreshSeconds:     options.primary.LiveTailRefreshSeconds,
+		cluster:                    cluster,
+		queryProgressTracker:       queryprogress.NewQueryProgressTracker(logger),
+		logsTableV2:                options.primary.LogsTableV2,
+		logsLocalTableV2:           options.primary.LogsLocalTableV2,
+		logsResourceTableV2:        options.primary.LogsResourceTableV2,
+		logsResourceLocalTableV2:   options.primary.LogsResourceLocalTableV2,
+		logsTableName:              logsTableName,
+		logsLocalTableName:         logsLocalTableName,
+		traceLocalTableName:        traceLocalTableName,
+		traceTableName:             traceTableName,
+		traceResourceTableV3:       options.primary.TraceResourceTableV3,
+		traceSummaryTable:          options.primary.TraceSummaryTable,
+		fluxIntervalForTraceDetail: fluxIntervalForTraceDetail,
+		cache:                      cache,
+		cacheForTraceDetail:        cacheForTraceDetail,
+		metadataDB:                 options.primary.MetadataDB,
+		metadataTable:              options.primary.MetadataTable,
 	}
 }
 
@@ -891,6 +897,23 @@ func (r *ClickHouseReader) GetSpansForTrace(ctx context.Context, traceID string,
 	return searchScanResponses, nil
 }
 
+func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadataCache(ctx context.Context, orgID valuer.UUID, traceID string) (*model.GetWaterfallSpansForTraceWithMetadataCache, error) {
+	cachedTraceData := new(model.GetWaterfallSpansForTraceWithMetadataCache)
+	err := r.cacheForTraceDetail.Get(ctx, orgID, strings.Join([]string{"getWaterfallSpansForTraceWithMetadata", traceID}, "-"), cachedTraceData)
+	if err != nil {
+		r.logger.Debug("error in retrieving getWaterfallSpansForTraceWithMetadata cache", errorsV2.Attr(err), "traceID", traceID)
+		return nil, err
+	}
+
+	if time.Since(time.UnixMilli(int64(cachedTraceData.EndTime))) < r.fluxIntervalForTraceDetail {
+		r.logger.Info("the trace end time falls under the flux interval, skipping getWaterfallSpansForTraceWithMetadata cache", "traceID", traceID)
+		return nil, errors.Errorf("the trace end time falls under the flux interval, skipping getWaterfallSpansForTraceWithMetadata cache, traceID: %s", traceID)
+	}
+
+	r.logger.Info("cache is successfully hit, applying cache for getWaterfallSpansForTraceWithMetadata", "traceID", traceID)
+	return cachedTraceData, nil
+}
+
 func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Context, orgID valuer.UUID, traceID string, req *model.GetWaterfallSpansForTraceWithMetadataParams) (*model.GetWaterfallSpansForTraceWithMetadataResponse, error) {
 	response := new(model.GetWaterfallSpansForTraceWithMetadataResponse)
 	var startTime, endTime, durationNano, totalErrorSpans, totalSpans uint64
@@ -900,136 +923,172 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
 	var serviceNameIntervalMap = map[string][]tracedetail.Interval{}
 	var hasMissingSpans bool
 
-	searchScanResponses, err := r.GetSpansForTrace(ctx, traceID, fmt.Sprintf("SELECT DISTINCT ON (span_id) timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, links as references, attributes_string, attributes_number, attributes_bool, resources_string, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3 ORDER BY timestamp ASC, name ASC", r.TraceDB, r.traceTableName))
+	cachedTraceData, err := r.GetWaterfallSpansForTraceWithMetadataCache(ctx, orgID, traceID)
+	if err == nil {
+		startTime = cachedTraceData.StartTime
+		endTime = cachedTraceData.EndTime
+		durationNano = cachedTraceData.DurationNano
+		spanIdToSpanNodeMap = cachedTraceData.SpanIdToSpanNodeMap
+		serviceNameToTotalDurationMap = cachedTraceData.ServiceNameToTotalDurationMap
+		traceRoots = cachedTraceData.TraceRoots
+		totalSpans = cachedTraceData.TotalSpans
+		totalErrorSpans = cachedTraceData.TotalErrorSpans
+		hasMissingSpans = cachedTraceData.HasMissingSpans
+	}
+
 	if err != nil {
-		return nil, err
-	}
-	if len(searchScanResponses) == 0 {
-		return response, nil
-	}
-	totalSpans = uint64(len(searchScanResponses))
-	for _, item := range searchScanResponses {
-		ref := []model.OtelSpanRef{}
-		err := json.Unmarshal([]byte(item.References), &ref)
+		r.logger.Info("cache miss for getWaterfallSpansForTraceWithMetadata", "traceID", traceID)
+
+		searchScanResponses, err := r.GetSpansForTrace(ctx, traceID, fmt.Sprintf("SELECT DISTINCT ON (span_id) timestamp, duration_nano, span_id, trace_id, has_error, kind, resource_string_service$$name, name, links as references, attributes_string, attributes_number, attributes_bool, resources_string, events, status_message, status_code_string, kind_string FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3 ORDER BY timestamp ASC, name ASC", r.TraceDB, r.traceTableName))
 		if err != nil {
-			r.logger.Error("getWaterfallSpansForTraceWithMetadata: error unmarshalling references", errorsV2.Attr(err), "traceID", traceID)
-			return nil, errorsV2.Newf(errorsV2.TypeInvalidInput, errorsV2.CodeInvalidInput, "getWaterfallSpansForTraceWithMetadata: error unmarshalling references %s", err.Error())
+			return nil, err
 		}
-
-		// merge attributes_number and attributes_bool to attributes_string
-		for k, v := range item.Attributes_bool {
-			item.Attributes_string[k] = fmt.Sprintf("%v", v)
+		if len(searchScanResponses) == 0 {
+			return response, nil
 		}
-		for k, v := range item.Attributes_number {
-			item.Attributes_string[k] = strconv.FormatFloat(v, 'f', -1, 64)
-		}
-		for k, v := range item.Resources_string {
-			item.Attributes_string[k] = v
-		}
-
-		events := make([]model.Event, 0)
-		for _, event := range item.Events {
-			var eventMap model.Event
-			err = json.Unmarshal([]byte(event), &eventMap)
+		totalSpans = uint64(len(searchScanResponses))
+		processingBeforeCache := time.Now()
+		for _, item := range searchScanResponses {
+			ref := []model.OtelSpanRef{}
+			err := json.Unmarshal([]byte(item.References), &ref)
 			if err != nil {
-				r.logger.Error("Error unmarshalling events", errorsV2.Attr(err))
-				return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "getWaterfallSpansForTraceWithMetadata: error in unmarshalling events %s", err.Error())
+				r.logger.Error("getWaterfallSpansForTraceWithMetadata: error unmarshalling references", errorsV2.Attr(err), "traceID", traceID)
+				return nil, errorsV2.Newf(errorsV2.TypeInvalidInput, errorsV2.CodeInvalidInput, "getWaterfallSpansForTraceWithMetadata: error unmarshalling references %s", err.Error())
 			}
-			events = append(events, eventMap)
+
+			// merge attributes_number and attributes_bool to attributes_string
+			for k, v := range item.Attributes_bool {
+				item.Attributes_string[k] = fmt.Sprintf("%v", v)
+			}
+			for k, v := range item.Attributes_number {
+				item.Attributes_string[k] = strconv.FormatFloat(v, 'f', -1, 64)
+			}
+			for k, v := range item.Resources_string {
+				item.Attributes_string[k] = v
+			}
+
+			events := make([]model.Event, 0)
+			for _, event := range item.Events {
+				var eventMap model.Event
+				err = json.Unmarshal([]byte(event), &eventMap)
+				if err != nil {
+					r.logger.Error("Error unmarshalling events", errorsV2.Attr(err))
+					return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "getWaterfallSpansForTraceWithMetadata: error in unmarshalling events %s", err.Error())
+				}
+				events = append(events, eventMap)
+			}
+
+			startTimeUnixNano := uint64(item.TimeUnixNano.UnixNano())
+
+			jsonItem := model.Span{
+				SpanID:           item.SpanID,
+				TraceID:          item.TraceID,
+				ServiceName:      item.ServiceName,
+				Name:             item.Name,
+				Kind:             int32(item.Kind),
+				DurationNano:     item.DurationNano,
+				HasError:         item.HasError,
+				StatusMessage:    item.StatusMessage,
+				StatusCodeString: item.StatusCodeString,
+				SpanKind:         item.SpanKind,
+				References:       ref,
+				Events:           events,
+				TagMap:           item.Attributes_string,
+				Children:         make([]*model.Span, 0),
+				TimeUnixNano:     startTimeUnixNano, // Store nanoseconds temporarily
+			}
+
+			// metadata calculation
+			if startTime == 0 || startTimeUnixNano < startTime {
+				startTime = startTimeUnixNano
+			}
+			if endTime == 0 || (startTimeUnixNano+jsonItem.DurationNano) > endTime {
+				endTime = (startTimeUnixNano + jsonItem.DurationNano)
+			}
+			if durationNano == 0 || jsonItem.DurationNano > durationNano {
+				durationNano = jsonItem.DurationNano
+			}
+
+			if jsonItem.HasError {
+				totalErrorSpans = totalErrorSpans + 1
+			}
+
+			// collect the intervals for service for execution time calculation
+			serviceNameIntervalMap[jsonItem.ServiceName] =
+				append(serviceNameIntervalMap[jsonItem.ServiceName], tracedetail.Interval{StartTime: jsonItem.TimeUnixNano, Duration: jsonItem.DurationNano, Service: jsonItem.ServiceName})
+
+			// append to the span node map
+			spanIdToSpanNodeMap[jsonItem.SpanID] = &jsonItem
 		}
 
-		startTimeUnixNano := uint64(item.TimeUnixNano.UnixNano())
+		// traverse through the map and append each node to the children array of the parent node
+		// and add the missing spans
+		for _, spanNode := range spanIdToSpanNodeMap {
+			hasParentSpanNode := false
+			for _, reference := range spanNode.References {
+				if reference.RefType == "CHILD_OF" && reference.SpanId != "" {
+					hasParentSpanNode = true
 
-		jsonItem := model.Span{
-			SpanID:           item.SpanID,
-			TraceID:          item.TraceID,
-			ServiceName:      item.ServiceName,
-			Name:             item.Name,
-			Kind:             int32(item.Kind),
-			DurationNano:     item.DurationNano,
-			HasError:         item.HasError,
-			StatusMessage:    item.StatusMessage,
-			StatusCodeString: item.StatusCodeString,
-			SpanKind:         item.SpanKind,
-			References:       ref,
-			Events:           events,
-			TagMap:           item.Attributes_string,
-			Children:         make([]*model.Span, 0),
-			TimeUnixNano:     startTimeUnixNano, // Store nanoseconds temporarily
-		}
-
-		// metadata calculation
-		if startTime == 0 || startTimeUnixNano < startTime {
-			startTime = startTimeUnixNano
-		}
-		if endTime == 0 || (startTimeUnixNano+jsonItem.DurationNano) > endTime {
-			endTime = (startTimeUnixNano + jsonItem.DurationNano)
-		}
-		if durationNano == 0 || jsonItem.DurationNano > durationNano {
-			durationNano = jsonItem.DurationNano
-		}
-
-		if jsonItem.HasError {
-			totalErrorSpans = totalErrorSpans + 1
-		}
-
-		// collect the intervals for service for execution time calculation
-		serviceNameIntervalMap[jsonItem.ServiceName] =
-			append(serviceNameIntervalMap[jsonItem.ServiceName], tracedetail.Interval{StartTime: jsonItem.TimeUnixNano, Duration: jsonItem.DurationNano, Service: jsonItem.ServiceName})
-
-		// append to the span node map
-		spanIdToSpanNodeMap[jsonItem.SpanID] = &jsonItem
-	}
-
-	// traverse through the map and append each node to the children array of the parent node
-	// and add the missing spans
-	for _, spanNode := range spanIdToSpanNodeMap {
-		hasParentSpanNode := false
-		for _, reference := range spanNode.References {
-			if reference.RefType == "CHILD_OF" && reference.SpanId != "" {
-				hasParentSpanNode = true
-
-				if parentNode, exists := spanIdToSpanNodeMap[reference.SpanId]; exists {
-					parentNode.Children = append(parentNode.Children, spanNode)
-				} else {
-					// insert the missing span
-					missingSpan := model.Span{
-						SpanID:           reference.SpanId,
-						TraceID:          spanNode.TraceID,
-						ServiceName:      "",
-						Name:             "Missing Span",
-						TimeUnixNano:     spanNode.TimeUnixNano,
-						Kind:             0,
-						DurationNano:     spanNode.DurationNano,
-						HasError:         false,
-						StatusMessage:    "",
-						StatusCodeString: "",
-						SpanKind:         "",
-						Events:           make([]model.Event, 0),
-						Children:         make([]*model.Span, 0),
+					if parentNode, exists := spanIdToSpanNodeMap[reference.SpanId]; exists {
+						parentNode.Children = append(parentNode.Children, spanNode)
+					} else {
+						// insert the missing span
+						missingSpan := model.Span{
+							SpanID:           reference.SpanId,
+							TraceID:          spanNode.TraceID,
+							ServiceName:      "",
+							Name:             "Missing Span",
+							TimeUnixNano:     spanNode.TimeUnixNano,
+							Kind:             0,
+							DurationNano:     spanNode.DurationNano,
+							HasError:         false,
+							StatusMessage:    "",
+							StatusCodeString: "",
+							SpanKind:         "",
+							Events:           make([]model.Event, 0),
+							Children:         make([]*model.Span, 0),
+						}
+						missingSpan.Children = append(missingSpan.Children, spanNode)
+						spanIdToSpanNodeMap[missingSpan.SpanID] = &missingSpan
+						traceRoots = append(traceRoots, &missingSpan)
+						hasMissingSpans = true
 					}
-					missingSpan.Children = append(missingSpan.Children, spanNode)
-					spanIdToSpanNodeMap[missingSpan.SpanID] = &missingSpan
-					traceRoots = append(traceRoots, &missingSpan)
-					hasMissingSpans = true
 				}
 			}
+			if !hasParentSpanNode && !tracedetail.ContainsWaterfallSpan(traceRoots, spanNode) {
+				traceRoots = append(traceRoots, spanNode)
+			}
 		}
-		if !hasParentSpanNode && !tracedetail.ContainsWaterfallSpan(traceRoots, spanNode) {
-			traceRoots = append(traceRoots, spanNode)
+
+		// sort the trace roots to add missing spans at the right order
+		sort.Slice(traceRoots, func(i, j int) bool {
+			if traceRoots[i].TimeUnixNano == traceRoots[j].TimeUnixNano {
+				return traceRoots[i].Name < traceRoots[j].Name
+			}
+			return traceRoots[i].TimeUnixNano < traceRoots[j].TimeUnixNano
+		})
+
+		serviceNameToTotalDurationMap = tracedetail.CalculateServiceTime(serviceNameIntervalMap)
+
+		traceCache := model.GetWaterfallSpansForTraceWithMetadataCache{
+			StartTime:                     startTime,
+			EndTime:                       endTime,
+			DurationNano:                  durationNano,
+			TotalSpans:                    totalSpans,
+			TotalErrorSpans:               totalErrorSpans,
+			SpanIdToSpanNodeMap:           spanIdToSpanNodeMap,
+			ServiceNameToTotalDurationMap: serviceNameToTotalDurationMap,
+			TraceRoots:                    traceRoots,
+			HasMissingSpans:               hasMissingSpans,
+		}
+
+		r.logger.Info("getWaterfallSpansForTraceWithMetadata: processing pre cache", "duration", time.Since(processingBeforeCache), "traceID", traceID)
+		cacheErr := r.cacheForTraceDetail.Set(ctx, orgID, strings.Join([]string{"getWaterfallSpansForTraceWithMetadata", traceID}, "-"), &traceCache, time.Minute*5)
+		if cacheErr != nil {
+			r.logger.Debug("failed to store cache for getWaterfallSpansForTraceWithMetadata", "traceID", traceID, errorsV2.Attr(err))
 		}
 	}
 
-	// sort the trace roots to add missing spans at the right order
-	sort.Slice(traceRoots, func(i, j int) bool {
-		if traceRoots[i].TimeUnixNano == traceRoots[j].TimeUnixNano {
-			return traceRoots[i].Name < traceRoots[j].Name
-		}
-		return traceRoots[i].TimeUnixNano < traceRoots[j].TimeUnixNano
-	})
-
-	serviceNameToTotalDurationMap = tracedetail.CalculateServiceTime(serviceNameIntervalMap)
-
 	processingPostCache := time.Now()
 	// When req.Limit is 0 (not set by the client),  selectAllSpans is set to false
 	// preserving the old paged behaviour for backward compatibility
@@ -1071,6 +1130,23 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
 	return response, nil
 }
 
+func (r *ClickHouseReader) GetFlamegraphSpansForTraceCache(ctx context.Context, orgID valuer.UUID, traceID string) (*model.GetFlamegraphSpansForTraceCache, error) {
+	cachedTraceData := new(model.GetFlamegraphSpansForTraceCache)
+	err := r.cacheForTraceDetail.Get(ctx, orgID, strings.Join([]string{"getFlamegraphSpansForTrace", traceID}, "-"), cachedTraceData)
+	if err != nil {
+		r.logger.Debug("error in retrieving getFlamegraphSpansForTrace cache", errorsV2.Attr(err), "traceID", traceID)
+		return nil, err
+	}
+
+	if time.Since(time.UnixMilli(int64(cachedTraceData.EndTime))) < r.fluxIntervalForTraceDetail {
+		r.logger.Info("the trace end time falls under the flux interval, skipping getFlamegraphSpansForTrace cache", "traceID", traceID)
+		return nil, errors.Errorf("the trace end time falls under the flux interval, skipping getFlamegraphSpansForTrace cache, traceID: %s", traceID)
+	}
+
+	r.logger.Info("cache is successfully hit, applying cache for getFlamegraphSpansForTrace", "traceID", traceID)
+	return cachedTraceData, nil
+}
+
 func (r *ClickHouseReader) GetFlamegraphSpansForTrace(ctx context.Context, orgID valuer.UUID, traceID string, req *model.GetFlamegraphSpansForTraceParams) (*model.GetFlamegraphSpansForTraceResponse, error) {
 	trace := new(model.GetFlamegraphSpansForTraceResponse)
 	var startTime, endTime, durationNano uint64
@@ -1079,96 +1155,125 @@ func (r *ClickHouseReader) GetFlamegraphSpansForTrace(ctx context.Context, orgID
 	var selectedSpans = [][]*model.FlamegraphSpan{}
 	var traceRoots []*model.FlamegraphSpan
 
-	searchScanResponses, err := r.GetSpansForTrace(ctx, traceID, fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error,links as references, resource_string_service$$name, name, events FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3 ORDER BY timestamp ASC, name ASC", r.TraceDB, r.traceTableName))
+	// get the trace tree from cache!
+	cachedTraceData, err := r.GetFlamegraphSpansForTraceCache(ctx, orgID, traceID)
+
+	if err == nil {
+		startTime = cachedTraceData.StartTime
+		endTime = cachedTraceData.EndTime
+		durationNano = cachedTraceData.DurationNano
+		selectedSpans = cachedTraceData.SelectedSpans
+		traceRoots = cachedTraceData.TraceRoots
+	}
+
 	if err != nil {
-		return nil, err
-	}
-	if len(searchScanResponses) == 0 {
-		return trace, nil
-	}
+		r.logger.Info("cache miss for getFlamegraphSpansForTrace", "traceID", traceID)
 
-	for _, item := range searchScanResponses {
-		ref := []model.OtelSpanRef{}
-		err := json.Unmarshal([]byte(item.References), &ref)
+		searchScanResponses, err := r.GetSpansForTrace(ctx, traceID, fmt.Sprintf("SELECT timestamp, duration_nano, span_id, trace_id, has_error,links as references, resource_string_service$$name, name, events FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3 ORDER BY timestamp ASC, name ASC", r.TraceDB, r.traceTableName))
 		if err != nil {
-			r.logger.Error("Error unmarshalling references", errorsV2.Attr(err))
-			return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "getFlamegraphSpansForTrace: error in unmarshalling references %s", err.Error())
+			return nil, err
+		}
+		if len(searchScanResponses) == 0 {
+			return trace, nil
 		}
 
-		events := make([]model.Event, 0)
-		for _, event := range item.Events {
-			var eventMap model.Event
-			err = json.Unmarshal([]byte(event), &eventMap)
+		processingBeforeCache := time.Now()
+		for _, item := range searchScanResponses {
+			ref := []model.OtelSpanRef{}
+			err := json.Unmarshal([]byte(item.References), &ref)
 			if err != nil {
-				r.logger.Error("Error unmarshalling events", errorsV2.Attr(err))
-				return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "getFlamegraphSpansForTrace: error in unmarshalling events %s", err.Error())
+				r.logger.Error("Error unmarshalling references", errorsV2.Attr(err))
+				return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "getFlamegraphSpansForTrace: error in unmarshalling references %s", err.Error())
 			}
-			events = append(events, eventMap)
+
+			events := make([]model.Event, 0)
+			for _, event := range item.Events {
+				var eventMap model.Event
+				err = json.Unmarshal([]byte(event), &eventMap)
+				if err != nil {
+					r.logger.Error("Error unmarshalling events", errorsV2.Attr(err))
+					return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "getFlamegraphSpansForTrace: error in unmarshalling events %s", err.Error())
+				}
+				events = append(events, eventMap)
+			}
+
+			jsonItem := model.FlamegraphSpan{
+				SpanID:       item.SpanID,
+				TraceID:      item.TraceID,
+				ServiceName:  item.ServiceName,
+				Name:         item.Name,
+				DurationNano: item.DurationNano,
+				HasError:     item.HasError,
+				References:   ref,
+				Events:       events,
+				Children:     make([]*model.FlamegraphSpan, 0),
+			}
+
+			// metadata calculation
+			startTimeUnixNano := uint64(item.TimeUnixNano.UnixNano())
+			if startTime == 0 || startTimeUnixNano < startTime {
+				startTime = startTimeUnixNano
+			}
+			if endTime == 0 || (startTimeUnixNano+jsonItem.DurationNano) > endTime {
+				endTime = (startTimeUnixNano + jsonItem.DurationNano)
+			}
+			if durationNano == 0 || jsonItem.DurationNano > durationNano {
+				durationNano = jsonItem.DurationNano
+			}
+
+			jsonItem.TimeUnixNano = uint64(item.TimeUnixNano.UnixNano() / 1000000)
+			spanIdToSpanNodeMap[jsonItem.SpanID] = &jsonItem
 		}
 
-		jsonItem := model.FlamegraphSpan{
-			SpanID:       item.SpanID,
-			TraceID:      item.TraceID,
-			ServiceName:  item.ServiceName,
-			Name:         item.Name,
-			DurationNano: item.DurationNano,
-			HasError:     item.HasError,
-			References:   ref,
-			Events:       events,
-			Children:     make([]*model.FlamegraphSpan, 0),
-		}
-
-		// metadata calculation
-		startTimeUnixNano := uint64(item.TimeUnixNano.UnixNano())
-		if startTime == 0 || startTimeUnixNano < startTime {
-			startTime = startTimeUnixNano
-		}
-		if endTime == 0 || (startTimeUnixNano+jsonItem.DurationNano) > endTime {
-			endTime = (startTimeUnixNano + jsonItem.DurationNano)
-		}
-		if durationNano == 0 || jsonItem.DurationNano > durationNano {
-			durationNano = jsonItem.DurationNano
-		}
-
-		jsonItem.TimeUnixNano = uint64(item.TimeUnixNano.UnixNano() / 1000000)
-		spanIdToSpanNodeMap[jsonItem.SpanID] = &jsonItem
-	}
-
-	// traverse through the map and append each node to the children array of the parent node
-	// and add missing spans
-	for _, spanNode := range spanIdToSpanNodeMap {
-		hasParentSpanNode := false
-		for _, reference := range spanNode.References {
-			if reference.RefType == "CHILD_OF" && reference.SpanId != "" {
-				hasParentSpanNode = true
-				if parentNode, exists := spanIdToSpanNodeMap[reference.SpanId]; exists {
-					parentNode.Children = append(parentNode.Children, spanNode)
-				} else {
-					// insert the missing spans
-					missingSpan := model.FlamegraphSpan{
-						SpanID:       reference.SpanId,
-						TraceID:      spanNode.TraceID,
-						ServiceName:  "",
-						Name:         "Missing Span",
-						TimeUnixNano: spanNode.TimeUnixNano,
-						DurationNano: spanNode.DurationNano,
-						HasError:     false,
-						Events:       make([]model.Event, 0),
-						Children:     make([]*model.FlamegraphSpan, 0),
+		// traverse through the map and append each node to the children array of the parent node
+		// and add missing spans
+		for _, spanNode := range spanIdToSpanNodeMap {
+			hasParentSpanNode := false
+			for _, reference := range spanNode.References {
+				if reference.RefType == "CHILD_OF" && reference.SpanId != "" {
+					hasParentSpanNode = true
+					if parentNode, exists := spanIdToSpanNodeMap[reference.SpanId]; exists {
+						parentNode.Children = append(parentNode.Children, spanNode)
+					} else {
+						// insert the missing spans
+						missingSpan := model.FlamegraphSpan{
+							SpanID:       reference.SpanId,
+							TraceID:      spanNode.TraceID,
+							ServiceName:  "",
+							Name:         "Missing Span",
+							TimeUnixNano: spanNode.TimeUnixNano,
+							DurationNano: spanNode.DurationNano,
+							HasError:     false,
+							Events:       make([]model.Event, 0),
+							Children:     make([]*model.FlamegraphSpan, 0),
+						}
+						missingSpan.Children = append(missingSpan.Children, spanNode)
+						spanIdToSpanNodeMap[missingSpan.SpanID] = &missingSpan
+						traceRoots = append(traceRoots, &missingSpan)
 					}
-					missingSpan.Children = append(missingSpan.Children, spanNode)
-					spanIdToSpanNodeMap[missingSpan.SpanID] = &missingSpan
-					traceRoots = append(traceRoots, &missingSpan)
 				}
 			}
+			if !hasParentSpanNode && !tracedetail.ContainsFlamegraphSpan(traceRoots, spanNode) {
+				traceRoots = append(traceRoots, spanNode)
+			}
 		}
-		if !hasParentSpanNode && !tracedetail.ContainsFlamegraphSpan(traceRoots, spanNode) {
-			traceRoots = append(traceRoots, spanNode)
+
+		selectedSpans = tracedetail.GetAllSpansForFlamegraph(traceRoots, spanIdToSpanNodeMap)
+		traceCache := model.GetFlamegraphSpansForTraceCache{
+			StartTime:     startTime,
+			EndTime:       endTime,
+			DurationNano:  durationNano,
+			SelectedSpans: selectedSpans,
+			TraceRoots:    traceRoots,
+		}
+
+		r.logger.Info("getFlamegraphSpansForTrace: processing pre cache", "duration", time.Since(processingBeforeCache), "traceID", traceID)
+		cacheErr := r.cacheForTraceDetail.Set(ctx, orgID, strings.Join([]string{"getFlamegraphSpansForTrace", traceID}, "-"), &traceCache, time.Minute*5)
+		if cacheErr != nil {
+			r.logger.Debug("failed to store cache for getFlamegraphSpansForTrace", "traceID", traceID, errorsV2.Attr(err))
 		}
 	}
 
-	selectedSpans = tracedetail.GetAllSpansForFlamegraph(traceRoots, spanIdToSpanNodeMap)
-
 	processingPostCache := time.Now()
 	selectedSpansForRequest := selectedSpans
 	clientLimit := min(req.Limit, tracedetail.MaxLimitWithoutSampling)

pkg/query-service/app/server.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"slices"
 
+	"github.com/SigNoz/signoz/pkg/cache/memorycache"
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/queryparser"
@@ -78,12 +79,25 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 		return nil, err
 	}
 
+	cacheForTraceDetail, err := memorycache.New(context.TODO(), signoz.Instrumentation.ToProviderSettings(), cache.Config{
+		Provider: "memory",
+		Memory: cache.Memory{
+			NumCounters: 10 * 10000,
+			MaxCost:     1 << 27, // 128 MB
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	reader := clickhouseReader.NewReader(
 		signoz.Instrumentation.Logger(),
 		signoz.SQLStore,
 		signoz.TelemetryStore,
 		signoz.Prometheus,
 		signoz.TelemetryStore.Cluster(),
+		config.Querier.FluxInterval,
+		cacheForTraceDetail,
 		signoz.Cache,
 		nil,
 	)

pkg/query-service/model/cacheable.go

@@ -1 +1,74 @@
 package model
+
+import (
+	"encoding/json"
+	"maps"
+
+	"github.com/SigNoz/signoz/pkg/types/cachetypes"
+)
+
+type GetWaterfallSpansForTraceWithMetadataCache struct {
+	StartTime                     uint64            `json:"startTime"`
+	EndTime                       uint64            `json:"endTime"`
+	DurationNano                  uint64            `json:"durationNano"`
+	TotalSpans                    uint64            `json:"totalSpans"`
+	TotalErrorSpans               uint64            `json:"totalErrorSpans"`
+	ServiceNameToTotalDurationMap map[string]uint64 `json:"serviceNameToTotalDurationMap"`
+	SpanIdToSpanNodeMap           map[string]*Span  `json:"spanIdToSpanNodeMap"`
+	TraceRoots                    []*Span           `json:"traceRoots"`
+	HasMissingSpans               bool              `json:"hasMissingSpans"`
+}
+
+func (c *GetWaterfallSpansForTraceWithMetadataCache) Clone() cachetypes.Cacheable {
+	copyOfServiceNameToTotalDurationMap := make(map[string]uint64)
+	maps.Copy(copyOfServiceNameToTotalDurationMap, c.ServiceNameToTotalDurationMap)
+
+	copyOfSpanIdToSpanNodeMap := make(map[string]*Span)
+	maps.Copy(copyOfSpanIdToSpanNodeMap, c.SpanIdToSpanNodeMap)
+
+	copyOfTraceRoots := make([]*Span, len(c.TraceRoots))
+	copy(copyOfTraceRoots, c.TraceRoots)
+	return &GetWaterfallSpansForTraceWithMetadataCache{
+		StartTime:                     c.StartTime,
+		EndTime:                       c.EndTime,
+		DurationNano:                  c.DurationNano,
+		TotalSpans:                    c.TotalSpans,
+		TotalErrorSpans:               c.TotalErrorSpans,
+		ServiceNameToTotalDurationMap: copyOfServiceNameToTotalDurationMap,
+		SpanIdToSpanNodeMap:           copyOfSpanIdToSpanNodeMap,
+		TraceRoots:                    copyOfTraceRoots,
+		HasMissingSpans:               c.HasMissingSpans,
+	}
+}
+
+func (c *GetWaterfallSpansForTraceWithMetadataCache) MarshalBinary() (data []byte, err error) {
+	return json.Marshal(c)
+}
+func (c *GetWaterfallSpansForTraceWithMetadataCache) UnmarshalBinary(data []byte) error {
+	return json.Unmarshal(data, c)
+}
+
+type GetFlamegraphSpansForTraceCache struct {
+	StartTime     uint64              `json:"startTime"`
+	EndTime       uint64              `json:"endTime"`
+	DurationNano  uint64              `json:"durationNano"`
+	SelectedSpans [][]*FlamegraphSpan `json:"selectedSpans"`
+	TraceRoots    []*FlamegraphSpan   `json:"traceRoots"`
+}
+
+func (c *GetFlamegraphSpansForTraceCache) Clone() cachetypes.Cacheable {
+	return &GetFlamegraphSpansForTraceCache{
+		StartTime:     c.StartTime,
+		EndTime:       c.EndTime,
+		DurationNano:  c.DurationNano,
+		SelectedSpans: c.SelectedSpans,
+		TraceRoots:    c.TraceRoots,
+	}
+}
+
+func (c *GetFlamegraphSpansForTraceCache) MarshalBinary() (data []byte, err error) {
+	return json.Marshal(c)
+}
+func (c *GetFlamegraphSpansForTraceCache) UnmarshalBinary(data []byte) error {
+	return json.Unmarshal(data, c)
+}

pkg/types/alertmanagertypes/template.go

@@ -11,19 +11,22 @@ import (
 	alertmanagertemplate "github.com/prometheus/alertmanager/template"
 )
 
+func AdditionalFuncMap() tmpltext.FuncMap {
+	return tmpltext.FuncMap{
+		// urlescape escapes the string for use in a URL query parameter.
+		// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
+		// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
+		"urlescape": func(value string) tmplhtml.HTML {
+			return tmplhtml.HTML(url.QueryEscape(value))
+		},
+	}
+}
+
 // customTemplateOption returns an Option that adds custom functions to the template.
 func customTemplateOption() alertmanagertemplate.Option {
 	return func(text *tmpltext.Template, html *tmplhtml.Template) {
-		funcs := tmpltext.FuncMap{
-			// urlescape escapes the string for use in a URL query parameter.
-			// It returns tmplhtml.HTML to prevent the template engine from escaping the already escaped string.
-			// url.QueryEscape escapes spaces as "+", and html/template escapes "+" as "+" if tmplhtml.HTML is not used.
-			"urlescape": func(value string) tmplhtml.HTML {
-				return tmplhtml.HTML(url.QueryEscape(value))
-			},
-		}
-		text.Funcs(funcs)
-		html.Funcs(funcs)
+		text.Funcs(AdditionalFuncMap())
+		html.Funcs(AdditionalFuncMap())
 	}
 }
 

pkg/types/alertmanagertypes/templatedata.go

@@ -0,0 +1,144 @@
+package alertmanagertypes
+
+import (
+	"strings"
+	"time"
+
+	"github.com/prometheus/alertmanager/template"
+)
+
+// privateAnnotationPrefix marks annotations the rule manager attaches for
+// alertmanager-internal use (raw template strings, threshold metadata, link
+// targets). Annotations whose key starts with this prefix are stripped from
+// any surface that ends up visible to a template author or a notification
+// recipient; the alertmanager reads them off the raw alert before stripping.
+const privateAnnotationPrefix = "_"
+
+// IsPrivateAnnotation reports whether an annotation key is considered
+// private — i.e. internal to alertmanager and should not be rendered in
+// notifications.
+func IsPrivateAnnotation(key string) bool {
+	return strings.HasPrefix(key, privateAnnotationPrefix)
+}
+
+// FilterPublicAnnotations returns a copy of kv with all private-prefixed
+// keys removed. Callers that expose annotations to templates or notification
+// payloads should pass them through this first.
+func FilterPublicAnnotations(kv template.KV) template.KV {
+	out := make(template.KV, len(kv))
+	for k, v := range kv {
+		if IsPrivateAnnotation(k) {
+			continue
+		}
+		out[k] = v
+	}
+	return out
+}
+
+// ExpandRequest carries the title/body templates and their defaults handed to
+// Templater.Expand. Default templates are used when the custom templates
+// expand to empty strings.
+type ExpandRequest struct {
+	TitleTemplate        string
+	BodyTemplate         string
+	DefaultTitleTemplate string
+	DefaultBodyTemplate  string
+}
+
+// ExpandResult is the rendered output of Templater.Expand.
+type ExpandResult struct {
+	// Title is the expanded notification title (plain text).
+	Title string
+	// Body is the expanded notification body, one entry per input alert. The
+	// body template is applied per-alert and concatenated by the caller.
+	Body []string
+	// IsDefaultBody is true when Body came from the default template (no
+	// user-authored body was supplied), false when a custom template was used.
+	IsDefaultBody bool
+	// MissingVars is the union of $-references in the title and body templates
+	// that did not resolve to any known field. Surfaced for preview warnings;
+	// at runtime these render as "<no value>".
+	MissingVars []string
+	// NotificationData is the aggregate data that fed the title template,
+	// exposed so callers can reuse it when rendering a channel-specific layout
+	// (e.g. the email HTML shell) without rebuilding it from the alerts.
+	NotificationData *NotificationTemplateData
+}
+
+// AlertData holds per-alert data used when expanding body templates.
+//
+// Field paths follow OpenTelemetry-style dotted namespaces (via mapstructure
+// tags) so user templates can reference paths like $alert.is_firing,
+// $rule.threshold.value, or $log.url. JSON tags use camelCase for the wire
+// format.
+type AlertData struct {
+	Alert       AlertInfo   `json:"alert"       mapstructure:"alert"`
+	Rule        RuleInfo    `json:"rule"        mapstructure:"rule"`
+	Log         LinkInfo    `json:"log"         mapstructure:"log"`
+	Trace       LinkInfo    `json:"trace"       mapstructure:"trace"`
+	Labels      template.KV `json:"labels"      mapstructure:"labels"`
+	Annotations template.KV `json:"annotations" mapstructure:"annotations"`
+}
+
+// AlertInfo holds the per-alert state and timing data.
+type AlertInfo struct {
+	Status        string    `json:"status"        mapstructure:"status"`
+	Receiver      string    `json:"receiver"      mapstructure:"receiver"`
+	Value         string    `json:"value"         mapstructure:"value"`
+	StartsAt      time.Time `json:"startsAt"      mapstructure:"starts_at"`
+	EndsAt        time.Time `json:"endsAt"        mapstructure:"ends_at"`
+	GeneratorURL  string    `json:"generatorURL"  mapstructure:"generator_url"`
+	Fingerprint   string    `json:"fingerprint"   mapstructure:"fingerprint"`
+	IsFiring      bool      `json:"isFiring"      mapstructure:"is_firing"`
+	IsResolved    bool      `json:"isResolved"    mapstructure:"is_resolved"`
+	IsMissingData bool      `json:"isMissingData" mapstructure:"is_missing_data"`
+	IsRecovering  bool      `json:"isRecovering"  mapstructure:"is_recovering"`
+}
+
+// RuleInfo holds the rule metadata extracted from well-known labels and
+// annotations.
+type RuleInfo struct {
+	Name      string    `json:"name"      mapstructure:"name"`
+	ID        string    `json:"id"        mapstructure:"id"`
+	URL       string    `json:"url"       mapstructure:"url"`
+	Severity  string    `json:"severity"  mapstructure:"severity"`
+	MatchType string    `json:"matchType" mapstructure:"match_type"`
+	Threshold Threshold `json:"threshold" mapstructure:"threshold"`
+}
+
+// Threshold describes the breach condition.
+type Threshold struct {
+	Value string `json:"value" mapstructure:"value"`
+	Op    string `json:"op"    mapstructure:"op"`
+}
+
+// LinkInfo groups a single URL so templates can reference $log.url and
+// $trace.url uniformly.
+type LinkInfo struct {
+	URL string `json:"url" mapstructure:"url"`
+}
+
+// NotificationTemplateData is the top-level data struct provided to title
+// templates, representing the aggregate of a grouped notification.
+type NotificationTemplateData struct {
+	Alert             NotificationAlert `json:"alert"             mapstructure:"alert"`
+	Rule              RuleInfo          `json:"rule"              mapstructure:"rule"`
+	Labels            template.KV       `json:"labels"            mapstructure:"labels"`
+	Annotations       template.KV       `json:"annotations"       mapstructure:"annotations"`
+	CommonLabels      template.KV       `json:"commonLabels"      mapstructure:"common_labels"`
+	CommonAnnotations template.KV       `json:"commonAnnotations" mapstructure:"common_annotations"`
+	GroupLabels       template.KV       `json:"groupLabels"       mapstructure:"group_labels"`
+	ExternalURL       string            `json:"externalURL"       mapstructure:"external_url"`
+
+	// Per-alert data kept for body expansion; not exposed to the title template.
+	Alerts []AlertData `json:"-" mapstructure:"-"`
+}
+
+// NotificationAlert holds the aggregate alert fields available to title
+// templates (counts, overall status, receiver).
+type NotificationAlert struct {
+	Receiver      string `json:"receiver"      mapstructure:"receiver"`
+	Status        string `json:"status"        mapstructure:"status"`
+	TotalFiring   int    `json:"totalFiring"   mapstructure:"total_firing"`
+	TotalResolved int    `json:"totalResolved" mapstructure:"total_resolved"`
+}

pkg/types/ruletypes/constants.go

@@ -9,4 +9,33 @@ const (
 	LabelSeverityName     = "severity"
 	LabelLastSeen         = "lastSeen"
 	LabelRuleID           = "ruleId"
+	LabelRuleSource       = "ruleSource"
+	LabelNoData           = "nodata"
+	LabelTestAlert        = "testalert"
+	LabelAlertName        = "alertname"
+	LabelIsRecovering     = "is_recovering"
+)
+
+// Annotations set by the rule manager and consumed by the alertmanager
+// templating layer.
+//
+// Those prefixed with "_" are private: they're stripped from
+// notification-visible surfaces by alertmanagertypes.FilterPublicAnnotations
+// before rendering. Only the raw template strings are private — echoing
+// them into a notification is circular and never useful.
+//
+// The rest are public: they describe the firing alert (the breached value,
+// the configured threshold, the comparator, the match type, and deep links
+// to relevant logs/traces) and users may reference them directly as
+// {{ .Annotations.value }}, {{ .Annotations.threshold.value }}, etc. in
+// their channel templates.
+const (
+	AnnotationTitleTemplate  = "_title_template"
+	AnnotationBodyTemplate   = "_body_template"
+	AnnotationRelatedLogs    = "related_logs"
+	AnnotationRelatedTraces  = "related_traces"
+	AnnotationValue          = "value"
+	AnnotationThresholdValue = "threshold.value"
+	AnnotationCompareOp      = "compare_op"
+	AnnotationMatchType      = "match_type"
 )