Merge remote-tracking branch 'origin/main' into chore/trace-waterfall-cleanup

* chore: add tests for trace waterfall

* chore: remove unhelpful tests

conf/example.yaml

@@ -144,6 +144,8 @@ telemetrystore:
 
 ##################### Prometheus #####################
 prometheus:
+  # The maximum time a PromQL query is allowed to run before being aborted.
+  timeout: 2m
   active_query_tracker:
     # Whether to enable the active query tracker.
     enabled: true

deploy/docker-swarm/docker-compose.ha.yaml

@@ -190,7 +190,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:v0.116.1
+    image: signoz/signoz:v0.117.0
     ports:
       - "8080:8080" # signoz port
     #   - "6060:6060"     # pprof port

deploy/docker-swarm/docker-compose.yaml

@@ -117,7 +117,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:v0.116.1
+    image: signoz/signoz:v0.117.0
     ports:
       - "8080:8080" # signoz port
     volumes:

deploy/docker/docker-compose.ha.yaml

@@ -181,7 +181,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:${VERSION:-v0.116.1}
+    image: signoz/signoz:${VERSION:-v0.117.0}
     container_name: signoz
     ports:
       - "8080:8080" # signoz port

deploy/docker/docker-compose.yaml

@@ -109,7 +109,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:${VERSION:-v0.116.1}
+    image: signoz/signoz:${VERSION:-v0.117.0}
     container_name: signoz
     ports:
       - "8080:8080" # signoz port

ee/query-service/app/server.go

@@ -136,6 +136,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 	logParsingPipelineController, err := logparsingpipeline.NewLogParsingPipelinesController(
 		signoz.SQLStore,
 		integrationsController.GetPipelinesForInstalledIntegrations,
+		reader,
 	)
 	if err != nil {
 		return nil, err

ee/query-service/rules/manager_test.go

@@ -257,7 +257,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 						WillReturnRows(samplesRows)
 
 					// Create Prometheus provider for this test
-					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, store)
+					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, store)
 				},
 				ManagerOptionsHook: func(opts *rules.ManagerOptions) {
 					// Set Prometheus provider for PromQL queries

pkg/prometheus/config.go

@@ -3,6 +3,7 @@ package prometheus
 import (
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
 )
 
@@ -20,6 +21,9 @@ type Config struct {
 	//
 	// If not set, the prometheus default is used (currently 5m).
 	LookbackDelta time.Duration `mapstructure:"lookback_delta"`
+
+	// Timeout is the maximum time a query is allowed to run before being aborted.
+	Timeout time.Duration `mapstructure:"timeout"`
 }
 
 func NewConfigFactory() factory.ConfigFactory {
@@ -33,10 +37,14 @@ func newConfig() factory.Config {
 			Path:          "",
 			MaxConcurrent: 20,
 		},
+		Timeout: 2 * time.Minute,
 	}
 }
 
 func (c Config) Validate() error {
+	if c.Timeout <= 0 {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
+	}
 	return nil
 }
 

pkg/prometheus/constants.go

@@ -1,3 +0,0 @@
-package prometheus
-
-const FingerprintAsPromLabelName = "fingerprint"

pkg/prometheus/engine.go

@@ -2,7 +2,6 @@ package prometheus
 
 import (
 	"log/slog"
-	"time"
 
 	"github.com/prometheus/prometheus/promql"
 )
@@ -21,7 +20,7 @@ func NewEngine(logger *slog.Logger, cfg Config) *Engine {
 		Logger:             logger,
 		Reg:                nil,
 		MaxSamples:         5_0000_000,
-		Timeout:            2 * time.Minute,
+		Timeout:            cfg.Timeout,
 		ActiveQueryTracker: activeQueryTracker,
 		LookbackDelta:      cfg.LookbackDelta,
 	})

pkg/prometheus/label.go

@@ -6,6 +6,8 @@ import (
 	"github.com/prometheus/prometheus/promql"
 )
 
+const FingerprintAsPromLabelName string = "fingerprint"
+
 func RemoveExtraLabels(res *promql.Result, labelsToRemove ...string) error {
 	if len(labelsToRemove) == 0 || res == nil {
 		return nil

pkg/querier/promql_query.go

@@ -36,6 +36,28 @@ var unquotedDottedNamePattern = regexp.MustCompile(`(?:^|[{,(\s])([a-zA-Z_][a-zA
 // This is a common mistake when migrating to UTF-8 syntax.
 var quotedMetricOutsideBracesPattern = regexp.MustCompile(`"([^"]+)"\s*\{`)
 
+// tryEnhancePromQLExecError attempts to convert a PromQL execution error into
+// a properly typed error. Returns nil if the error is not a recognized execution error.
+func tryEnhancePromQLExecError(execErr error) error {
+	var eqc promql.ErrQueryCanceled
+	var eqt promql.ErrQueryTimeout
+	var es promql.ErrStorage
+	switch {
+	case errors.As(execErr, &eqc):
+		return errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled").WithAdditional(eqc.Error())
+	case errors.As(execErr, &eqt):
+		return errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timed out").WithAdditional(eqt.Error())
+	case errors.Is(execErr, context.DeadlineExceeded):
+		return errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timed out")
+	case errors.Is(execErr, context.Canceled):
+		return errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
+	case errors.As(execErr, &es):
+		return errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", execErr)
+	default:
+		return nil
+	}
+}
+
 // enhancePromQLError adds helpful context to PromQL parse errors,
 // particularly for UTF-8 syntax migration issues where metric and label
 // names containing dots need to be quoted.
@@ -213,27 +235,20 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		time.Unix(0, end),
 		q.query.Step.Duration,
 	)
-
 	if err != nil {
+		// NewRangeQuery can fail with execution errors (e.g. context deadline exceeded)
+		// during the query queue/scheduling stage, not just parse errors.
+		if err := tryEnhancePromQLExecError(err); err != nil {
+			return nil, err
+		}
+
 		return nil, enhancePromQLError(query, err)
 	}
 
 	res := qry.Exec(ctx)
 	if res.Err != nil {
-		var eqc promql.ErrQueryCanceled
-		var eqt promql.ErrQueryTimeout
-		var es promql.ErrStorage
-		switch {
-		case errors.As(res.Err, &eqc):
-			return nil, errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
-		case errors.As(res.Err, &eqt):
-			return nil, errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timeout")
-		case errors.As(res.Err, &es):
-			return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err)
-		}
-
-		if errors.Is(res.Err, context.Canceled) {
-			return nil, errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
+		if err := tryEnhancePromQLExecError(res.Err); err != nil {
+			return nil, err
 		}
 
 		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err)

pkg/query-service/app/logparsingpipeline/controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"slices"
 	"strings"
 
@@ -12,6 +13,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
 	"github.com/SigNoz/signoz/pkg/query-service/constants"
+	"github.com/SigNoz/signoz/pkg/query-service/interfaces"
 	"github.com/SigNoz/signoz/pkg/query-service/model"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
 	"github.com/SigNoz/signoz/pkg/query-service/utils"
@@ -34,19 +36,101 @@ type LogParsingPipelineController struct {
 	Repo
 
 	GetIntegrationPipelines func(context.Context, string) ([]pipelinetypes.GettablePipeline, error)
+	// TODO(Piyush): remove with qbv5 migration
+	reader interfaces.Reader
 }
 
 func NewLogParsingPipelinesController(
 	sqlStore sqlstore.SQLStore,
 	getIntegrationPipelines func(context.Context, string) ([]pipelinetypes.GettablePipeline, error),
+	reader interfaces.Reader,
 ) (*LogParsingPipelineController, error) {
 	repo := NewRepo(sqlStore)
 	return &LogParsingPipelineController{
 		Repo:                    repo,
 		GetIntegrationPipelines: getIntegrationPipelines,
+		reader:                  reader,
 	}, nil
 }
 
+// enrichPipelinesFilters resolves the type (tag vs resource) for filter keys that are
+// missing type info, by looking them up in the store.
+//
+// TODO(Piyush): remove with qbv5 migration
+func (pc *LogParsingPipelineController) enrichPipelinesFilters(
+	ctx context.Context, pipelines []pipelinetypes.GettablePipeline,
+) ([]pipelinetypes.GettablePipeline, error) {
+	// Collect names of non-static keys that are missing type info.
+	// Static fields (body, trace_id, etc.) are intentionally Unspecified and map
+	// to top-level OTEL fields — they do not need enrichment.
+	unspecifiedNames := map[string]struct{}{}
+	for _, p := range pipelines {
+		if p.Filter != nil {
+			for _, item := range p.Filter.Items {
+				if item.Key.Type == v3.AttributeKeyTypeUnspecified {
+					// Skip static fields
+					if _, isStatic := constants.StaticFieldsLogsV3[item.Key.Key]; isStatic {
+						continue
+					}
+					// Skip enrich body.* fields
+					if strings.HasPrefix(item.Key.Key, "body.") {
+						continue
+					}
+					unspecifiedNames[item.Key.Key] = struct{}{}
+				}
+			}
+		}
+	}
+	if len(unspecifiedNames) == 0 {
+		return pipelines, nil
+	}
+
+	logFields, apiErr := pc.reader.GetLogFieldsFromNames(ctx, slices.Collect(maps.Keys(unspecifiedNames)))
+	if apiErr != nil {
+		slog.ErrorContext(ctx, "failed to fetch log fields for pipeline filter enrichment", "error", apiErr)
+		return pipelines, apiErr
+	}
+
+	// Build a simple name → AttributeKeyType map from the response.
+	fieldTypes := map[string]v3.AttributeKeyType{}
+	for _, f := range append(logFields.Selected, logFields.Interesting...) {
+		switch f.Type {
+		case constants.Resources:
+			fieldTypes[f.Name] = v3.AttributeKeyTypeResource
+		case constants.Attributes:
+			fieldTypes[f.Name] = v3.AttributeKeyTypeTag
+		}
+	}
+
+	// Set the resolved type on each untyped filter key in-place.
+	for i := range pipelines {
+		if pipelines[i].Filter != nil {
+			for j := range pipelines[i].Filter.Items {
+				key := &pipelines[i].Filter.Items[j].Key
+				if key.Type == v3.AttributeKeyTypeUnspecified {
+					// Skip static fields
+					if _, isStatic := constants.StaticFieldsLogsV3[key.Key]; isStatic {
+						continue
+					}
+					// Skip enrich body.* fields
+					if strings.HasPrefix(key.Key, "body.") {
+						continue
+					}
+
+					if t, ok := fieldTypes[key.Key]; ok {
+						key.Type = t
+					} else {
+						// default to attribute
+						key.Type = v3.AttributeKeyTypeTag
+					}
+				}
+			}
+		}
+	}
+
+	return pipelines, nil
+}
+
 // PipelinesResponse is used to prepare http response for pipelines config related requests
 type PipelinesResponse struct {
 	*opamptypes.AgentConfigVersion
@@ -256,7 +340,12 @@ func (ic *LogParsingPipelineController) PreviewLogsPipelines(
 	ctx context.Context,
 	request *PipelinesPreviewRequest,
 ) (*PipelinesPreviewResponse, error) {
-	result, collectorLogs, err := SimulatePipelinesProcessing(ctx, request.Pipelines, request.Logs)
+	pipelines, err := ic.enrichPipelinesFilters(ctx, request.Pipelines)
+	if err != nil {
+		return nil, err
+	}
+
+	result, collectorLogs, err := SimulatePipelinesProcessing(ctx, pipelines, request.Logs)
 	if err != nil {
 		return nil, err
 	}
@@ -293,10 +382,8 @@ func (pc *LogParsingPipelineController) RecommendAgentConfig(
 	if configVersion != nil {
 		pipelinesVersion = configVersion.Version
 	}
-
-	pipelinesResp, err := pc.GetPipelinesByVersion(
-		context.Background(), orgId, pipelinesVersion,
-	)
+	ctx := context.Background()
+	pipelinesResp, err := pc.GetPipelinesByVersion(ctx, orgId, pipelinesVersion)
 	if err != nil {
 		return nil, "", err
 	}
@@ -306,12 +393,17 @@ func (pc *LogParsingPipelineController) RecommendAgentConfig(
 		return nil, "", errors.WrapInternalf(err, CodeRawPipelinesMarshalFailed, "could not serialize pipelines to JSON")
 	}
 
-	if querybuilder.BodyJSONQueryEnabled {
-		// add default normalize pipeline at the beginning, only for sending to collector
-		pipelinesResp.Pipelines = append([]pipelinetypes.GettablePipeline{pc.getNormalizePipeline()}, pipelinesResp.Pipelines...)
+	enrichedPipelines, err := pc.enrichPipelinesFilters(ctx, pipelinesResp.Pipelines)
+	if err != nil {
+		return nil, "", err
 	}
 
-	updatedConf, err := GenerateCollectorConfigWithPipelines(currentConfYaml, pipelinesResp.Pipelines)
+	if querybuilder.BodyJSONQueryEnabled {
+		// add default normalize pipeline at the beginning, only for sending to collector
+		enrichedPipelines = append([]pipelinetypes.GettablePipeline{pc.getNormalizePipeline()}, enrichedPipelines...)
+	}
+
+	updatedConf, err := GenerateCollectorConfigWithPipelines(currentConfYaml, enrichedPipelines)
 	if err != nil {
 		return nil, "", err
 	}

pkg/query-service/app/querier/querier_test.go

@@ -1407,7 +1407,7 @@ func Test_querier_Traces_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1633,7 +1633,7 @@ func Test_querier_Traces_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1934,7 +1934,7 @@ func Test_querier_Logs_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -2162,7 +2162,7 @@ func Test_querier_Logs_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,

pkg/query-service/app/querier/v2/querier_test.go

@@ -1459,7 +1459,7 @@ func Test_querier_Traces_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1685,7 +1685,7 @@ func Test_querier_Traces_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1985,7 +1985,7 @@ func Test_querier_Logs_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -2213,7 +2213,7 @@ func Test_querier_Logs_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,

pkg/query-service/app/server.go

@@ -121,6 +121,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 	logParsingPipelineController, err := logparsingpipeline.NewLogParsingPipelinesController(
 		signoz.SQLStore,
 		integrationsController.GetPipelinesForInstalledIntegrations,
+		reader,
 	)
 	if err != nil {
 		return nil, err

pkg/query-service/app/traces/tracedetail/waterfall.go

@@ -1,6 +1,7 @@
 package tracedetail
 
 import (
+	"maps"
 	"slices"
 	"sort"
 
@@ -63,32 +64,36 @@ func findIndexForSelectedSpanFromPreOrder(spans []*model.Span, selectedSpanId st
 	return selectedSpanIndex
 }
 
-func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, uncollapsedSpans []string, isSelectedSpanIDUnCollapsed bool) (bool, []string) {
+func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string) (bool, []string) {
 	spansFromRootToNode := []string{}
 
+	spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
 	if node.SpanID == selectedSpanId {
-		if isSelectedSpanIDUnCollapsed {
-			spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
-		}
 		return true, spansFromRootToNode
 	}
 
 	isPresentInSubtreeForTheNode := false
 	for _, child := range node.Children {
-		isPresentInThisSubtree, _spansFromRootToNode := getPathFromRootToSelectedSpanId(child, selectedSpanId, uncollapsedSpans, isSelectedSpanIDUnCollapsed)
+		isPresentInThisSubtree, _spansFromRootToNode := getPathFromRootToSelectedSpanId(child, selectedSpanId)
 		// if the interested node is present in the given subtree then add the span node to uncollapsed node list
 		if isPresentInThisSubtree {
-			if !slices.Contains(uncollapsedSpans, node.SpanID) {
-				spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
-			}
 			isPresentInSubtreeForTheNode = true
 			spansFromRootToNode = append(spansFromRootToNode, _spansFromRootToNode...)
+			break
 		}
 	}
 	return isPresentInSubtreeForTheNode, spansFromRootToNode
 }
 
-func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, isPartOfPreOrder bool, hasSibling bool, selectedSpanId string) []*model.Span {
+// traverseOpts holds the traversal configuration that remains constant
+// throughout the recursion. Per-call state (level, isPartOfPreOrder, etc.)
+// is passed as direct arguments.
+type traverseOpts struct {
+	uncollapsedSpans map[string]struct{}
+	selectedSpanID   string
+}
+
+func traverseTrace(span *model.Span, opts traverseOpts, level uint64, isPartOfPreOrder bool, hasSibling bool) []*model.Span {
 	preOrderTraversal := []*model.Span{}
 
 	// sort the children to maintain the order across requests
@@ -126,8 +131,9 @@ func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, is
 		preOrderTraversal = append(preOrderTraversal, &nodeWithoutChildren)
 	}
 
+	_, isAlreadyUncollapsed := opts.uncollapsedSpans[span.SpanID]
 	for index, child := range span.Children {
-		_childTraversal := traverseTrace(child, uncollapsedSpans, level+1, isPartOfPreOrder && slices.Contains(uncollapsedSpans, span.SpanID), index != (len(span.Children)-1), selectedSpanId)
+		_childTraversal := traverseTrace(child, opts, level+1, isPartOfPreOrder && isAlreadyUncollapsed, index != (len(span.Children)-1))
 		preOrderTraversal = append(preOrderTraversal, _childTraversal...)
 		nodeWithoutChildren.SubTreeNodeCount += child.SubTreeNodeCount + 1
 		span.SubTreeNodeCount += child.SubTreeNodeCount + 1
@@ -160,15 +166,31 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
 
 	var preOrderTraversal = make([]*model.Span, 0)
 	var rootServiceName, rootServiceEntryPoint string
-	updatedUncollapsedSpans := uncollapsedSpans
+
+	// create a map of uncollapsed spans for quick lookup
+	uncollapsedSpanMap := make(map[string]struct{})
+	for _, spanID := range uncollapsedSpans {
+		uncollapsedSpanMap[spanID] = struct{}{}
+	}
 
 	selectedSpanIndex := -1
 	for _, rootSpanID := range traceRoots {
 		if rootNode, exists := spanIdToSpanNodeMap[rootSpanID.SpanID]; exists {
-			_, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID, updatedUncollapsedSpans, isSelectedSpanIDUnCollapsed)
-			updatedUncollapsedSpans = append(updatedUncollapsedSpans, spansFromRootToNode...)
+			present, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID)
+			if present {
+				for _, spanID := range spansFromRootToNode {
+					if selectedSpanID == spanID && !isSelectedSpanIDUnCollapsed {
+						continue
+					}
+					uncollapsedSpanMap[spanID] = struct{}{}
+				}
+			}
 
-			_preOrderTraversal := traverseTrace(rootNode, updatedUncollapsedSpans, 0, true, false, selectedSpanID)
+			opts := traverseOpts{
+				uncollapsedSpans: uncollapsedSpanMap,
+				selectedSpanID:   selectedSpanID,
+			}
+			_preOrderTraversal := traverseTrace(rootNode, opts, 0, true, false)
 			_selectedSpanIndex := findIndexForSelectedSpanFromPreOrder(_preOrderTraversal, selectedSpanID)
 
 			if _selectedSpanIndex != -1 {
@@ -210,5 +232,5 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
 		startIndex = 0
 	}
 
-	return preOrderTraversal[startIndex:endIndex], updatedUncollapsedSpans, rootServiceName, rootServiceEntryPoint
+	return preOrderTraversal[startIndex:endIndex], slices.Collect(maps.Keys(uncollapsedSpanMap)), rootServiceName, rootServiceEntryPoint
 }

pkg/query-service/app/traces/tracedetail/waterfall_test.go

@@ -0,0 +1,355 @@
+// Package tracedetail tests — waterfall
+//
+// # Background
+//
+// The waterfall view renders a trace as a scrollable list of spans in
+// pre-order (parent before children, siblings left-to-right). Because a trace
+// can have thousands of spans, only a window of ~500 is returned per request.
+// The window is centred on the selected span.
+//
+// # Key concepts
+//
+// uncollapsedSpans
+//
+//	The set of span IDs the user has manually expanded in the UI.
+//	Only the direct children of an uncollapsed span are included in the
+//	output; grandchildren stay hidden until their parent is also uncollapsed.
+//	When multiple spans are uncollapsed their children are all visible at once.
+//
+// selectedSpanID
+//
+//	The span currently focused — set when the user clicks a span in the
+//	waterfall or selects one from the flamegraph.  The output window is always
+//	centred on this span.  The path from the trace root down to the selected
+//	span is automatically uncollapsed so ancestors are visible even if they are
+//	not in uncollapsedSpans.
+//
+// isSelectedSpanIDUnCollapsed
+//
+//	Controls whether the selected span's own children are shown:
+//	  true  — user expanded the span (click-to-open in waterfall or flamegraph);
+//	          direct children of the selected span are included.
+//	  false — user selected without expanding;
+//	          the span is visible but its children remain hidden.
+//
+// traceRoots
+//
+//	Root spans of the trace — spans with no parent in the current dataset.
+//	Normally one, but multiple roots are common when upstream services are
+//	not instrumented or their spans were not sampled/exported.
+
+package tracedetail
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/SigNoz/signoz/pkg/query-service/model"
+	"github.com/stretchr/testify/assert"
+)
+
+// Pre-order traversal is preserved: parent before children, siblings left-to-right.
+func TestGetSelectedSpans_PreOrderTraversal(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
+		mkSpan("child2", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{"root", "child1"}, "root", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "child1", "grandchild", "child2"}, spanIDs(spans))
+}
+
+// Multiple roots: both trees are flattened into a single pre-order list with
+// root1's subtree before root2's. Service/entry-point come from the first root.
+//
+//	root1 svc-a          ← selected
+//	  └─ child1
+//	root2 svc-b
+//	  └─ child2
+//
+// Expected output order: root1 → child1 → root2 → child2
+func TestGetSelectedSpans_MultipleRoots(t *testing.T) {
+	root1 := mkSpan("root1", "svc-a", mkSpan("child1", "svc-a"))
+	root2 := mkSpan("root2", "svc-b", mkSpan("child2", "svc-b"))
+	spanMap := buildSpanMap(root1, root2)
+
+	spans, _, svcName, entryPoint := GetSelectedSpans([]string{"root1", "root2"}, "root1", []*model.Span{root1, root2}, spanMap, false)
+
+	assert.Equal(t, []string{"root1", "child1", "root2", "child2"}, spanIDs(spans), "root1 subtree must precede root2 subtree")
+	assert.Equal(t, "svc-a", svcName, "metadata comes from first root")
+	assert.Equal(t, "root1-op", entryPoint, "metadata comes from first root")
+}
+
+// isSelectedSpanIDUnCollapsed=true opens only the selected span's direct children,
+// not deeper descendants.
+//
+//	root → selected (expanded)
+//	          ├─ child1 ✓
+//	          │    └─ grandchild ✗  (only one level opened)
+//	          └─ child2 ✓
+func TestGetSelectedSpans_ExpandedSelectedSpan(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("selected", "svc",
+			mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
+			mkSpan("child2", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, true)
+
+	// root and selected are on the auto-uncollapsed path; child1/child2 are direct
+	// children of the expanded selected span; grandchild stays hidden.
+	assert.Equal(t, []string{"root", "selected", "child1", "child2"}, spanIDs(spans))
+}
+
+// Multiple spans uncollapsed simultaneously: children of all uncollapsed spans
+// are visible at once.
+//
+//	root
+//	  ├─ childA (uncollapsed) → grandchildA ✓
+//	  └─ childB (uncollapsed) → grandchildB ✓
+func TestGetSelectedSpans_MultipleUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("childA", "svc", mkSpan("grandchildA", "svc")),
+		mkSpan("childB", "svc", mkSpan("grandchildB", "svc")),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{"root", "childA", "childB"}, "root", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "childA", "grandchildA", "childB", "grandchildB"}, spanIDs(spans))
+}
+
+// Collapsing a span with other uncollapsed spans
+//
+// root
+// ├─ childA  (previously expanded — in uncollapsedSpans)
+// │    ├─ grandchild1 ✓
+// │    │    └─ greatGrandchild ✗  (grandchild1 not in uncollapsedSpans)
+// │    └─ grandchild2 ✓
+// └─ childB                        ← selected (not expanded)
+func TestGetSelectedSpans_ManualUncollapse(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("childA", "svc",
+			mkSpan("grandchild1", "svc", mkSpan("greatGrandchild", "svc")),
+			mkSpan("grandchild2", "svc"),
+		),
+		mkSpan("childB", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	// childA was expanded in a previous interaction; childB is now selected without expanding
+	spans, _, _, _ := GetSelectedSpans([]string{"childA"}, "childB", []*model.Span{root}, spanMap, false)
+
+	// path to childB auto-uncollpases root → childA and childB appear; childA is in
+	// uncollapsedSpans so its children appear; greatGrandchild stays hidden.
+	assert.Equal(t, []string{"root", "childA", "grandchild1", "grandchild2", "childB"}, spanIDs(spans))
+}
+
+// A collapsed span hides all children.
+func TestGetSelectedSpans_CollapsedSpan(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("child1", "svc"),
+		mkSpan("child2", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{}, "root", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root"}, spanIDs(spans))
+}
+
+// Selecting a span auto-uncollpases the path from root to that span so it is visible.
+//
+//	root → parent → selected
+func TestGetSelectedSpans_PathToSelectedIsUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("parent", "svc",
+			mkSpan("selected", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	// no manually uncollapsed spans — path should still be opened
+	spans, _, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "parent", "selected"}, spanIDs(spans))
+}
+
+// The path-to-selected spans are returned in updatedUncollapsedSpans.
+func TestGetSelectedSpans_PathReturnedInUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("parent", "svc",
+			mkSpan("selected", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	spans, uncollapsed, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "parent"}, uncollapsed)
+	assert.Equal(t, []string{"root", "parent", "selected"}, spanIDs(spans))
+}
+
+// Siblings of ancestors are rendered as collapsed nodes but their subtrees
+// must NOT be expanded.
+//
+//	root
+//	  ├─ unrelated → unrelated-child (✗)
+//	  └─ parent → selected
+func TestGetSelectedSpans_SiblingsNotExpanded(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("unrelated", "svc", mkSpan("unrelated-child", "svc")),
+		mkSpan("parent", "svc",
+			mkSpan("selected", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	spans, uncollapsed, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
+
+	// children of root sort alphabetically: parent < unrelated; unrelated-child stays hidden
+	assert.Equal(t, []string{"root", "parent", "selected", "unrelated"}, spanIDs(spans))
+	// only the path nodes are tracked as uncollapsed — unrelated is not
+	assert.Equal(t, []string{"root", "parent"}, uncollapsed)
+}
+
+// An unknown selectedSpanID must not panic; returns a window from index 0.
+func TestGetSelectedSpans_UnknownSelectedSpan(t *testing.T) {
+	root := mkSpan("root", "svc", mkSpan("child", "svc"))
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{}, "nonexistent", []*model.Span{root}, spanMap, false)
+	assert.Equal(t, []string{"root"}, spanIDs(spans))
+}
+
+// Test to check if Level, HasChildren, HasSiblings, and SubTreeNodeCount are populated correctly.
+//
+//	root          level=0, hasChildren=true,  hasSiblings=false, subTree=4
+//	  child1      level=1, hasChildren=true,  hasSiblings=true,  subTree=2
+//	    grandchild level=2, hasChildren=false, hasSiblings=false, subTree=1
+//	  child2      level=1, hasChildren=false, hasSiblings=false, subTree=1
+func TestGetSelectedSpans_SpanMetadata(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
+		mkSpan("child2", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{"root", "child1"}, "root", []*model.Span{root}, spanMap, false)
+
+	byID := map[string]*model.Span{}
+	for _, s := range spans {
+		byID[s.SpanID] = s
+	}
+
+	assert.Equal(t, uint64(0), byID["root"].Level)
+	assert.Equal(t, uint64(1), byID["child1"].Level)
+	assert.Equal(t, uint64(1), byID["child2"].Level)
+	assert.Equal(t, uint64(2), byID["grandchild"].Level)
+
+	assert.True(t, byID["root"].HasChildren)
+	assert.True(t, byID["child1"].HasChildren)
+	assert.False(t, byID["child2"].HasChildren)
+	assert.False(t, byID["grandchild"].HasChildren)
+
+	assert.False(t, byID["root"].HasSiblings, "root has no siblings")
+	assert.True(t, byID["child1"].HasSiblings, "child1 has sibling child2")
+	assert.False(t, byID["child2"].HasSiblings, "child2 is the last child")
+	assert.False(t, byID["grandchild"].HasSiblings, "grandchild has no siblings")
+
+	assert.Equal(t, uint64(4), byID["root"].SubTreeNodeCount)
+	assert.Equal(t, uint64(2), byID["child1"].SubTreeNodeCount)
+	assert.Equal(t, uint64(1), byID["grandchild"].SubTreeNodeCount)
+	assert.Equal(t, uint64(1), byID["child2"].SubTreeNodeCount)
+}
+
+// If the selected span is already in uncollapsedSpans AND isSelectedSpanIDUnCollapsed=true,
+func TestGetSelectedSpans_DuplicateInUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("selected", "svc", mkSpan("child", "svc")),
+	)
+	spanMap := buildSpanMap(root)
+	_, uncollapsed, _, _ := GetSelectedSpans(
+		[]string{"selected"}, // already present
+		"selected",
+		[]*model.Span{root}, spanMap,
+		true,
+	)
+
+	count := 0
+	for _, id := range uncollapsed {
+		if id == "selected" {
+			count++
+		}
+	}
+	assert.Equal(t, 1, count, "should appear once")
+}
+
+// makeChain builds a linear trace: span0 → span1 → … → span(n-1).
+// All span IDs are "span0", "span1", … so the caller can reference them by index.
+func makeChain(n int) (*model.Span, map[string]*model.Span, []string) {
+	spans := make([]*model.Span, n)
+	for i := n - 1; i >= 0; i-- {
+		if i == n-1 {
+			spans[i] = mkSpan(fmt.Sprintf("span%d", i), "svc")
+		} else {
+			spans[i] = mkSpan(fmt.Sprintf("span%d", i), "svc", spans[i+1])
+		}
+	}
+	uncollapsed := make([]string, n)
+	for i := range spans {
+		uncollapsed[i] = fmt.Sprintf("span%d", i)
+	}
+	return spans[0], buildSpanMap(spans[0]), uncollapsed
+}
+
+// The selected span is centred: 200 spans before it, 300 after (0.4 / 0.6 split).
+func TestGetSelectedSpans_WindowCentredOnSelected(t *testing.T) {
+	root, spanMap, uncollapsed := makeChain(600)
+	spans, _, _, _ := GetSelectedSpans(uncollapsed, "span300", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, 500, len(spans), "window should be 500 spans")
+	// window is [100, 600): span300 lands at position 200 (300 - 100)
+	assert.Equal(t, "span100", spans[0].SpanID, "window starts 200 before selected")
+	assert.Equal(t, "span300", spans[200].SpanID, "selected span at position 200 in window")
+	assert.Equal(t, "span599", spans[499].SpanID, "window ends 300 after selected")
+}
+
+// When the selected span is near the start, the window shifts right so no
+// negative index is used — the result is still 500 spans.
+func TestGetSelectedSpans_WindowShiftsAtStart(t *testing.T) {
+	root, spanMap, uncollapsed := makeChain(600)
+	spans, _, _, _ := GetSelectedSpans(uncollapsed, "span10", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, 500, len(spans))
+	assert.Equal(t, "span0", spans[0].SpanID, "window clamped to start of trace")
+	assert.Equal(t, "span10", spans[10].SpanID, "selected span still in window")
+}
+
+func mkSpan(id, service string, children ...*model.Span) *model.Span {
+	return &model.Span{
+		SpanID:      id,
+		ServiceName: service,
+		Name:        id + "-op",
+		Children:    children,
+	}
+}
+
+// spanIDs returns SpanIDs in order.
+func spanIDs(spans []*model.Span) []string {
+	ids := make([]string, len(spans))
+	for i, s := range spans {
+		ids[i] = s.SpanID
+	}
+	return ids
+}
+
+// buildSpanMap indexes every span in a set of trees by SpanID.
+func buildSpanMap(roots ...*model.Span) map[string]*model.Span {
+	m := map[string]*model.Span{}
+	var walk func(*model.Span)
+	walk = func(s *model.Span) {
+		m[s.SpanID] = s
+		for _, c := range s.Children {
+			walk(c)
+		}
+	}
+	for _, r := range roots {
+		walk(r)
+	}
+	return m
+}

pkg/query-service/rules/base_rule_test.go

@@ -698,7 +698,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), settings, prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), settings, prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Second,
 				nil,

pkg/query-service/rules/manager_test.go

@@ -253,7 +253,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 						WillReturnRows(samplesRows)
 
 					// Create Prometheus provider for this test
-					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, store)
+					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, store)
 				},
 				ManagerOptionsHook: func(opts *ManagerOptions) {
 					// Set Prometheus provider for PromQL queries

pkg/query-service/rules/managertestfactory.go

@@ -99,7 +99,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 
 	options := clickhouseReader.NewOptions("", "", "archiveNamespace")
 	providerSettings := instrumentationtest.New().ToProviderSettings()
-	prometheus := prometheustest.New(context.Background(), providerSettings, prometheus.Config{}, telemetryStore)
+	prometheus := prometheustest.New(context.Background(), providerSettings, prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 	reader := clickhouseReader.NewReader(
 		instrumentationtest.New().Logger(),
 		nil,

pkg/query-service/rules/prom_rule_test.go

@@ -940,7 +940,7 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 			).
 			WillReturnRows(samplesRows)
 
-		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
 		postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
 		postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
@@ -1061,7 +1061,7 @@ func _Enable_this_after_9146_issue_fix_is_merged_TestPromRuleNoData(t *testing.T
 			WithArgs("test_metric", "__name__", "test_metric").
 			WillReturnRows(fingerprintRows)
 
-		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
 		var target float64 = 0
 		postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
@@ -1281,7 +1281,7 @@ func TestMultipleThresholdPromRule(t *testing.T) {
 			).
 			WillReturnRows(samplesRows)
 
-		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
 		postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
 		postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
@@ -1441,7 +1441,7 @@ func TestPromRule_NoData(t *testing.T) {
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
-				prometheus.Config{},
+				prometheus.Config{Timeout: 2 * time.Minute},
 				telemetryStore,
 			)
 			defer func() {
@@ -1590,7 +1590,7 @@ func TestPromRule_NoData_AbsentFor(t *testing.T) {
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
-				prometheus.Config{},
+				prometheus.Config{Timeout: 2 * time.Minute},
 				telemetryStore,
 			)
 			defer func() {
@@ -1748,7 +1748,7 @@ func TestPromRuleEval_RequireMinPoints(t *testing.T) {
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
-				prometheus.Config{LookbackDelta: lookBackDelta},
+				prometheus.Config{Timeout: 2 * time.Minute, LookbackDelta: lookBackDelta},
 				telemetryStore,
 			)
 			defer func() {

pkg/query-service/rules/threshold_rule_test.go

@@ -779,7 +779,7 @@ func TestThresholdRuleUnitCombinations(t *testing.T) {
 			},
 		)
 		require.NoError(t, err)
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
 			"signoz_calls_total": {
@@ -894,7 +894,7 @@ func TestThresholdRuleNoData(t *testing.T) {
 		)
 		assert.NoError(t, err)
 		options := clickhouseReader.NewOptions("", "", "archiveNamespace")
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
 
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
@@ -1014,7 +1014,7 @@ func TestThresholdRuleTracesLink(t *testing.T) {
 		}
 
 		options := clickhouseReader.NewOptions("", "", "archiveNamespace")
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
 
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
@@ -1151,7 +1151,7 @@ func TestThresholdRuleLogsLink(t *testing.T) {
 		}
 
 		options := clickhouseReader.NewOptions("", "", "archiveNamespace")
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
 
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
@@ -1418,7 +1418,7 @@ func TestMultipleThresholdRule(t *testing.T) {
 			},
 		)
 		require.NoError(t, err)
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Second, nil, readerCache, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Second, nil, readerCache, options)
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
 			"signoz_calls_total": {
@@ -2220,7 +2220,7 @@ func TestThresholdEval_RequireMinPoints(t *testing.T) {
 			)
 			require.NoError(t, err)
 
-			prometheusProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+			prometheusProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 			reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheusProvider, "", time.Second, nil, readerCache, options)
 
 			rule, err := NewThresholdRule("some-id", valuer.GenerateUUID(), &postableRule, reader, nil, logger)