Compare commits

...

4 Commits

Author SHA1 Message Date
Nikhil Soni
c8fa1507ed chore: use typed argument in logs 2026-06-03 10:28:18 +05:30
Nikhil Soni
4b311c90c4 chore: try removing high cardinality attributes 2026-06-03 10:27:52 +05:30
Nikhil Soni
8f200f2c1d chore: use traces instead of tracedetail in metric name 2026-06-03 09:24:49 +05:30
Nikhil Soni
17f6028647 chore: add metric for waterfall monitoring
Gauge for config limit and counter to count large traces
2026-06-03 09:24:49 +05:30
2 changed files with 52 additions and 1 deletions

View File

@@ -4,6 +4,7 @@ import (
"context"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/modules/tracedetail"
"github.com/SigNoz/signoz/pkg/types/spantypes"
@@ -13,15 +14,27 @@ type module struct {
store spantypes.TraceStore
settings factory.ScopedProviderSettings
config tracedetail.Config
metrics *moduleMetrics
}
func NewModule(traceStore spantypes.TraceStore, providerSettings factory.ProviderSettings, cfg tracedetail.Config) *module {
scopedProviderSettings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/tracedetail/impltracedetail")
return &module{
metrics, err := newModuleMetrics(scopedProviderSettings.Meter())
if err != nil {
scopedProviderSettings.Logger().WarnContext(context.Background(), "tracedetail: failed to initialize metrics", errors.Attr(err))
}
m := &module{
config: cfg,
store: traceStore,
settings: scopedProviderSettings,
metrics: metrics,
}
m.metrics.waterfallMaxLimitToSelectAllSpans.Record(context.Background(), int64(cfg.Waterfall.MaxLimitToSelectAllSpans))
return m
}
func (m *module) GetWaterfall(ctx context.Context, traceID string, req *spantypes.PostableWaterfall) (*spantypes.GettableWaterfallTrace, error) {
@@ -80,6 +93,7 @@ func (m *module) GetWaterfallV4(ctx context.Context, traceID string, selectedSpa
}
effectiveLimit := min(selectAllLimit, m.config.Waterfall.MaxLimitToSelectAllSpans)
if summary.NumSpans > uint64(effectiveLimit) {
m.metrics.waterfallWindowedResponseCount.Add(ctx, 1)
return m.getWindowedWaterfall(ctx, traceID, selectedSpanID, uncollapsedSpans, summary.Start, summary.End)
}
return m.getFullWaterfall(ctx, traceID, summary)

View File

@@ -0,0 +1,37 @@
package impltracedetail
import (
"github.com/SigNoz/signoz/pkg/errors"
"go.opentelemetry.io/otel/metric"
)
type moduleMetrics struct {
waterfallMaxLimitToSelectAllSpans metric.Int64Gauge
waterfallWindowedResponseCount metric.Int64Counter
}
func newModuleMetrics(meter metric.Meter) (*moduleMetrics, error) {
var errs error
maxLimitGauge, err := meter.Int64Gauge(
"signoz.traces.waterfall.max_limit_to_select_all_spans",
metric.WithDescription("The span count limit above which windowed waterfall is returned instead of the full waterfall."),
metric.WithUnit("{spans}"),
)
if err != nil {
errs = errors.Join(errs, err)
}
windowedCounter, err := meter.Int64Counter(
"signoz.traces.waterfall.windowed_responses",
metric.WithDescription("Total number of waterfall requests that used the windowed path."),
)
if err != nil {
errs = errors.Join(errs, err)
}
return &moduleMetrics{
waterfallMaxLimitToSelectAllSpans: maxLimitGauge,
waterfallWindowedResponseCount: windowedCounter,
}, errs
}