feat: include monotonicity in metrics' properties API responses

.gitignore

@@ -1,11 +1,8 @@
 
 node_modules
 
-# editor
 .vscode
 !.vscode/settings.json
-.zed
-.idea
 
 deploy/docker/environment_tiny/common_test
 frontend/node_modules
@@ -34,6 +31,8 @@ frontend/yarn-debug.log*
 frontend/yarn-error.log*
 frontend/src/constants/env.ts
 
+.idea
+
 **/build
 **/storage
 **/locust-scripts/__pycache__/

frontend/src/container/ApiMonitoring/Explorer/Domains/DomainDetails/constants.ts

@@ -14,6 +14,6 @@ export const VIEW_TYPES = {
 export const SPAN_ATTRIBUTES = {
 	URL_PATH: 'http.url',
 	RESPONSE_STATUS_CODE: 'response_status_code',
-	SERVER_NAME: 'http_host',
+	SERVER_NAME: 'net.peer.name',
 	SERVER_PORT: 'net.peer.port',
 } as const;

frontend/src/container/ApiMonitoring/__tests__/TopErrors.test.tsx

@@ -4,7 +4,6 @@ import { rest, server } from 'mocks-server/server';
 import { fireEvent, render, screen, waitFor, within } from 'tests/test-utils';
 import { DataSource } from 'types/common/queryBuilder';
 
-import { SPAN_ATTRIBUTES } from '../Explorer/Domains/DomainDetails/constants';
 import TopErrors from '../Explorer/Domains/DomainDetails/TopErrors';
 import { getTopErrorsQueryPayload } from '../utils';
 
@@ -216,7 +215,7 @@ describe('TopErrors', () => {
 					value: 'true',
 				}),
 				expect.objectContaining({
-					key: expect.objectContaining({ key: SPAN_ATTRIBUTES.SERVER_NAME }),
+					key: expect.objectContaining({ key: 'net.peer.name' }),
 					op: '=',
 					value: 'test-domain',
 				}),

frontend/src/container/ApiMonitoring/utils.tsx

@@ -638,7 +638,7 @@ export const getEndPointsQueryPayload = (
 										key: {
 											dataType: DataTypes.String,
 											key: SPAN_ATTRIBUTES.SERVER_NAME,
-											type: '',
+											type: 'tag',
 										},
 										op: '=',
 										value: domainName,
@@ -685,7 +685,7 @@ export const getEndPointsQueryPayload = (
 										key: {
 											dataType: DataTypes.String,
 											key: SPAN_ATTRIBUTES.SERVER_NAME,
-											type: '',
+											type: 'tag',
 										},
 										op: '=',
 										value: domainName,
@@ -733,7 +733,7 @@ export const getEndPointsQueryPayload = (
 										key: {
 											dataType: DataTypes.String,
 											key: SPAN_ATTRIBUTES.SERVER_NAME,
-											type: '',
+											type: 'tag',
 										},
 										op: '=',
 										value: domainName,
@@ -780,7 +780,7 @@ export const getEndPointsQueryPayload = (
 										key: {
 											dataType: DataTypes.String,
 											key: SPAN_ATTRIBUTES.SERVER_NAME,
-											type: '',
+											type: 'tag',
 										},
 										op: '=',
 										value: domainName,
@@ -1302,7 +1302,7 @@ export const getTopErrorsCoRelationQueryFilters = (
 		{
 			id: 'e8a043b7',
 			key: {
-				key: SPAN_ATTRIBUTES.SERVER_NAME,
+				key: 'net.peer.name',
 				dataType: DataTypes.String,
 				type: '',
 			},
@@ -2198,7 +2198,7 @@ export const getEndPointZeroStateQueryPayload = (
 									key: {
 										key: SPAN_ATTRIBUTES.SERVER_NAME,
 										dataType: DataTypes.String,
-										type: '',
+										type: 'tag',
 									},
 									op: '=',
 									value: domainName,
@@ -2793,7 +2793,7 @@ export const getStatusCodeBarChartWidgetData = (
 								key: {
 									dataType: DataTypes.String,
 									key: SPAN_ATTRIBUTES.SERVER_NAME,
-									type: '',
+									type: 'tag',
 								},
 								op: '=',
 								value: domainName,

pkg/modules/thirdpartyapi/translator.go

@@ -12,30 +12,60 @@ import (
 )
 
 const (
-	derivedKeyHTTPURL  = "http_url" // https://signoz.io/docs/traces-management/guides/derived-fields-spans/#http_url
-	derivedKeyHTTPHost = "http_host"
+	urlPathKeyLegacy       = "http.url"
+	serverAddressKeyLegacy = "net.peer.name"
+
+	urlPathKey       = "url.full"
+	serverAddressKey = "server.address"
 )
 
 var defaultStepInterval = 60 * time.Second
 
-var (
-	groupByKeyHTTPHost = qbtypes.GroupByKey{
-		TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-			Name:          derivedKeyHTTPHost,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
-			FieldContext:  telemetrytypes.FieldContextSpan,
-			Signal:        telemetrytypes.SignalTraces,
+type SemconvFieldMapping struct {
+	LegacyField  string
+	CurrentField string
+	FieldType    telemetrytypes.FieldDataType
+	Context      telemetrytypes.FieldContext
+}
+
+var dualSemconvGroupByKeys = map[string][]qbtypes.GroupByKey{
+	"server": {
+		{
+			TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+				Name:          serverAddressKey,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				Signal:        telemetrytypes.SignalTraces,
+			},
 		},
-	}
-	groupByKeyHTTPURL = qbtypes.GroupByKey{
-		TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-			Name:          derivedKeyHTTPURL,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
-			FieldContext:  telemetrytypes.FieldContextSpan,
-			Signal:        telemetrytypes.SignalTraces,
+		{
+			TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+				Name:          serverAddressKeyLegacy,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				Signal:        telemetrytypes.SignalTraces,
+			},
 		},
-	}
-)
+	},
+	"url": {
+		{
+			TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+				Name:          urlPathKey,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				Signal:        telemetrytypes.SignalTraces,
+			},
+		},
+		{
+			TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+				Name:          urlPathKeyLegacy,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				FieldContext:  telemetrytypes.FieldContextAttribute,
+				Signal:        telemetrytypes.SignalTraces,
+			},
+		},
+	},
+}
 
 func FilterIntermediateColumns(result *qbtypes.QueryRangeResponse) *qbtypes.QueryRangeResponse {
 	if result == nil || result.Data.Results == nil {
@@ -84,6 +114,103 @@ func FilterIntermediateColumns(result *qbtypes.QueryRangeResponse) *qbtypes.Quer
 	return result
 }
 
+func MergeSemconvColumns(result *qbtypes.QueryRangeResponse) *qbtypes.QueryRangeResponse {
+	if result == nil || result.Data.Results == nil {
+		return result
+	}
+
+	for _, res := range result.Data.Results {
+		scalarData, ok := res.(*qbtypes.ScalarData)
+		if !ok {
+			continue
+		}
+
+		serverAddressKeyIdx := -1
+		serverAddressKeyLegacyIdx := -1
+
+		for i, col := range scalarData.Columns {
+			if col.Name == serverAddressKey {
+				serverAddressKeyIdx = i
+			} else if col.Name == serverAddressKeyLegacy {
+				serverAddressKeyLegacyIdx = i
+			}
+		}
+
+		if serverAddressKeyIdx == -1 || serverAddressKeyLegacyIdx == -1 {
+			continue
+		}
+
+		var newRows [][]any
+		for _, row := range scalarData.Data {
+			if len(row) <= serverAddressKeyIdx || len(row) <= serverAddressKeyLegacyIdx {
+				continue
+			}
+
+			var serverName any
+			if isValidValue(row[serverAddressKeyIdx]) {
+				serverName = row[serverAddressKeyIdx]
+			} else if isValidValue(row[serverAddressKeyLegacyIdx]) {
+				serverName = row[serverAddressKeyLegacyIdx]
+			}
+
+			if serverName != nil {
+				newRow := make([]any, len(row)-1)
+				newRow[0] = serverName
+
+				targetIdx := 1
+				for i, val := range row {
+					if i != serverAddressKeyLegacyIdx && i != serverAddressKeyIdx {
+						if targetIdx < len(newRow) {
+							newRow[targetIdx] = val
+							targetIdx++
+						}
+					}
+				}
+				newRows = append(newRows, newRow)
+			}
+		}
+
+		newColumns := make([]*qbtypes.ColumnDescriptor, len(scalarData.Columns)-1)
+		targetIdx := 0
+		for i, col := range scalarData.Columns {
+			if i == serverAddressKeyIdx {
+				newCol := &qbtypes.ColumnDescriptor{
+					TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
+						Name:          serverAddressKeyLegacy,
+						FieldDataType: col.FieldDataType,
+						FieldContext:  col.FieldContext,
+						Signal:        col.Signal,
+					},
+					QueryName:        col.QueryName,
+					AggregationIndex: col.AggregationIndex,
+					Meta:             col.Meta,
+					Type:             col.Type,
+				}
+				newColumns[targetIdx] = newCol
+				targetIdx++
+			} else if i != serverAddressKeyLegacyIdx {
+				newColumns[targetIdx] = col
+				targetIdx++
+			}
+		}
+
+		scalarData.Columns = newColumns
+		scalarData.Data = newRows
+	}
+
+	return result
+}
+
+func isValidValue(val any) bool {
+	if val == nil {
+		return false
+	}
+	if str, ok := val.(string); ok {
+		return str != "" && str != "n/a"
+	}
+	return true
+}
+
 func FilterResponse(results []*qbtypes.QueryRangeResponse) []*qbtypes.QueryRangeResponse {
 	filteredResults := make([]*qbtypes.QueryRangeResponse, 0, len(results))
 
@@ -134,7 +261,7 @@ func FilterResponse(results []*qbtypes.QueryRangeResponse) []*qbtypes.QueryRange
 
 func shouldIncludeSeries(series *qbtypes.TimeSeries) bool {
 	for _, label := range series.Labels {
-		if label.Key.Name == derivedKeyHTTPHost {
+		if label.Key.Name == serverAddressKeyLegacy || label.Key.Name == serverAddressKey {
 			if strVal, ok := label.Value.(string); ok {
 				if net.ParseIP(strVal) != nil {
 					return false
@@ -147,10 +274,12 @@ func shouldIncludeSeries(series *qbtypes.TimeSeries) bool {
 
 func shouldIncludeRow(row *qbtypes.RawRow) bool {
 	if row.Data != nil {
-		if domainVal, ok := row.Data[derivedKeyHTTPHost]; ok {
-			if domainStr, ok := domainVal.(string); ok {
-				if net.ParseIP(domainStr) != nil {
-					return false
+		for _, key := range []string{serverAddressKeyLegacy, serverAddressKey} {
+			if domainVal, ok := row.Data[key]; ok {
+				if domainStr, ok := domainVal.(string); ok {
+					if net.ParseIP(domainStr) != nil {
+						return false
+					}
 				}
 			}
 		}
@@ -158,8 +287,8 @@ func shouldIncludeRow(row *qbtypes.RawRow) bool {
 	return true
 }
 
-func mergeGroupBy(base qbtypes.GroupByKey, additional []qbtypes.GroupByKey) []qbtypes.GroupByKey {
-	return append([]qbtypes.GroupByKey{base}, additional...)
+func mergeGroupBy(base, additional []qbtypes.GroupByKey) []qbtypes.GroupByKey {
+	return append(base, additional...)
 }
 
 func BuildDomainList(req *thirdpartyapitypes.ThirdPartyApiRequest) (*qbtypes.QueryRangeRequest, error) {
@@ -225,10 +354,10 @@ func buildEndpointsQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtypes.Q
 			Signal:       telemetrytypes.SignalTraces,
 			StepInterval: qbtypes.Step{Duration: defaultStepInterval},
 			Aggregations: []qbtypes.TraceAggregation{
-				{Expression: fmt.Sprintf("count_distinct(%s)", derivedKeyHTTPURL)},
+				{Expression: "count_distinct(http.url)"},
 			},
 			Filter:  buildBaseFilter(req.Filter),
-			GroupBy: mergeGroupBy(groupByKeyHTTPHost, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["server"], req.GroupBy),
 		},
 	}
 }
@@ -244,7 +373,7 @@ func buildLastSeenQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtypes.Qu
 				{Expression: "max(timestamp)"},
 			},
 			Filter:  buildBaseFilter(req.Filter),
-			GroupBy: mergeGroupBy(groupByKeyHTTPHost, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["server"], req.GroupBy),
 		},
 	}
 }
@@ -260,7 +389,7 @@ func buildRpsQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtypes.QueryEn
 				{Expression: "rate()"},
 			},
 			Filter:  buildBaseFilter(req.Filter),
-			GroupBy: mergeGroupBy(groupByKeyHTTPHost, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["server"], req.GroupBy),
 		},
 	}
 }
@@ -278,7 +407,7 @@ func buildErrorQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtypes.Query
 				{Expression: "count()"},
 			},
 			Filter:  filter,
-			GroupBy: mergeGroupBy(groupByKeyHTTPHost, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["server"], req.GroupBy),
 		},
 	}
 }
@@ -294,7 +423,7 @@ func buildTotalSpanQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtypes.Q
 				{Expression: "count()"},
 			},
 			Filter:  buildBaseFilter(req.Filter),
-			GroupBy: mergeGroupBy(groupByKeyHTTPHost, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["server"], req.GroupBy),
 		},
 	}
 }
@@ -310,7 +439,7 @@ func buildP99Query(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtypes.QueryEn
 				{Expression: "p99(duration_nano)"},
 			},
 			Filter:  buildBaseFilter(req.Filter),
-			GroupBy: mergeGroupBy(groupByKeyHTTPHost, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["server"], req.GroupBy),
 		},
 	}
 }
@@ -333,10 +462,10 @@ func buildEndpointsInfoQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtyp
 			Signal:       telemetrytypes.SignalTraces,
 			StepInterval: qbtypes.Step{Duration: defaultStepInterval},
 			Aggregations: []qbtypes.TraceAggregation{
-				{Expression: fmt.Sprintf("rate(%s)", derivedKeyHTTPURL)},
+				{Expression: "rate(http.url)"},
 			},
 			Filter:  buildBaseFilter(req.Filter),
-			GroupBy: mergeGroupBy(groupByKeyHTTPURL, req.GroupBy),
+			GroupBy: mergeGroupBy(dualSemconvGroupByKeys["url"], req.GroupBy),
 		},
 	}
 }
@@ -390,7 +519,8 @@ func buildLastSeenInfoQuery(req *thirdpartyapitypes.ThirdPartyApiRequest) qbtype
 }
 
 func buildBaseFilter(additionalFilter *qbtypes.Filter) *qbtypes.Filter {
-	baseExpression := fmt.Sprintf("%s EXISTS AND kind_string = 'Client'", derivedKeyHTTPURL)
+	baseExpression := fmt.Sprintf("(%s EXISTS OR %s EXISTS) AND kind_string = 'Client'",
+		urlPathKeyLegacy, urlPathKey)
 
 	if additionalFilter != nil && additionalFilter.Expression != "" {
 		// even if it contains kind_string we add with an AND so it doesn't matter if the user is overriding it.

pkg/modules/thirdpartyapi/translator_test.go

@@ -1,9 +1,8 @@
 package thirdpartyapi
 
 import (
-	"testing"
-
 	"github.com/SigNoz/signoz/pkg/types/thirdpartyapitypes"
+	"testing"
 
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -29,7 +28,7 @@ func TestFilterResponse(t *testing.T) {
 											{
 												Labels: []*qbtypes.Label{
 													{
-														Key:   telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
+														Key:   telemetrytypes.TelemetryFieldKey{Name: "net.peer.name"},
 														Value: "192.168.1.1",
 													},
 												},
@@ -37,7 +36,7 @@ func TestFilterResponse(t *testing.T) {
 											{
 												Labels: []*qbtypes.Label{
 													{
-														Key:   telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
+														Key:   telemetrytypes.TelemetryFieldKey{Name: "net.peer.name"},
 														Value: "example.com",
 													},
 												},
@@ -61,7 +60,7 @@ func TestFilterResponse(t *testing.T) {
 											{
 												Labels: []*qbtypes.Label{
 													{
-														Key:   telemetrytypes.TelemetryFieldKey{Name: derivedKeyHTTPHost},
+														Key:   telemetrytypes.TelemetryFieldKey{Name: "net.peer.name"},
 														Value: "example.com",
 													},
 												},
@@ -85,12 +84,12 @@ func TestFilterResponse(t *testing.T) {
 								Rows: []*qbtypes.RawRow{
 									{
 										Data: map[string]any{
-											derivedKeyHTTPHost: "192.168.1.1",
+											"net.peer.name": "192.168.1.1",
 										},
 									},
 									{
 										Data: map[string]any{
-											derivedKeyHTTPHost: "example.com",
+											"net.peer.name": "example.com",
 										},
 									},
 								},
@@ -107,7 +106,7 @@ func TestFilterResponse(t *testing.T) {
 								Rows: []*qbtypes.RawRow{
 									{
 										Data: map[string]any{
-											derivedKeyHTTPHost: "example.com",
+											"net.peer.name": "example.com",
 										},
 									},
 								},

pkg/query-service/app/clickhouseReader/reader.go

@@ -3261,20 +3261,14 @@ func (r *ClickHouseReader) GetMetricAggregateAttributes(ctx context.Context, org
 		metadata := metadataMap[name]
 
 		typ := string(metadata.MetricType)
-		temporality := string(metadata.Temporality)
-		isMonotonic := metadata.IsMonotonic
-
-		// Non-monotonic cumulative sums are treated as gauges
-		if typ == "Sum" && !isMonotonic && temporality == string(v3.Cumulative) {
-			typ = "Gauge"
-		}
 
 		// unlike traces/logs `tag`/`resource` type, the `Type` will be metric type
 		key := v3.AttributeKey{
-			Key:      name,
-			DataType: v3.AttributeKeyDataTypeFloat64,
-			Type:     v3.AttributeKeyType(typ),
-			IsColumn: true,
+			Key:         name,
+			DataType:    v3.AttributeKeyDataTypeFloat64,
+			Type:        v3.AttributeKeyType(typ),
+			IsMonotonic: metadata.IsMonotonic,
+			IsColumn:    true,
 		}
 
 		if _, ok := seen[name+typ]; ok {
@@ -5419,6 +5413,7 @@ func (r *ClickHouseReader) ListSummaryMetrics(ctx context.Context, orgID valuer.
 		    t.metric_name AS metric_name,
 		    ANY_VALUE(t.description) AS description,
 		    ANY_VALUE(t.type) AS metric_type,
+			ANY_VALUE(t.is_monotonic) AS metric_is_monotonic,
 		    ANY_VALUE(t.unit) AS metric_unit,
 		    uniq(t.fingerprint) AS timeseries,
 			uniq(metric_name) OVER() AS total
@@ -5450,7 +5445,7 @@ func (r *ClickHouseReader) ListSummaryMetrics(ctx context.Context, orgID valuer.
 
 	for rows.Next() {
 		var metric metrics_explorer.MetricDetail
-		if err := rows.Scan(&metric.MetricName, &metric.Description, &metric.MetricType, &metric.MetricUnit, &metric.TimeSeries, &response.Total); err != nil {
+		if err := rows.Scan(&metric.MetricName, &metric.Description, &metric.MetricType, &metric.IsMonotonic, &metric.MetricUnit, &metric.TimeSeries, &response.Total); err != nil {
 			zap.L().Error("Error scanning metric row", zap.Error(err))
 			return &response, &model.ApiError{Typ: "ClickHouseError", Err: err}
 		}

pkg/query-service/app/http_handler.go

@@ -4996,6 +4996,7 @@ func (aH *APIHandler) getDomainList(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	result = thirdpartyapi.MergeSemconvColumns(result)
 	result = thirdpartyapi.FilterIntermediateColumns(result)
 
 	// Filter IP addresses if ShowIp is false
@@ -5052,6 +5053,7 @@ func (aH *APIHandler) getDomainInfo(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	result = thirdpartyapi.MergeSemconvColumns(result)
 	result = thirdpartyapi.FilterIntermediateColumns(result)
 
 	// Filter IP addresses if ShowIp is false

pkg/query-service/model/metrics_explorer/summary.go

@@ -36,6 +36,7 @@ type MetricDetail struct {
 	TimeSeries   uint64 `json:"timeseries"`
 	Samples      uint64 `json:"samples"`
 	LastReceived int64  `json:"lastReceived"`
+	IsMonotonic  bool   `json:"is_monotonic"`
 }
 
 type TreeMapResponseItem struct {

pkg/query-service/model/v3/v3.go

@@ -381,11 +381,12 @@ func (t AttributeKeyType) String() string {
 }
 
 type AttributeKey struct {
-	Key      string               `json:"key"`
-	DataType AttributeKeyDataType `json:"dataType"`
-	Type     AttributeKeyType     `json:"type"`
-	IsColumn bool                 `json:"isColumn"`
-	IsJSON   bool                 `json:"isJSON"`
+	Key         string               `json:"key"`
+	DataType    AttributeKeyDataType `json:"dataType"`
+	Type        AttributeKeyType     `json:"type"`
+	IsColumn    bool                 `json:"isColumn"`
+	IsMonotonic bool                 `json:"is_monotonic"`
+	IsJSON      bool                 `json:"isJSON"`
 }
 
 func (a AttributeKey) CacheKey() string {