signoz/pkg/querybuilder/collision.go

package querybuilder

import (
	"fmt"

	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)

// AdjustDuplicateKeys adjusts duplicate keys in the query by removing specific context and data type
// if the same key appears with different contexts or data types across SelectFields, GroupBy, and OrderBy.
// This ensures that each key is unique and generic enough to cover all its usages in the query.
func AdjustDuplicateKeys[T any](query *qbtypes.QueryBuilderQuery[T]) []string {

	// Create a map to track unique keys across SelectFields, GroupBy, and OrderBy
	globalUniqueKeysMap := map[string]telemetrytypes.TelemetryFieldKey{}

	// for recording modifications
	actions := []string{}

	// SelectFields
	for _, key := range query.SelectFields {
		deduplicateKeys(key, globalUniqueKeysMap, &actions)
	}

	// GroupBy
	for _, key := range query.GroupBy {
		deduplicateKeys(key.TelemetryFieldKey, globalUniqueKeysMap, &actions)
	}

	// OrderBy
	for _, key := range query.Order {
		deduplicateKeys(key.Key.TelemetryFieldKey, globalUniqueKeysMap, &actions)
	}

	// Reconstruct SelectFields slice

	newSelectFields := make([]telemetrytypes.TelemetryFieldKey, 0, len(query.SelectFields))

	seen := map[string]bool{}
	for _, key := range query.SelectFields {
		if !seen[key.Name] {
			newSelectFields = append(newSelectFields, globalUniqueKeysMap[key.Name])
			seen[key.Name] = true
		} else {
			actions = append(actions, fmt.Sprintf("Skipped duplicate SelectField key %s", key))
		}
	}
	query.SelectFields = newSelectFields

	// Reconstruct GroupBy slice
	newGroupBy := make([]qbtypes.GroupByKey, 0, len(query.GroupBy))
	seen = map[string]bool{}
	for _, key := range query.GroupBy {
		if !seen[key.Name] {
			newGroupBy = append(newGroupBy, qbtypes.GroupByKey{TelemetryFieldKey: globalUniqueKeysMap[key.Name]})
			seen[key.Name] = true
		} else {
			actions = append(actions, fmt.Sprintf("Skipped duplicate GroupBy key %s", key))
		}
	}
	query.GroupBy = newGroupBy

	// Reconstruct OrderBy slice
	// NOTE: 1 Edge case here is that if there are two order by on same key with different directions,
	// we will only keep one of them (the first one encountered). This is acceptable as such queries
	// don't make much sense.
	newOrderBy := make([]qbtypes.OrderBy, 0, len(query.Order))
	seen = map[string]bool{}
	for _, key := range query.Order {
		if !seen[key.Key.Name] {
			newOrderBy = append(newOrderBy, qbtypes.OrderBy{Key: qbtypes.OrderByKey{TelemetryFieldKey: globalUniqueKeysMap[key.Key.Name]}, Direction: key.Direction})
			seen[key.Key.Name] = true
		} else {
			actions = append(actions, fmt.Sprintf("Skipped duplicate OrderBy key %s", key.Key))
		}
	}
	query.Order = newOrderBy

	return actions
}

func deduplicateKeys(key telemetrytypes.TelemetryFieldKey, keysMap map[string]telemetrytypes.TelemetryFieldKey, actions *[]string) {
	if existingKey, ok := keysMap[key.Name]; !ok {
		keysMap[key.Name] = key
	} else {
		if existingKey.FieldContext != key.FieldContext && existingKey.FieldContext != telemetrytypes.FieldContextUnspecified {
			// remove field context in the map to make it generic
			*actions = append(*actions, fmt.Sprintf("Removed field context from %s for duplicate key %s", existingKey, key))
			existingKey.FieldContext = telemetrytypes.FieldContextUnspecified
		}
		if existingKey.FieldDataType != key.FieldDataType && existingKey.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
			// remove field data type in the map to make it generic
			*actions = append(*actions, fmt.Sprintf("Removed field data type from %s for duplicate key %s", existingKey, key))
			existingKey.FieldDataType = telemetrytypes.FieldDataTypeUnspecified
		}
		// Update the map with the modified key
		keysMap[key.Name] = existingKey
	}
}

func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey, intrinsicOrCalculatedField *telemetrytypes.TelemetryFieldKey) []string {

	// for recording modifications
	actions := []string{}

	if intrinsicOrCalculatedField != nil {
		/*
			Check if it also matches with any of the metadata keys

			For example, lets consider trace_id exists in attributes and is also an intrinsic field

			Now if user is using trace_id, we don't know if they mean intrinsic field or attribute.trace_id
			So we cannot take a call here (we'll leave this upto query builder to decide).

			However, if user is using attribute.trace_id, we can safely assume they mean attribute field
			and not intrinsic field.
			Similarly, if user is using trace_id with a field context or data type that doesn't match
			the intrinsic field, and there is no matching key in the metadata with the same name,
			we can safely assume they mean the intrinsic field and override the context and data type.

		*/

		// Check if there is any matching key in the metadata with the same name and it is not the same intrinsic/calculated field
		match := false
		for _, mapKey := range keys[key.Name] {
			// Either field context is unspecified or matches
			// and
			// Either field data type is unspecified or matches
			if (key.FieldContext == telemetrytypes.FieldContextUnspecified || mapKey.FieldContext == key.FieldContext) &&
				(key.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || mapKey.FieldDataType == key.FieldDataType) &&
				!mapKey.Equal(intrinsicOrCalculatedField) {
				match = true
				break
			}
		}

		// NOTE: if a user is highly opinionated and use attribute.duration_nano:string
		// It will be defaulted to intrinsic field duration_nano as the actual attribute might be attribute.duration_nano:number

		// We don't have a match, then it doesn't exist in attribute or resource attribute
		// use the intrinsic/calculated field
		if !match {
			// This is the case where user is using an intrinsic/calculated field
			// with a context or data type that may or may not match the intrinsic/calculated field
			// and there is no matching key in the metadata with the same name
			// So we can safely override the context and data type

			actions = append(actions, fmt.Sprintf("Overriding key: %s to %s", key, intrinsicOrCalculatedField))
			key.FieldContext = intrinsicOrCalculatedField.FieldContext
			key.FieldDataType = intrinsicOrCalculatedField.FieldDataType
			key.JSONDataType = intrinsicOrCalculatedField.JSONDataType
			key.Indexes = intrinsicOrCalculatedField.Indexes
			key.Materialized = intrinsicOrCalculatedField.Materialized
			key.JSONPlan = intrinsicOrCalculatedField.JSONPlan
			return actions

		}
	}

	// This means that the key provided by the user cannot be overridden to a single field because of ambiguity
	// So we need to look into metadata keys to find the best match

	// check if all the keys for the given field with matching context and data type
	matchingKeys := []*telemetrytypes.TelemetryFieldKey{}
	for _, metadataKey := range keys[key.Name] {
		// Only consider keys that match the context and data type (if specified)
		if (key.FieldContext == telemetrytypes.FieldContextUnspecified || key.FieldContext == metadataKey.FieldContext) &&
			(key.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || key.FieldDataType == metadataKey.FieldDataType) {
			matchingKeys = append(matchingKeys, metadataKey)
		}
	}

	// Also consider if context is actually part of the key name
	contextPrefixedMatchingKeys := []*telemetrytypes.TelemetryFieldKey{}
	if key.FieldContext != telemetrytypes.FieldContextUnspecified {
		for _, metadataKey := range keys[key.FieldContext.StringValue()+"."+key.Name] {
			// Since we prefixed the context in the name, we only need to match data type
			if key.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || key.FieldDataType == metadataKey.FieldDataType {
				contextPrefixedMatchingKeys = append(contextPrefixedMatchingKeys, metadataKey)
			}
		}
	}

	if len(matchingKeys)+len(contextPrefixedMatchingKeys) == 0 {
		// we do not have any matching keys, most likely user made a mistake, let downstream query builder handle it
		// Set materialized to false explicitly to avoid QB looking for materialized column
		key.Materialized = false
	} else if len(matchingKeys)+len(contextPrefixedMatchingKeys) == 1 {
		// only one matching key, use it
		var matchingKey *telemetrytypes.TelemetryFieldKey
		if len(matchingKeys) == 1 {
			matchingKey = matchingKeys[0]
		} else {
			matchingKey = contextPrefixedMatchingKeys[0]
		}

		if !key.Equal(matchingKey) {
			actions = append(actions, fmt.Sprintf("Adjusting key %s to %s", key, matchingKey))
		}
		key.Name = matchingKey.Name
		key.FieldContext = matchingKey.FieldContext
		key.FieldDataType = matchingKey.FieldDataType
		key.JSONDataType = matchingKey.JSONDataType
		key.Indexes = matchingKey.Indexes
		key.Materialized = matchingKey.Materialized
		key.JSONPlan = matchingKey.JSONPlan
		
		return actions
	} else {
		// multiple matching keys, set materialized only if all the keys are materialized
		// TODO: This could all be redundant if it is not, it should be.
		// Downstream query builder should handle multiple matching keys with their own metadata
		// and not rely on this function to do so.
		materialized := true
		indexes := []telemetrytypes.JSONDataTypeIndex{}
		fieldContextsSeen := map[telemetrytypes.FieldContext]bool{}
		dataTypesSeen := map[telemetrytypes.FieldDataType]bool{}
		jsonTypesSeen := map[string]*telemetrytypes.JSONDataType{}
		for _, matchingKey := range matchingKeys {
			materialized = materialized && matchingKey.Materialized
			fieldContextsSeen[matchingKey.FieldContext] = true
			dataTypesSeen[matchingKey.FieldDataType] = true
			if matchingKey.JSONDataType != nil {
				jsonTypesSeen[matchingKey.JSONDataType.StringValue()] = matchingKey.JSONDataType
			}
			indexes = append(indexes, matchingKey.Indexes...)
		}
		for _, matchingKey := range contextPrefixedMatchingKeys {
			materialized = materialized && matchingKey.Materialized
			fieldContextsSeen[matchingKey.FieldContext] = true
			dataTypesSeen[matchingKey.FieldDataType] = true
			if matchingKey.JSONDataType != nil {
				jsonTypesSeen[matchingKey.JSONDataType.StringValue()] = matchingKey.JSONDataType
			}
			indexes = append(indexes, matchingKey.Indexes...)
		}
		key.Materialized = materialized
		if len(indexes) > 0 {
			key.Indexes = indexes
		}

		if len(fieldContextsSeen) == 1 && key.FieldContext == telemetrytypes.FieldContextUnspecified {
			// all matching keys have same field context, use it
			for context := range fieldContextsSeen {
				actions = append(actions, fmt.Sprintf("Adjusting key %s to have field context %s", key, context.StringValue()))
				key.FieldContext = context
				break
			}
		}

		if len(dataTypesSeen) == 1 && key.FieldDataType == telemetrytypes.FieldDataTypeUnspecified {
			// all matching keys have same data type, use it
			for dt := range dataTypesSeen {
				actions = append(actions, fmt.Sprintf("Adjusting key %s to have data type %s", key, dt.StringValue()))
				key.FieldDataType = dt
				break
			}
		}

		if len(jsonTypesSeen) == 1 && key.JSONDataType == nil {
			// all matching keys have same JSON data type, use it
			for _, jt := range jsonTypesSeen {
				actions = append(actions, fmt.Sprintf("Adjusting key %s to have JSON data type %s", key, jt.StringValue()))
				key.JSONDataType = jt
				break
			}
		}
	}

	return actions
}

func AdjustKeysForAliasExpressions[T any](query *qbtypes.QueryBuilderQuery[T], requestType qbtypes.RequestType) []string {
	/*
		For example, if user is using `body.count` as an alias for aggregation and
		Uses it in orderBy, upstream code will convert it to just `count` with fieldContext as Body
		But we need to adjust it back to `body.count` with fieldContext as unspecified
	*/
	actions := []string{}
	if requestType != qbtypes.RequestTypeRaw && requestType != qbtypes.RequestTypeRawStream {
		aliasExpressions := map[string]bool{}
		for _, agg := range query.Aggregations {
			switch v := any(agg).(type) {
			case qbtypes.LogAggregation:
				if v.Alias != "" {
					aliasExpressions[v.Alias] = true
				}
			case qbtypes.TraceAggregation:
				if v.Alias != "" {
					aliasExpressions[v.Alias] = true
				}
			default:
				continue
			}
		}
		if len(aliasExpressions) > 0 {
			for idx := range query.Order {
				contextPrefixedKeyName := fmt.Sprintf("%s.%s", query.Order[idx].Key.FieldContext.StringValue(), query.Order[idx].Key.Name)
				if aliasExpressions[contextPrefixedKeyName] {
					actions = append(actions, fmt.Sprintf("Adjusting OrderBy key %s to %s", query.Order[idx].Key, contextPrefixedKeyName))
					query.Order[idx].Key.FieldContext = telemetrytypes.FieldContextUnspecified
					query.Order[idx].Key.Name = contextPrefixedKeyName
				}
			}
		}
	}
	return actions
}