fix: handle datatype collision

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.40.1
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.129.10-rc.9
+	github.com/SigNoz/signoz-otel-collector v0.129.13-rc.2
 	github.com/antlr4-go/antlr/v4 v4.13.1
 	github.com/antonmedv/expr v1.15.3
 	github.com/bytedance/sonic v1.14.1

go.sum

@@ -108,6 +108,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/signoz-otel-collector v0.129.10-rc.9 h1:WmYDSSwzyW2yiJ3tPq5AFdjsrz3NBdtPkygtFKOsACw=
 github.com/SigNoz/signoz-otel-collector v0.129.10-rc.9/go.mod h1:4eJCRUd/P4OiCHXvGYZK8q6oyBVGJFVj/G6qKSoN/TQ=
+github.com/SigNoz/signoz-otel-collector v0.129.13-rc.2 h1:225Seh4b0tBavJ7dRTqX2BNG0egEXc4polH4Di5LJJA=
+github.com/SigNoz/signoz-otel-collector v0.129.13-rc.2/go.mod h1:4eJCRUd/P4OiCHXvGYZK8q6oyBVGJFVj/G6qKSoN/TQ=
 github.com/Yiling-J/theine-go v0.6.2 h1:1GeoXeQ0O0AUkiwj2S9Jc0Mzx+hpqzmqsJ4kIC4M9AY=
 github.com/Yiling-J/theine-go v0.6.2/go.mod h1:08QpMa5JZ2pKN+UJCRrCasWYO1IKCdl54Xa836rpmDU=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=

pkg/querybuilder/fallback_expr.go

@@ -204,7 +204,10 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
 	// While we expect user not to send the mixed data types, it inevitably happens
 	// So we handle the data type collisions here
 	switch key.FieldDataType {
-	case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString:
+	case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString, telemetrytypes.FieldDataTypeJSON:
+		if key.FieldDataType == telemetrytypes.FieldDataTypeJSON {
+			tblFieldName = fmt.Sprintf("toString(%s)", tblFieldName)
+		}
 		switch v := value.(type) {
 		case float64:
 			// try to convert the string value to to number
@@ -219,7 +222,6 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
 			// we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string
 			value = fmt.Sprintf("%t", v)
 		}
-
 	case telemetrytypes.FieldDataTypeInt64,
 		telemetrytypes.FieldDataTypeArrayInt64,
 		telemetrytypes.FieldDataTypeNumber,

pkg/querybuilder/where_clause_visitor.go

@@ -855,7 +855,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
 	// 1. either user meant key ( this is already handled above in fieldKeysForName )
 	// 2. or user meant `attribute.key` we look up in the map for all possible field keys with name 'attribute.key'
 
-	// Note: 
+	// Note:
 	// If user only wants to search `attribute.key`, then they have to use `attribute.attribute.key`
 	// If user only wants to search `key`, then they have to use `key`
 	// If user wants to search both, they can use `attribute.key` and we will resolve the ambiguity

pkg/telemetrylogs/condition_builder.go

@@ -35,13 +35,24 @@ func (c *conditionBuilder) conditionFor(
 		return "", err
 	}
 
-	if column.IsJSONColumn() && querybuilder.BodyJSONQueryEnabled {
-		valueType, value := InferDataType(value, operator, key)
-		cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
-		if err != nil {
-			return "", err
+	if column.Type.GetType() == schema.ColumnTypeEnumJSON {
+		// If field data is Not JSON Column Itself, then we need to build a JSON condition
+		if key.FieldDataType != telemetrytypes.FieldDataTypeJSON && querybuilder.BodyJSONQueryEnabled {
+			valueType, value := InferDataType(value, operator, key)
+			cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
+			if err != nil {
+				return "", err
+			}
+			return cond, nil
+		} else if !operator.IsOpValidForJSON() {
+			// Skip building condition for invalid operators on JSON columns
+			return "", nil
 		}
-		return cond, nil
+	}
+
+	if key.IsPsuedoKey() && slices.Contains(PsuedoIntrinsicFieldsInvalidOps[key.Name], operator) {
+		// Skip building a condition for psuedo intrinsic fields with invalid operators
+		return "", nil
 	}
 
 	if operator.IsStringSearchOperator() {
@@ -108,7 +119,6 @@ func (c *conditionBuilder) conditionFor(
 		return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
 	case qbtypes.FilterOperatorNotContains:
 		return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
-
 	case qbtypes.FilterOperatorRegexp:
 		// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
 		// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
@@ -177,9 +187,9 @@ func (c *conditionBuilder) conditionFor(
 		switch column.Type.GetType() {
 		case schema.ColumnTypeEnumJSON:
 			if operator == qbtypes.FilterOperatorExists {
-				return sb.IsNotNull(tblFieldName), nil
+				return sb.EQ(fmt.Sprintf("empty(%s)", tblFieldName), false), nil
 			} else {
-				return sb.IsNull(tblFieldName), nil
+				return sb.EQ(fmt.Sprintf("empty(%s)", tblFieldName), true), nil
 			}
 		case schema.ColumnTypeEnumLowCardinality:
 			switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {

pkg/telemetrylogs/const.go

@@ -1,7 +1,11 @@
 package telemetrylogs
 
 import (
+	"fmt"
+	"strings"
+
 	"github.com/SigNoz/signoz-otel-collector/constants"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
@@ -45,57 +49,66 @@ var (
 		FieldContext:  telemetrytypes.FieldContextLog,
 		FieldDataType: telemetrytypes.FieldDataTypeString,
 	}
-	IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{
-		"body": {
-			Name:          "body",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextLog,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
+	IntrinsicFields = map[string]func() []telemetrytypes.TelemetryFieldKey{
+		"body": buildBodyIntrinsicFields,
+		"trace_id": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "trace_id",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			}}
 		},
-		"trace_id": {
-			Name:          "trace_id",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextLog,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
+		"span_id": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "span_id",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			}}
 		},
-		"span_id": {
-			Name:          "span_id",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextLog,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
+		"trace_flags": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "trace_flags",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			}}
 		},
-		"trace_flags": {
-			Name:          "trace_flags",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextLog,
-			FieldDataType: telemetrytypes.FieldDataTypeNumber,
+		"severity_text": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "severity_text",
+				Description:   "Log level. Learn more [here](https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext)",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			}}
 		},
-		"severity_text": {
-			Name:          "severity_text",
-			Description:   "Log level. Learn more [here](https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitytext)",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextLog,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
+		"severity_number": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "severity_number",
+				Description:   "Numerical value of the severity. Learn more [here](https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber)",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			}}
 		},
-		"severity_number": {
-			Name:          "severity_number",
-			Description:   "Numerical value of the severity. Learn more [here](https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber)",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextLog,
-			FieldDataType: telemetrytypes.FieldDataTypeNumber,
+		"scope_name": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "scope_name",
+				Description:   "Logger name. Learn more about instrumentation scope [here](https://opentelemetry.io/docs/concepts/instrumentation-scope/)",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextScope,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			}}
 		},
-		"scope_name": {
-			Name:          "scope_name",
-			Description:   "Logger name. Learn more about instrumentation scope [here](https://opentelemetry.io/docs/concepts/instrumentation-scope/)",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextScope,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
-		},
-		"scope_version": {
-			Name:          "scope_version",
-			Signal:        telemetrytypes.SignalLogs,
-			FieldContext:  telemetrytypes.FieldContextScope,
-			FieldDataType: telemetrytypes.FieldDataTypeString,
+		"scope_version": func() []telemetrytypes.TelemetryFieldKey {
+			return []telemetrytypes.TelemetryFieldKey{{
+				Name:          "scope_version",
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextScope,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+			}}
 		},
 	}
 
@@ -117,4 +130,76 @@ var (
 			Direction: qbtypes.OrderDirectionDesc,
 		},
 	}
+
+	PsuedoIntrinsicFieldsInvalidOps = map[string][]qbtypes.FilterOperator{
+		jsonMergeExpr(): {
+			qbtypes.FilterOperatorExists,
+			qbtypes.FilterOperatorNotExists,
+		},
+	}
 )
+
+func jsonMergeExpr() string {
+	jsonStringExpressions := []string{}
+	for _, key := range []string{LogsV2BodyJSONColumn, LogsV2BodyPromotedColumn} {
+		jsonStringExpressions = append(jsonStringExpressions, fmt.Sprintf("toString(%s)", key))
+	}
+
+	return fmt.Sprintf("jsonMergePatch(%s)", strings.Join(jsonStringExpressions, ", "))
+}
+
+// bodyExpression returns the core ClickHouse expression representing the
+// conceptual "body" field when JSON body querying is enabled.
+//
+// NOTE: This intentionally does NOT include an alias so it can be reused
+// both in SELECT lists (with an "AS body" suffix) and in WHERE clauses.
+func bodyExpression() string {
+	if !querybuilder.BodyJSONQueryEnabled {
+		return LogsV2BodyColumn
+	}
+	return fmt.Sprintf("if(empty(%s), %s, %s)", LogsV2BodyColumn, LogsV2BodyColumn, jsonMergeExpr())
+}
+
+func bodyAliasExpression() string {
+	if !querybuilder.BodyJSONQueryEnabled {
+		return LogsV2BodyColumn
+	}
+
+	return fmt.Sprintf("%s as body", bodyExpression())
+}
+
+func buildBodyIntrinsicFields() []telemetrytypes.TelemetryFieldKey {
+	base := []telemetrytypes.TelemetryFieldKey{{
+		Name:          "body",
+		Signal:        telemetrytypes.SignalLogs,
+		FieldContext:  telemetrytypes.FieldContextLog,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+	}}
+
+	if querybuilder.BodyJSONQueryEnabled {
+		base = append(base, telemetrytypes.TelemetryFieldKey{
+			Name:          LogsV2BodyJSONColumn,
+			Signal:        telemetrytypes.SignalLogs,
+			FieldContext:  telemetrytypes.FieldContextLog,
+			FieldDataType: telemetrytypes.FieldDataTypeJSON,
+			LogicalName:   "body",
+		},
+			telemetrytypes.TelemetryFieldKey{
+				Name:          LogsV2BodyPromotedColumn,
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeJSON,
+				LogicalName:   "body",
+			},
+			telemetrytypes.TelemetryFieldKey{
+				Name:          jsonMergeExpr(),
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextLog,
+				FieldDataType: telemetrytypes.FieldDataTypeString,
+				LogicalName:   "body",
+			},
+		)
+	}
+
+	return base
+}

pkg/telemetrylogs/field_mapper.go

@@ -35,6 +35,7 @@ var (
 			MaxDynamicPaths: utils.ToPointer(uint(0)),
 		}},
 		LogsV2BodyPromotedColumn: {Name: LogsV2BodyPromotedColumn, Type: schema.JSONColumnType{}},
+		jsonMergeExpr():         {Name: jsonMergeExpr(), Type: schema.ColumnTypeString},
 		"attributes_string": {Name: "attributes_string", Type: schema.MapColumnType{
 			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
 			ValueType: schema.ColumnTypeString,
@@ -149,6 +150,9 @@ func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.Telemetr
 			}
 
 			return m.buildFieldForJSON(key)
+		case telemetrytypes.FieldContextLog:
+			// return the column name as is for log context fields
+			return column.Name, nil
 		default:
 			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String)
 		}

pkg/telemetrylogs/statement_builder.go

@@ -207,7 +207,8 @@ func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldK
 	// First check if it matches with any intrinsic fields
 	var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
 	if _, ok := IntrinsicFields[key.Name]; ok {
-		intrinsicOrCalculatedField = IntrinsicFields[key.Name]
+		// TODO(Piyush): Handle better
+		intrinsicOrCalculatedField = IntrinsicFields[key.Name]()[0]
 		return querybuilder.AdjustKey(key, keys, &intrinsicOrCalculatedField)
 	}
 
@@ -249,11 +250,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
 		sb.SelectMore(LogsV2SeverityNumberColumn)
 		sb.SelectMore(LogsV2ScopeNameColumn)
 		sb.SelectMore(LogsV2ScopeVersionColumn)
-		sb.SelectMore(LogsV2BodyColumn)
-		if querybuilder.BodyJSONQueryEnabled {
-			sb.SelectMore(LogsV2BodyJSONColumn)
-			sb.SelectMore(LogsV2BodyPromotedColumn)
-		}
+		sb.SelectMore(bodyAliasExpression())
 		sb.SelectMore(LogsV2AttributesStringColumn)
 		sb.SelectMore(LogsV2AttributesNumberColumn)
 		sb.SelectMore(LogsV2AttributesBoolColumn)

pkg/telemetrylogs/stmt_builder_test.go

@@ -634,191 +634,117 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
 	}
 }
 
-func TestAdjustKey(t *testing.T) {
+func TestStmtBuilderBodyField(t *testing.T) {
 	cases := []struct {
-		name        string
-		inputKey    telemetrytypes.TelemetryFieldKey
-		keysMap     map[string][]*telemetrytypes.TelemetryFieldKey
-		expectedKey telemetrytypes.TelemetryFieldKey
+		name                string
+		requestType         qbtypes.RequestType
+		query               qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
+		enableBodyJSONQuery bool
+		expected            qbtypes.Statement
+		expectedErr         error
+		expectWarn          bool
 	}{
 		{
-			name: "intrinsic field with no other key match - use intrinsic",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "severity_text",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			name:        "body_exists",
+			requestType: qbtypes.RequestTypeRaw,
+			query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
+				Signal: telemetrytypes.SignalLogs,
+				Filter: &qbtypes.Filter{Expression: "body Exists"},
+				Limit:  10,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: IntrinsicFields["severity_text"],
+			enableBodyJSONQuery: true,
+			expected: qbtypes.Statement{
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, if(empty(body), body, jsonMergePatch(toString(body_json), toString(body_json_promoted))) as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (body <> ? OR empty(body_json) = ? OR empty(body_json_promoted) = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Args:  []any{uint64(1747945619), uint64(1747983448), "", false, false, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
+			},
+			expectedErr: nil,
+			expectWarn:  true,
 		},
 		{
-			name: "intrinsic field with other key match - no override",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "body",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			name:        "body_exists_disabled",
+			requestType: qbtypes.RequestTypeRaw,
+			query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
+				Signal: telemetrytypes.SignalLogs,
+				Filter: &qbtypes.Filter{Expression: "body Exists"},
+				Limit:  10,
 			},
-			keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{
-				"body": {
-					{
-						Name:          "body",
-						FieldContext:  telemetrytypes.FieldContextBody,
-						FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-					},
-					{
-						Name:          "body",
-						FieldContext:  telemetrytypes.FieldContextAttribute,
-						FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-					},
-				},
-			},
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "body",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			enableBodyJSONQuery: false,
+			expected: qbtypes.Statement{
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Args:  []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
+			expectedErr: nil,
+			expectWarn:  false,
 		},
 		{
-			name: "json field with no context specified",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "severity_number",
-				FieldContext:  telemetrytypes.FieldContextBody,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			name:        "body_empty",
+			requestType: qbtypes.RequestTypeRaw,
+			query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
+				Signal: telemetrytypes.SignalLogs,
+				Filter: &qbtypes.Filter{Expression: "body == ''"},
+				Limit:  10,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "severity_number",
-				FieldContext:  telemetrytypes.FieldContextBody,
-				FieldDataType: telemetrytypes.FieldDataTypeNumber,
+			enableBodyJSONQuery: true,
+			expected: qbtypes.Statement{
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, if(empty(body), body, jsonMergePatch(toString(body_json), toString(body_json_promoted))) as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (body = ? OR (empty(body_json) = ?) OR (empty(body_json_promoted) = ?) OR (jsonMergePatch(toString(body_json), toString(body_json_promoted)) = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Args:  []any{uint64(1747945619), uint64(1747983448), "", false, false, "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
+			expectedErr: nil,
+			expectWarn:  true,
 		},
 		{
-			name: "single matching key in metadata",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "service.name",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			name:        "body_empty_disabled",
+			requestType: qbtypes.RequestTypeRaw,
+			query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
+				Signal: telemetrytypes.SignalLogs,
+				Filter: &qbtypes.Filter{Expression: "body == ''"},
+				Limit:  10,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
+			enableBodyJSONQuery: false,
+			expected: qbtypes.Statement{
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Args:  []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
+			},
+			expectedErr: nil,
+			expectWarn:  false,
 		},
 		{
-			name: "single matching key with incorrect context specified - no override",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "service.name",
-				FieldContext:  telemetrytypes.FieldContextAttribute,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			name:        "body_contains",
+			requestType: qbtypes.RequestTypeRaw,
+			query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
+				Signal: telemetrytypes.SignalLogs,
+				Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"},
+				Limit:  10,
 			},
-			keysMap: buildCompleteFieldKeyMap(),
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "service.name",
-				FieldContext:  telemetrytypes.FieldContextAttribute,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			enableBodyJSONQuery: true,
+			expected: qbtypes.Statement{
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, if(empty(body), body, jsonMergePatch(toString(body_json), toString(body_json_promoted))) as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(body) LIKE LOWER(?) OR (empty(body_json) = ?) OR (empty(body_json_promoted) = ?) OR (LOWER(jsonMergePatch(toString(body_json), toString(body_json_promoted))) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Args:  []any{uint64(1747945619), uint64(1747983448), "%error%", false, false, "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
+			expectedErr: nil,
+			expectWarn:  true,
 		},
 		{
-			name: "single matching key with no context specified - override",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "service.name",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			name:        "body_contains_disabled",
+			requestType: qbtypes.RequestTypeRaw,
+			query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
+				Signal: telemetrytypes.SignalLogs,
+				Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"},
+				Limit:  10,
 			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
-		},
-		{
-			name: "multiple matching keys - all materialized",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "multi.mat.key",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
+			enableBodyJSONQuery: false,
+			expected: qbtypes.Statement{
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Args:  []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
-			keysMap: buildCompleteFieldKeyMap(),
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "multi.mat.key",
-				FieldDataType: telemetrytypes.FieldDataTypeString,
-				Materialized:  true,
-			},
-		},
-		{
-			name: "multiple matching keys - mixed materialization",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "mixed.materialization.key",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-			},
-			keysMap: buildCompleteFieldKeyMap(),
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "mixed.materialization.key",
-				FieldDataType: telemetrytypes.FieldDataTypeString,
-				Materialized:  false,
-			},
-		},
-		{
-			name: "multiple matching keys with context specified",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "mixed.materialization.key",
-				FieldContext:  telemetrytypes.FieldContextAttribute,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["mixed.materialization.key"][0],
-		},
-		{
-			name: "no matching keys - unknown field",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "unknown.field",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-			},
-			keysMap: buildCompleteFieldKeyMap(),
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "unknown.field",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-				Materialized:  false,
-			},
-		},
-		{
-			name: "no matching keys with context filter",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "unknown.field",
-				FieldContext:  telemetrytypes.FieldContextAttribute,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-			},
-			keysMap: buildCompleteFieldKeyMap(),
-			expectedKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "unknown.field",
-				FieldContext:  telemetrytypes.FieldContextAttribute,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-				Materialized:  false,
-			},
-		},
-		{
-			name: "materialized field",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "mat.key",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["mat.key"][0],
-		},
-		{
-			name: "non-materialized field",
-			inputKey: telemetrytypes.TelemetryFieldKey{
-				Name:          "user.id",
-				FieldContext:  telemetrytypes.FieldContextUnspecified,
-				FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
-			},
-			keysMap:     buildCompleteFieldKeyMap(),
-			expectedKey: *buildCompleteFieldKeyMap()["user.id"][0],
+			expectedErr: nil,
+			expectWarn:  false,
 		},
 	}
 
-	fm := NewFieldMapper()
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
-	mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
+	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	fm := NewFieldMapper()
 	cb := NewConditionBuilder(fm)
 
 	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
@@ -838,19 +764,26 @@ func TestAdjustKey(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			// Create a copy of the input key to avoid modifying the original
-			key := c.inputKey
+			if c.enableBodyJSONQuery {
+				enableBodyJSONQuery(t)
+			} else {
+				disableBodyJSONQuery(t)
+			}
+			// rebuild the key map after enabling/disabling body JSON query
+			mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
 
-			// Call adjustKey
-			statementBuilder.adjustKey(&key, c.keysMap)
-
-			// Verify the key was adjusted as expected
-			require.Equal(t, c.expectedKey.Name, key.Name, "key name should match")
-			require.Equal(t, c.expectedKey.FieldContext, key.FieldContext, "field context should match")
-			require.Equal(t, c.expectedKey.FieldDataType, key.FieldDataType, "field data type should match")
-			require.Equal(t, c.expectedKey.Materialized, key.Materialized, "materialized should match")
-			require.Equal(t, c.expectedKey.JSONDataType, key.JSONDataType, "json data type should match")
-			require.Equal(t, c.expectedKey.Indexes, key.Indexes, "json exists should match")
+			q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
+			if c.expectedErr != nil {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), c.expectedErr.Error())
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, c.expected.Query, q.Query)
+				require.Equal(t, c.expected.Args, q.Args)
+				if c.expectWarn {
+					require.True(t, len(q.Warnings) > 0)
+				}
+			}
 		})
 	}
 }

pkg/telemetrylogs/test_data.go

@@ -27,13 +27,6 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
 				FieldDataType: telemetrytypes.FieldDataTypeString,
 			},
 		},
-		"body": {
-			{
-				Name:          "body",
-				FieldContext:  telemetrytypes.FieldContextLog,
-				FieldDataType: telemetrytypes.FieldDataTypeString,
-			},
-		},
 		"http.status_code": {
 			{
 				Name:          "http.status_code",
@@ -945,6 +938,13 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
 			key.Signal = telemetrytypes.SignalLogs
 		}
 	}
+
+	// add intrinsic fields to the map
+	for fieldName, keys := range IntrinsicFields {
+		for _, key := range keys() {
+			keysMap[fieldName] = append(keysMap[fieldName], &key)
+		}
+	}
 	return keysMap
 }
 

pkg/telemetrymetadata/metadata.go

@@ -540,9 +540,11 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 		}
 
 		if found {
-			if field, exists := telemetrylogs.IntrinsicFields[key]; exists {
-				if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
-					keys = append(keys, &field)
+			if fields, exists := telemetrylogs.IntrinsicFields[key]; exists {
+				for _, field := range fields() {
+					if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
+						keys = append(keys, &field)
+					}
 				}
 				continue
 			}

pkg/types/querybuildertypes/querybuildertypesv5/builder_elements.go

@@ -159,6 +159,15 @@ func (f FilterOperator) IsStringSearchOperator() bool {
 	}
 }
 
+func (f FilterOperator) IsOpValidForJSON() bool {
+	switch f {
+	case FilterOperatorExists, FilterOperatorNotExists, FilterOperatorContains, FilterOperatorNotContains:
+		return true
+	default:
+		return false
+	}
+}
+
 type OrderDirection struct {
 	valuer.String
 }

pkg/types/querybuildertypes/querybuildertypesv5/qb.go

@@ -28,6 +28,10 @@ type FieldMapper interface {
 	ColumnExpressionFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, error)
 }
 
+// ConditionForOpFunc is a function that builds the condition for a specific operator and value.
+type ConditionForOpFunc func(sb *sqlbuilder.SelectBuilder, value any) (string, error)
+type OperatorToConditionRegistry map[FilterOperator]ConditionForOpFunc
+
 // ConditionBuilder builds the condition for the filter.
 type ConditionBuilder interface {
 	// ConditionFor returns the condition for the given key, operator and value.

pkg/types/telemetrytypes/field.go

@@ -35,6 +35,7 @@ type TelemetryFieldKey struct {
 	FieldContext  FieldContext  `json:"fieldContext,omitempty"`
 	FieldDataType FieldDataType `json:"fieldDataType,omitempty"`
 
+	LogicalName  string              `json:"-"` // refers to fields that are to be considered same as Name for logical fields
 	JSONDataType *JSONDataType       `json:"-"`
 	JSONPlan     JSONAccessPlan      `json:"-"`
 	Indexes      []JSONDataTypeIndex `json:"-"`
@@ -156,6 +157,10 @@ func (f *TelemetryFieldKey) Normalize() {
 
 }
 
+func (f *TelemetryFieldKey) IsPsuedoKey() bool {
+	return f.LogicalName != ""
+}
+
 // GetFieldKeyFromKeyText returns a TelemetryFieldKey from a key text.
 // The key text is expected to be in the format of `fieldContext.fieldName:fieldDataType` in the search query.
 // Both fieldContext and :fieldDataType are optional.

pkg/types/telemetrytypes/field_datatype.go

@@ -21,6 +21,7 @@ var (
 	// int64 and number are synonyms for float64
 	FieldDataTypeInt64       = FieldDataType{valuer.NewString("int64")}
 	FieldDataTypeNumber      = FieldDataType{valuer.NewString("number")}
+	FieldDataTypeJSON        = FieldDataType{valuer.NewString("json")}
 	FieldDataTypeUnspecified = FieldDataType{valuer.NewString("")}
 
 	FieldDataTypeArrayString  = FieldDataType{valuer.NewString("[]string")}

pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go

@@ -177,7 +177,7 @@ func matchesKey(selector *telemetrytypes.FieldKeySelector, key *telemetrytypes.T
 	}
 
 	// Check name (already checked in matchesName, but double-check here)
-	if selector.Name != "" && !matchesName(selector, key.Name) {
+	if selector.Name != "" && !(matchesName(selector, key.Name) || matchesName(selector, key.LogicalName)) {
 		return false
 	}