Merge branch 'main' into feat/billing

* refactor(telemetrytypes): replace JSONDataTypeIndex with TelemetryFieldKeySkipIndex

* revert: mcp related changes

* fix: openapi fails

---------

Co-authored-by: Ankit Nayan <ankit@signoz.io>

docs/api/openapi.yml

@@ -2933,8 +2933,8 @@ components:
       type: object
     PromotetypesWrappedIndex:
       properties:
-        column_type:
-          type: string
+        fieldDataType:
+          $ref: '#/components/schemas/TelemetrytypesFieldDataType'
         granularity:
           type: integer
         type:

ee/query-service/app/api/license.go

@@ -5,12 +5,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/SigNoz/signoz/ee/query-service/constants"
 	"github.com/SigNoz/signoz/ee/query-service/model"
-	"github.com/SigNoz/signoz/pkg/flagger"
-	"github.com/SigNoz/signoz/pkg/types/authtypes"
-	"github.com/SigNoz/signoz/pkg/types/featuretypes"
-	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
 type DayWiseBreakdown struct {
@@ -70,53 +65,18 @@ func (ah *APIHandler) getBilling(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	claims, err := authtypes.ClaimsFromContext(r.Context())
+	data, err := ah.Signoz.Zeus.GetMeters(r.Context(), licenseKey)
 	if err != nil {
 		RespondError(w, model.InternalError(err), nil)
 		return
 	}
 
-	orgID := valuer.MustNewUUID(claims.OrgID)
-	evalCtx := featuretypes.NewFlaggerEvaluationContext(orgID)
-	useZeus := ah.Signoz.Flagger.BooleanOrEmpty(r.Context(), flagger.FeatureGetMetersFromZeus, evalCtx)
-
-	if useZeus {
-		data, err := ah.Signoz.Zeus.GetMeters(r.Context(), licenseKey)
-		if err != nil {
-			RespondError(w, model.InternalError(err), nil)
-			return
-		}
-
-		var billing billingData
-		if err := json.Unmarshal(data, &billing); err != nil {
-			RespondError(w, model.InternalError(err), nil)
-			return
-		}
-
-		ah.Respond(w, billing)
-		return
-	}
-
-	billingURL := fmt.Sprintf("%s/usage?licenseKey=%s", constants.LicenseSignozIo, licenseKey)
-
-	hClient := &http.Client{}
-	req, err := http.NewRequest("GET", billingURL, nil)
-	if err != nil {
-		RespondError(w, model.InternalError(err), nil)
-		return
-	}
-	req.Header.Add("X-SigNoz-SecretKey", constants.LicenseAPIKey)
-	billingResp, err := hClient.Do(req)
-	if err != nil {
+	var billing billingData
+	if err := json.Unmarshal(data, &billing); err != nil {
 		RespondError(w, model.InternalError(err), nil)
 		return
 	}
 
-	var billingResponse billingDetails
-	if err := json.NewDecoder(billingResp.Body).Decode(&billingResponse); err != nil {
-		RespondError(w, model.InternalError(err), nil)
-		return
-	}
-
-	ah.Respond(w, billingResponse.Data)
+	ah.Respond(w, billing)
+	return
 }

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -3701,10 +3701,7 @@ export interface PromotetypesPromotePathDTO {
 }
 
 export interface PromotetypesWrappedIndexDTO {
-	/**
-	 * @type string
-	 */
-	column_type?: string;
+	fieldDataType?: TelemetrytypesFieldDataTypeDTO;
 	/**
 	 * @type integer
 	 */

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.40.1
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.144.3-rc.4
+	github.com/SigNoz/signoz-otel-collector v0.144.3
 	github.com/antlr4-go/antlr/v4 v4.13.1
 	github.com/antonmedv/expr v1.15.3
 	github.com/cespare/xxhash/v2 v2.3.0

go.sum

@@ -108,8 +108,8 @@ github.com/SigNoz/expr v1.17.7-beta h1:FyZkleM5dTQ0O6muQfwGpoH5A2ohmN/XTasRCO72g
 github.com/SigNoz/expr v1.17.7-beta/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkbj57eGXx8H3ZJ4zhmQXBnrW523ktj8=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
-github.com/SigNoz/signoz-otel-collector v0.144.3-rc.4 h1:EskJkEMfuuIyArWhV8SleDV/fuKxiaEGTXrCZIFqDT4=
-github.com/SigNoz/signoz-otel-collector v0.144.3-rc.4/go.mod h1:9pLVpcIQvUT88ZiNnZN/MI+XLruvwC+Xm2UzPmGjNfA=
+github.com/SigNoz/signoz-otel-collector v0.144.3 h1:/7PPIqIpPsaWtrgnfHam2XVYP41ZlgEKLHzQO8oVxcA=
+github.com/SigNoz/signoz-otel-collector v0.144.3/go.mod h1:9pLVpcIQvUT88ZiNnZN/MI+XLruvwC+Xm2UzPmGjNfA=
 github.com/Yiling-J/theine-go v0.6.2 h1:1GeoXeQ0O0AUkiwj2S9Jc0Mzx+hpqzmqsJ4kIC4M9AY=
 github.com/Yiling-J/theine-go v0.6.2/go.mod h1:08QpMa5JZ2pKN+UJCRrCasWYO1IKCdl54Xa836rpmDU=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=

pkg/flagger/registry.go

@@ -3,11 +3,10 @@ package flagger
 import "github.com/SigNoz/signoz/pkg/types/featuretypes"
 
 var (
-	FeatureUseSpanMetrics    = featuretypes.MustNewName("use_span_metrics")
-	FeatureKafkaSpanEval     = featuretypes.MustNewName("kafka_span_eval")
-	FeatureHideRootUser      = featuretypes.MustNewName("hide_root_user")
-	FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
-	FeaturePutMetersInZeus   = featuretypes.MustNewName("put_meters_in_zeus")
+	FeatureUseSpanMetrics  = featuretypes.MustNewName("use_span_metrics")
+	FeatureKafkaSpanEval   = featuretypes.MustNewName("kafka_span_eval")
+	FeatureHideRootUser    = featuretypes.MustNewName("hide_root_user")
+	FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
 )
 
 func MustNewRegistry() featuretypes.Registry {
@@ -36,14 +35,6 @@ func MustNewRegistry() featuretypes.Registry {
 			DefaultVariant: featuretypes.MustNewName("disabled"),
 			Variants:       featuretypes.NewBooleanVariants(),
 		},
-		&featuretypes.Feature{
-			Name:           FeatureGetMetersFromZeus,
-			Kind:           featuretypes.KindBoolean,
-			Stage:          featuretypes.StageExperimental,
-			Description:    "Controls whether billing details are fetched from Zeus instead of the legacy subscriptions service",
-			DefaultVariant: featuretypes.MustNewName("disabled"),
-			Variants:       featuretypes.NewBooleanVariants(),
-		},
 		&featuretypes.Feature{
 			Name:           FeaturePutMetersInZeus,
 			Kind:           featuretypes.KindBoolean,

pkg/modules/promote/implpromote/module.go

@@ -32,27 +32,17 @@ func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore teleme
 }
 
 func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error) {
-	logsIndexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
+	indexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
 	if err != nil {
 		return nil, err
 	}
-	// Flatten the map values (which are slices) into a single slice
-	indexes := slices.Concat(slices.Collect(maps.Values(logsIndexes))...)
 
 	aggr := map[string][]promotetypes.WrappedIndex{}
 	for _, index := range indexes {
-		path, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(index.Expression)
-		if err != nil {
-			return nil, err
-		}
-
-		// clean backticks from the path
-		path = strings.ReplaceAll(path, "`", "")
-
-		aggr[path] = append(aggr[path], promotetypes.WrappedIndex{
-			ColumnType:  columnType,
-			Type:        index.Type,
-			Granularity: index.Granularity,
+		aggr[index.Name] = append(aggr[index.Name], promotetypes.WrappedIndex{
+			FieldDataType: index.FieldDataType,
+			Type:          index.IndexType,
+			Granularity:   index.Granularity,
 		})
 	}
 	promotedPaths, err := m.listPromotedPaths(ctx)

pkg/querybuilder/collision.go

@@ -204,7 +204,7 @@ func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemet
 		// Downstream query builder should handle multiple matching keys with their own metadata
 		// and not rely on this function to do so.
 		materialized := true
-		indexes := []telemetrytypes.JSONDataTypeIndex{}
+		indexes := []telemetrytypes.TelemetryFieldKeySkipIndex{}
 		fieldContextsSeen := map[telemetrytypes.FieldContext]bool{}
 		dataTypesSeen := map[telemetrytypes.FieldDataType]bool{}
 		for _, matchingKey := range matchingKeys {

pkg/telemetrylogs/json_condition_builder.go

@@ -195,8 +195,8 @@ func (c *jsonConditionBuilder) buildPrimitiveTerminalCondition(node *telemetryty
 	// the field genuinely holds the empty/zero value.
 	//
 	// Note: indexing is also skipped for Array Nested fields because they cannot be indexed.
-	indexed := slices.ContainsFunc(node.TerminalConfig.Key.Indexes, func(index telemetrytypes.JSONDataTypeIndex) bool {
-		return index.Type == node.TerminalConfig.ElemType
+	indexed := slices.ContainsFunc(node.TerminalConfig.Key.Indexes, func(index telemetrytypes.TelemetryFieldKeySkipIndex) bool {
+		return telemetrytypes.MappingFieldDataTypeToJSONDataType[index.FieldDataType] == node.TerminalConfig.ElemType
 	})
 	isExistsCheck := operator == qbtypes.FilterOperatorExists || operator == qbtypes.FilterOperatorNotExists
 	if node.TerminalConfig.ElemType.IndexSupported && indexed && !isExistsCheck {

pkg/telemetrylogs/json_stmt_builder_test.go

@@ -1127,9 +1127,12 @@ func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetryty
 					return entry.Path == path && entry.Type == jsonType
 				})
 				if idx >= 0 {
-					key.Indexes = append(key.Indexes, telemetrytypes.JSONDataTypeIndex{
-						Type:             jsonType,
-						ColumnExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
+					key.Indexes = append(key.Indexes, telemetrytypes.TelemetryFieldKeySkipIndex{
+						Name:            path,
+						FieldContext:    telemetrytypes.FieldContextBody,
+						FieldDataType:   fdt,
+						BaseColumn:      LogsV2BodyV2Column,
+						IndexExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
 					})
 				}
 			}

pkg/telemetrymetadata/body_json_metadata.go

@@ -106,7 +106,7 @@ func (t *telemetryMetaStore) buildJSONPlans(keys []*telemetrytypes.TelemetryFiel
 	return nil
 }
 
-func (t *telemetryMetaStore) getJSONPathIndexes(ctx context.Context, paths ...string) (map[string][]telemetrytypes.JSONDataTypeIndex, error) {
+func (t *telemetryMetaStore) getJSONPathIndexes(ctx context.Context, paths ...string) (map[string][]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
 	filteredPaths := []string{}
 	for _, path := range paths {
 		// skip array paths; since they don't have any indexes
@@ -116,47 +116,22 @@ func (t *telemetryMetaStore) getJSONPathIndexes(ctx context.Context, paths ...st
 		filteredPaths = append(filteredPaths, path)
 	}
 	if len(filteredPaths) == 0 {
-		return make(map[string][]telemetrytypes.JSONDataTypeIndex), nil
+		return make(map[string][]telemetrytypes.TelemetryFieldKeySkipIndex), nil
 	}
 
 	// list indexes for the paths
-	indexesMap, err := t.ListLogsJSONIndexes(ctx, filteredPaths...)
+	indexes, err := t.ListLogsJSONIndexes(ctx, filteredPaths...)
 	if err != nil {
 		return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to list JSON path indexes")
 	}
 
 	// build a set of indexes
-	cleanIndexes := make(map[string][]telemetrytypes.JSONDataTypeIndex)
-	for path, indexes := range indexesMap {
-		for _, index := range indexes {
-			columnExpr, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(index.Expression)
-			if err != nil {
-				return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to unfold JSON sub column index expression: %s", index.Expression)
-			}
-
-			jsonDataType, found := telemetrytypes.MappingStringToJSONDataType[columnType]
-			if !found {
-				t.logger.ErrorContext(ctx, "failed to map column type to JSON data type", slog.String("column_type", columnType), slog.String("column_expr", columnExpr))
-				continue
-			}
-
-			if jsonDataType == telemetrytypes.String {
-				cleanIndexes[path] = append(cleanIndexes[path], telemetrytypes.JSONDataTypeIndex{
-					Type:             telemetrytypes.String,
-					ColumnExpression: columnExpr,
-					IndexExpression:  index.Expression,
-				})
-			} else if strings.HasPrefix(index.Type, "minmax") {
-				cleanIndexes[path] = append(cleanIndexes[path], telemetrytypes.JSONDataTypeIndex{
-					Type:             jsonDataType,
-					ColumnExpression: columnExpr,
-					IndexExpression:  index.Expression,
-				})
-			}
-		}
+	fieldPathToIndexes := make(map[string][]telemetrytypes.TelemetryFieldKeySkipIndex)
+	for _, index := range indexes {
+		fieldPathToIndexes[index.Name] = append(fieldPathToIndexes[index.Name], index)
 	}
 
-	return cleanIndexes, nil
+	return fieldPathToIndexes, nil
 }
 
 func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, []any) {
@@ -173,14 +148,15 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
 
 	filterExprs := []string{}
 	for _, filter := range filters {
-		filterExprs = append(filterExprs, sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(filter))))
+		// Remove backticks from actual expr cuz paths from metadata doesn't have backticks
+		filterExprs = append(filterExprs, sb.ILike("replaceAll(expr, '`', '')", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(filter))))
 	}
 	sb.Where(sb.Or(filterExprs...))
 
 	return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
 }
 
-func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) (map[string][]schemamigrator.Index, error) {
+func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
 	ctx = withTelemetryContext(ctx, "ListLogsJSONIndexes")
 	query, args := buildListLogsJSONIndexesQuery(t.telemetrystore.Cluster(), filters...)
 	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
@@ -189,7 +165,7 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
 	}
 	defer rows.Close()
 
-	indexes := make(map[string][]schemamigrator.Index)
+	indexes := []telemetrytypes.TelemetryFieldKeySkipIndex{}
 	for rows.Next() {
 		var name string
 		var typeFull string
@@ -198,11 +174,39 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
 		if err := rows.Scan(&name, &typeFull, &expr, &granularity); err != nil {
 			return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to scan string indexed column")
 		}
-		indexes[name] = append(indexes[name], schemamigrator.Index{
-			Name:        name,
-			Type:        typeFull,
-			Expression:  expr,
-			Granularity: int(granularity),
+
+		columnExpr, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(expr)
+		if err != nil {
+			return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to unfold JSON sub column index expression: %s", expr)
+		}
+
+		fdt, found := telemetrytypes.MappingJSONDataTypeToFieldDataType[columnType]
+		if !found {
+			t.logger.ErrorContext(ctx, "failed to map JSON data type to field data type", slog.String("column_type", columnType), slog.String("column_expr", columnExpr))
+			continue
+		}
+
+		baseColumn := ""
+		fieldName := ""
+		switch {
+		case strings.HasPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix):
+			baseColumn = telemetrylogs.BodyV2ColumnPrefix
+			fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix)
+		case strings.HasPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix):
+			baseColumn = telemetrylogs.BodyPromotedColumnPrefix
+			fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix)
+		}
+		fieldName = strings.ReplaceAll(fieldName, "`", "")
+
+		indexes = append(indexes, telemetrytypes.TelemetryFieldKeySkipIndex{
+			Name:            fieldName,
+			FieldContext:    telemetrytypes.FieldContextBody,
+			FieldDataType:   fdt,
+			BaseColumn:      baseColumn,
+			IndexName:       name,
+			IndexType:       typeFull,
+			IndexExpression: expr,
+			Granularity:     int(granularity),
 		})
 	}
 

pkg/telemetrymetadata/body_json_metadata_test.go

@@ -36,7 +36,7 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
 			cluster: "test-cluster",
 			filters: []string{"foo", "bar"},
 			expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
-				"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?))",
+				"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?) OR LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?))",
 			expectedArgs: []any{
 				telemetrylogs.DBName,
 				telemetrylogs.LogsV2LocalTableName,

pkg/types/promotetypes/types.go

@@ -10,10 +10,10 @@ import (
 )
 
 type WrappedIndex struct {
-	JSONDataType telemetrytypes.JSONDataType `json:"-"`
-	ColumnType   string                      `json:"column_type"`
-	Type         string                      `json:"type"`
-	Granularity  int                         `json:"granularity"`
+	JSONDataType  telemetrytypes.JSONDataType  `json:"-"`
+	FieldDataType telemetrytypes.FieldDataType `json:"fieldDataType"`
+	Type          string                       `json:"type"`
+	Granularity   int                          `json:"granularity"`
 }
 
 type PromotePath struct {
@@ -60,12 +60,12 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
 			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index granularity must be greater than 0")
 		}
 
-		jsonDataType, ok := telemetrytypes.MappingStringToJSONDataType[index.ColumnType]
+		jsonDataType, ok := telemetrytypes.MappingFieldDataTypeToJSONDataType[index.FieldDataType]
 		if !ok {
-			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid column type: %s", index.ColumnType)
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid column type: %s", index.FieldDataType)
 		}
 		if !jsonDataType.IndexSupported {
-			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index is not supported for column type: %s", index.ColumnType)
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index is not supported for column type: %s", index.FieldDataType)
 		}
 
 		i.Indexes[idx].JSONDataType = jsonDataType

pkg/types/telemetrytypes/field.go

@@ -37,9 +37,9 @@ type TelemetryFieldKey struct {
 	FieldContext  FieldContext  `json:"fieldContext,omitzero"`
 	FieldDataType FieldDataType `json:"fieldDataType,omitzero"`
 
-	JSONPlan     JSONAccessPlan      `json:"-"`
-	Indexes      []JSONDataTypeIndex `json:"-"`
-	Materialized bool                `json:"-"` // refers to promoted in case of body.... fields
+	JSONPlan     JSONAccessPlan               `json:"-"`
+	Indexes      []TelemetryFieldKeySkipIndex `json:"-"`
+	Materialized bool                         `json:"-"` // refers to promoted in case of body.... fields
 
 	Evolutions []*EvolutionEntry `json:"-"`
 }
@@ -102,7 +102,7 @@ func (f TelemetryFieldKey) String() string {
 			if i > 0 {
 				sb.WriteString("; ")
 			}
-			fmt.Fprintf(&sb, "{type=%s, columnExpr=%s, indexExpr=%s}", index.Type.StringValue(), index.ColumnExpression, index.IndexExpression)
+			fmt.Fprintf(&sb, "{type=%s, indexExpr=%s}", MappingFieldDataTypeToJSONDataType[index.FieldDataType].StringValue(), index.IndexExpression)
 		}
 		sb.WriteString("]")
 	}
@@ -400,3 +400,14 @@ func NewFieldValueSelectorFromPostableFieldValueParams(params PostableFieldValue
 
 	return fieldValueSelector
 }
+
+type TelemetryFieldKeySkipIndex struct {
+	Name            string        `json:"name"` // Name is TelemetryFieldKey.Name not IndexName from ClickHouse
+	FieldContext    FieldContext  `json:"fieldContext,omitzero"`
+	FieldDataType   FieldDataType `json:"fieldDataType,omitzero"`
+	BaseColumn      string        `json:"baseColumn"`
+	IndexName       string        `json:"indexName"`
+	IndexType       string        `json:"indexType"`
+	IndexExpression string        `json:"indexExpression"`
+	Granularity     int           `json:"granularity"`
+}

pkg/types/telemetrytypes/json_datatype.go

@@ -1,11 +1,5 @@
 package telemetrytypes
 
-type JSONDataTypeIndex struct {
-	Type             JSONDataType
-	ColumnExpression string
-	IndexExpression  string
-}
-
 type JSONDataType struct {
 	str            string // Store the correct case for ClickHouse
 	IsArray        bool
@@ -32,18 +26,17 @@ var (
 	ArrayJSON    = JSONDataType{"Array(JSON)", true, "JSON", false}
 )
 
-var MappingStringToJSONDataType = map[string]JSONDataType{
-	"String":                   String,
-	"Int64":                    Int64,
-	"Float64":                  Float64,
-	"Bool":                     Bool,
-	"Dynamic":                  Dynamic,
-	"Array(Nullable(String))":  ArrayString,
-	"Array(Nullable(Int64))":   ArrayInt64,
-	"Array(Nullable(Float64))": ArrayFloat64,
-	"Array(Nullable(Bool))":    ArrayBool,
-	"Array(Dynamic)":           ArrayDynamic,
-	"Array(JSON)":              ArrayJSON,
+var MappingJSONDataTypeToFieldDataType = map[string]FieldDataType{
+	"String":                   FieldDataTypeString,
+	"Int64":                    FieldDataTypeInt64,
+	"Float64":                  FieldDataTypeFloat64,
+	"Bool":                     FieldDataTypeBool,
+	"Array(Nullable(String))":  FieldDataTypeArrayString,
+	"Array(Nullable(Int64))":   FieldDataTypeArrayInt64,
+	"Array(Nullable(Float64))": FieldDataTypeArrayFloat64,
+	"Array(Nullable(Bool))":    FieldDataTypeArrayBool,
+	"Array(Dynamic)":           FieldDataTypeArrayDynamic,
+	"Array(JSON)":              FieldDataTypeArrayJSON,
 }
 
 var MappingFieldDataTypeToJSONDataType = map[FieldDataType]JSONDataType{

pkg/types/telemetrytypes/store.go

@@ -3,7 +3,6 @@ package telemetrytypes
 import (
 	"context"
 
-	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
 	"github.com/SigNoz/signoz/pkg/types/metrictypes"
 )
 
@@ -35,7 +34,7 @@ type MetadataStore interface {
 	FetchTemporalityAndTypeMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error)
 
 	// ListLogsJSONIndexes lists the JSON indexes for the logs table.
-	ListLogsJSONIndexes(ctx context.Context, filters ...string) (map[string][]schemamigrator.Index, error)
+	ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]TelemetryFieldKeySkipIndex, error)
 
 	// ListPromotedPaths lists the promoted paths.
 	GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error)

pkg/types/telemetrytypes/telemetrytypestest/metadata_store.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"strings"
 
-	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
 	"github.com/SigNoz/signoz/pkg/types/metrictypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
@@ -18,7 +17,7 @@ type MockMetadataStore struct {
 	TemporalityMap             map[string]metrictypes.Temporality
 	TypeMap                    map[string]metrictypes.Type
 	PromotedPathsMap           map[string]bool
-	LogsJSONIndexesMap         map[string][]schemamigrator.Index
+	LogsJSONIndexes            []telemetrytypes.TelemetryFieldKeySkipIndex
 	ColumnEvolutionMetadataMap map[string][]*telemetrytypes.EvolutionEntry
 	LookupKeysMap              map[telemetrytypes.MetricMetadataLookupKey]int64
 	// StaticFields holds signal-specific intrinsic field definitions (e.g. telemetrylogs.IntrinsicFields).
@@ -34,7 +33,7 @@ func NewMockMetadataStore() *MockMetadataStore {
 		TemporalityMap:             make(map[string]metrictypes.Temporality),
 		TypeMap:                    make(map[string]metrictypes.Type),
 		PromotedPathsMap:           make(map[string]bool),
-		LogsJSONIndexesMap:         make(map[string][]schemamigrator.Index),
+		LogsJSONIndexes:            []telemetrytypes.TelemetryFieldKeySkipIndex{},
 		ColumnEvolutionMetadataMap: make(map[string][]*telemetrytypes.EvolutionEntry),
 		LookupKeysMap:              make(map[telemetrytypes.MetricMetadataLookupKey]int64),
 		StaticFields:               make(map[string]telemetrytypes.TelemetryFieldKey),
@@ -369,8 +368,8 @@ func (m *MockMetadataStore) GetPromotedPaths(ctx context.Context, paths ...strin
 }
 
 // ListLogsJSONIndexes lists the JSON indexes for the logs table.
-func (m *MockMetadataStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) (map[string][]schemamigrator.Index, error) {
-	return m.LogsJSONIndexesMap, nil
+func (m *MockMetadataStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
+	return m.LogsJSONIndexes, nil
 }
 
 func (m *MockMetadataStore) updateColumnEvolutionMetadataForKeys(_ context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) map[string][]*telemetrytypes.EvolutionEntry {