feat: add support for scope fields in traces

* feat: adding query params in cloud integration APIs

* refactor: create account HTTP status change from OK to CREATED

docs/api/openapi.yml

@@ -3309,7 +3309,7 @@ paths:
             schema:
               $ref: '#/components/schemas/CloudintegrationtypesPostableAccount'
       responses:
-        "200":
+        "201":
           content:
             application/json:
               schema:
@@ -3322,7 +3322,7 @@ paths:
                 - status
                 - data
                 type: object
-          description: OK
+          description: Created
         "401":
           content:
             application/json:
@@ -3683,6 +3683,11 @@ paths:
         provider
       operationId: ListServicesMetadata
       parameters:
+      - in: query
+        name: cloud_integration_id
+        required: false
+        schema:
+          type: string
       - in: path
         name: cloud_provider
         required: true
@@ -3735,6 +3740,11 @@ paths:
       description: This endpoint gets a service for the specified cloud provider
       operationId: GetService
       parameters:
+      - in: query
+        name: cloud_integration_id
+        required: false
+        schema:
+          type: string
       - in: path
         name: cloud_provider
         required: true

frontend/src/api/generated/services/cloudintegration/index.ts

@@ -28,7 +28,7 @@ import type {
 	CloudintegrationtypesPostableAgentCheckInDTO,
 	CloudintegrationtypesUpdatableAccountDTO,
 	CloudintegrationtypesUpdatableServiceDTO,
-	CreateAccount200,
+	CreateAccount201,
 	CreateAccountPathParameters,
 	DisconnectAccountPathParameters,
 	GetAccount200,
@@ -36,10 +36,12 @@ import type {
 	GetConnectionCredentials200,
 	GetConnectionCredentialsPathParameters,
 	GetService200,
+	GetServiceParams,
 	GetServicePathParameters,
 	ListAccounts200,
 	ListAccountsPathParameters,
 	ListServicesMetadata200,
+	ListServicesMetadataParams,
 	ListServicesMetadataPathParameters,
 	RenderErrorResponseDTO,
 	UpdateAccountPathParameters,
@@ -260,7 +262,7 @@ export const createAccount = (
 	cloudintegrationtypesPostableAccountDTO: BodyType<CloudintegrationtypesPostableAccountDTO>,
 	signal?: AbortSignal,
 ) => {
-	return GeneratedAPIInstance<CreateAccount200>({
+	return GeneratedAPIInstance<CreateAccount201>({
 		url: `/api/v1/cloud_integrations/${cloudProvider}/accounts`,
 		method: 'POST',
 		headers: { 'Content-Type': 'application/json' },
@@ -940,19 +942,25 @@ export const invalidateGetConnectionCredentials = async (
  */
 export const listServicesMetadata = (
 	{ cloudProvider }: ListServicesMetadataPathParameters,
+	params?: ListServicesMetadataParams,
 	signal?: AbortSignal,
 ) => {
 	return GeneratedAPIInstance<ListServicesMetadata200>({
 		url: `/api/v1/cloud_integrations/${cloudProvider}/services`,
 		method: 'GET',
+		params,
 		signal,
 	});
 };
 
-export const getListServicesMetadataQueryKey = ({
-	cloudProvider,
-}: ListServicesMetadataPathParameters) => {
-	return [`/api/v1/cloud_integrations/${cloudProvider}/services`] as const;
+export const getListServicesMetadataQueryKey = (
+	{ cloudProvider }: ListServicesMetadataPathParameters,
+	params?: ListServicesMetadataParams,
+) => {
+	return [
+		`/api/v1/cloud_integrations/${cloudProvider}/services`,
+		...(params ? [params] : []),
+	] as const;
 };
 
 export const getListServicesMetadataQueryOptions = <
@@ -960,6 +968,7 @@ export const getListServicesMetadataQueryOptions = <
 	TError = ErrorType<RenderErrorResponseDTO>
 >(
 	{ cloudProvider }: ListServicesMetadataPathParameters,
+	params?: ListServicesMetadataParams,
 	options?: {
 		query?: UseQueryOptions<
 			Awaited<ReturnType<typeof listServicesMetadata>>,
@@ -971,11 +980,12 @@ export const getListServicesMetadataQueryOptions = <
 	const { query: queryOptions } = options ?? {};
 
 	const queryKey =
-		queryOptions?.queryKey ?? getListServicesMetadataQueryKey({ cloudProvider });
+		queryOptions?.queryKey ??
+		getListServicesMetadataQueryKey({ cloudProvider }, params);
 
 	const queryFn: QueryFunction<
 		Awaited<ReturnType<typeof listServicesMetadata>>
-	> = ({ signal }) => listServicesMetadata({ cloudProvider }, signal);
+	> = ({ signal }) => listServicesMetadata({ cloudProvider }, params, signal);
 
 	return {
 		queryKey,
@@ -1003,6 +1013,7 @@ export function useListServicesMetadata<
 	TError = ErrorType<RenderErrorResponseDTO>
 >(
 	{ cloudProvider }: ListServicesMetadataPathParameters,
+	params?: ListServicesMetadataParams,
 	options?: {
 		query?: UseQueryOptions<
 			Awaited<ReturnType<typeof listServicesMetadata>>,
@@ -1013,6 +1024,7 @@ export function useListServicesMetadata<
 ): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
 	const queryOptions = getListServicesMetadataQueryOptions(
 		{ cloudProvider },
+		params,
 		options,
 	);
 
@@ -1031,10 +1043,11 @@ export function useListServicesMetadata<
 export const invalidateListServicesMetadata = async (
 	queryClient: QueryClient,
 	{ cloudProvider }: ListServicesMetadataPathParameters,
+	params?: ListServicesMetadataParams,
 	options?: InvalidateOptions,
 ): Promise<QueryClient> => {
 	await queryClient.invalidateQueries(
-		{ queryKey: getListServicesMetadataQueryKey({ cloudProvider }) },
+		{ queryKey: getListServicesMetadataQueryKey({ cloudProvider }, params) },
 		options,
 	);
 
@@ -1047,21 +1060,24 @@ export const invalidateListServicesMetadata = async (
  */
 export const getService = (
 	{ cloudProvider, serviceId }: GetServicePathParameters,
+	params?: GetServiceParams,
 	signal?: AbortSignal,
 ) => {
 	return GeneratedAPIInstance<GetService200>({
 		url: `/api/v1/cloud_integrations/${cloudProvider}/services/${serviceId}`,
 		method: 'GET',
+		params,
 		signal,
 	});
 };
 
-export const getGetServiceQueryKey = ({
-	cloudProvider,
-	serviceId,
-}: GetServicePathParameters) => {
+export const getGetServiceQueryKey = (
+	{ cloudProvider, serviceId }: GetServicePathParameters,
+	params?: GetServiceParams,
+) => {
 	return [
 		`/api/v1/cloud_integrations/${cloudProvider}/services/${serviceId}`,
+		...(params ? [params] : []),
 	] as const;
 };
 
@@ -1070,6 +1086,7 @@ export const getGetServiceQueryOptions = <
 	TError = ErrorType<RenderErrorResponseDTO>
 >(
 	{ cloudProvider, serviceId }: GetServicePathParameters,
+	params?: GetServiceParams,
 	options?: {
 		query?: UseQueryOptions<
 			Awaited<ReturnType<typeof getService>>,
@@ -1081,11 +1098,12 @@ export const getGetServiceQueryOptions = <
 	const { query: queryOptions } = options ?? {};
 
 	const queryKey =
-		queryOptions?.queryKey ?? getGetServiceQueryKey({ cloudProvider, serviceId });
+		queryOptions?.queryKey ??
+		getGetServiceQueryKey({ cloudProvider, serviceId }, params);
 
 	const queryFn: QueryFunction<Awaited<ReturnType<typeof getService>>> = ({
 		signal,
-	}) => getService({ cloudProvider, serviceId }, signal);
+	}) => getService({ cloudProvider, serviceId }, params, signal);
 
 	return {
 		queryKey,
@@ -1111,6 +1129,7 @@ export function useGetService<
 	TError = ErrorType<RenderErrorResponseDTO>
 >(
 	{ cloudProvider, serviceId }: GetServicePathParameters,
+	params?: GetServiceParams,
 	options?: {
 		query?: UseQueryOptions<
 			Awaited<ReturnType<typeof getService>>,
@@ -1121,6 +1140,7 @@ export function useGetService<
 ): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
 	const queryOptions = getGetServiceQueryOptions(
 		{ cloudProvider, serviceId },
+		params,
 		options,
 	);
 
@@ -1139,10 +1159,11 @@ export function useGetService<
 export const invalidateGetService = async (
 	queryClient: QueryClient,
 	{ cloudProvider, serviceId }: GetServicePathParameters,
+	params?: GetServiceParams,
 	options?: InvalidateOptions,
 ): Promise<QueryClient> => {
 	await queryClient.invalidateQueries(
-		{ queryKey: getGetServiceQueryKey({ cloudProvider, serviceId }) },
+		{ queryKey: getGetServiceQueryKey({ cloudProvider, serviceId }, params) },
 		options,
 	);
 

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -3589,7 +3589,7 @@ export type ListAccounts200 = {
 export type CreateAccountPathParameters = {
 	cloudProvider: string;
 };
-export type CreateAccount200 = {
+export type CreateAccount201 = {
 	data: CloudintegrationtypesGettableAccountWithConnectionArtifactDTO;
 	/**
 	 * @type string
@@ -3647,6 +3647,14 @@ export type GetConnectionCredentials200 = {
 export type ListServicesMetadataPathParameters = {
 	cloudProvider: string;
 };
+export type ListServicesMetadataParams = {
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	cloud_integration_id?: string;
+};
+
 export type ListServicesMetadata200 = {
 	data: CloudintegrationtypesGettableServicesMetadataDTO;
 	/**
@@ -3659,6 +3667,14 @@ export type GetServicePathParameters = {
 	cloudProvider: string;
 	serviceId: string;
 };
+export type GetServiceParams = {
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	cloud_integration_id?: string;
+};
+
 export type GetService200 = {
 	data: CloudintegrationtypesServiceDTO;
 	/**

frontend/src/container/NewWidget/index.tsx

@@ -677,6 +677,18 @@ function NewWidget({
 			queryType: currentQuery.queryType,
 			isNewPanel,
 			dataSource: currentQuery?.builder?.queryData?.[0]?.dataSource,
+			...(currentQuery.queryType === EQueryType.CLICKHOUSE && {
+				clickhouseQueryCount: currentQuery.clickhouse_sql.length,
+				clickhouseQueries: currentQuery.clickhouse_sql.map((q) => ({
+					name: q.name,
+					query: (q.query ?? '')
+						.replace(/--[^\n]*/g, '') // strip line comments
+						.replace(/\/\*[\s\S]*?\*\//g, '') // strip block comments
+						.replace(/'(?:[^'\\]|\\.|'')*'/g, "'?'") // replace single-quoted strings (handles \' and '' escapes)
+						.replace(/\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b/g, '?'), // replace numeric literals (int, float, scientific)
+					disabled: q.disabled,
+				})),
+			}),
 		});
 		setSaveModal(true);
 		// eslint-disable-next-line react-hooks/exhaustive-deps

pkg/apiserver/signozapiserver/cloudintegration.go

@@ -41,7 +41,7 @@ func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
 			RequestContentType:  "application/json",
 			Response:            new(citypes.GettableAccountWithConnectionArtifact),
 			ResponseContentType: "application/json",
-			SuccessStatusCode:   http.StatusOK,
+			SuccessStatusCode:   http.StatusCreated,
 			ErrorStatusCodes:    []int{},
 			Deprecated:          false,
 			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
@@ -138,6 +138,7 @@ func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
 			Summary:             "List services metadata",
 			Description:         "This endpoint lists the services metadata for the specified cloud provider",
 			Request:             nil,
+			RequestQuery:        new(citypes.ListServicesMetadataParams),
 			RequestContentType:  "",
 			Response:            new(citypes.GettableServicesMetadata),
 			ResponseContentType: "application/json",
@@ -158,6 +159,7 @@ func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
 			Summary:             "Get service",
 			Description:         "This endpoint gets a service for the specified cloud provider",
 			Request:             nil,
+			RequestQuery:        new(citypes.GetServiceParams),
 			RequestContentType:  "",
 			Response:            new(citypes.Service),
 			ResponseContentType: "application/json",

pkg/telemetrytraces/const.go

@@ -4,6 +4,20 @@ import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 
 var (
 	IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{
+		"scope_name": {
+			Name:          "scope_name",
+			Description:   "Instrumentation scope name. Learn more about instrumentation scope [here](https://opentelemetry.io/docs/concepts/instrumentation-scope/)",
+			Signal:        telemetrytypes.SignalTraces,
+			FieldContext:  telemetrytypes.FieldContextScope,
+			FieldDataType: telemetrytypes.FieldDataTypeString,
+		},
+		"scope_version": {
+			Name:          "scope_version",
+			Description:   "Instrumentation scope version.",
+			Signal:        telemetrytypes.SignalTraces,
+			FieldContext:  telemetrytypes.FieldContextScope,
+			FieldDataType: telemetrytypes.FieldDataTypeString,
+		},
 		"trace_id": {
 			Name:          "trace_id",
 			Signal:        telemetrytypes.SignalTraces,

pkg/telemetrytraces/field_mapper.go

@@ -52,6 +52,11 @@ var (
 		}},
 		"resource": {Name: "resource", Type: schema.JSONColumnType{}},
 
+		// scope columns
+		"scope_name":       {Name: "scope_name", Type: schema.ColumnTypeString},
+		"scope_version":    {Name: "scope_version", Type: schema.ColumnTypeString},
+		"scope_attributes": {Name: "scope_attributes", Type: schema.JSONColumnType{}},
+
 		"events": {Name: "events", Type: schema.ArrayColumnType{
 			ElementType: schema.ColumnTypeString,
 		}},
@@ -176,7 +181,13 @@ func (m *defaultFieldMapper) getColumn(
 	case telemetrytypes.FieldContextResource:
 		return []*schema.Column{indexV3Columns["resource"]}, nil
 	case telemetrytypes.FieldContextScope:
-		return []*schema.Column{}, qbtypes.ErrColumnNotFound
+		switch key.Name {
+		case "name", "scope.name", "scope_name":
+			return []*schema.Column{indexV3Columns["scope_name"]}, nil
+		case "version", "scope.version", "scope_version":
+			return []*schema.Column{indexV3Columns["scope_version"]}, nil
+		}
+		return []*schema.Column{indexV3Columns["scope_attributes"]}, nil
 	case telemetrytypes.FieldContextAttribute:
 		switch key.FieldDataType {
 		case telemetrytypes.FieldDataTypeString:
@@ -261,21 +272,23 @@ func (m *defaultFieldMapper) FieldFor(
 
 	switch column.Type.GetType() {
 	case schema.ColumnTypeEnumJSON:
-		// json is only supported for resource context as of now
-		if key.FieldContext != telemetrytypes.FieldContextResource {
-			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
-		}
-		oldColumn := indexV3Columns["resources_string"]
-		oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
-		// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
-		// once clickHouse dependency is updated, we need to check if we can remove it.
-		if key.Materialized {
-			oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
-			oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
-			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
-		} else {
-			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
+		switch key.FieldContext {
+		case telemetrytypes.FieldContextScope:
+			return fmt.Sprintf("%s.`%s`::String", column.Name, key.Name), nil
+		case telemetrytypes.FieldContextResource:
+			oldColumn := indexV3Columns["resources_string"]
+			oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
+			// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
+			// once clickHouse dependency is updated, we need to check if we can remove it.
+			if key.Materialized {
+				oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
+				oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
+				return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
+			} else {
+				return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
+			}
 		}
+		return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/scope context fields are supported for json columns, got %s", key.FieldContext.String)
 	case schema.ColumnTypeEnumString,
 		schema.ColumnTypeEnumUInt64,
 		schema.ColumnTypeEnumUInt32,

pkg/types/cloudintegrationtypes/service.go

@@ -62,6 +62,10 @@ type GettableServicesMetadata struct {
 	Services []*ServiceMetadata `json:"services" required:"true" nullable:"false"`
 }
 
+type ListServicesMetadataParams struct {
+	CloudIntegrationID valuer.UUID `query:"cloud_integration_id" required:"false"`
+}
+
 // Service represents a cloud integration service with its definition,
 // cloud integration service is non nil only when the service entry exists in DB with ANY config (enabled or disabled).
 type Service struct {
@@ -69,6 +73,10 @@ type Service struct {
 	CloudIntegrationService *CloudIntegrationService `json:"cloudIntegrationService" required:"true" nullable:"true"`
 }
 
+type GetServiceParams struct {
+	CloudIntegrationID valuer.UUID `query:"cloud_integration_id" required:"false"`
+}
+
 type UpdatableService struct {
 	Config *ServiceConfig `json:"config" required:"true" nullable:"false"`
 }