Compare commits

..

1 Commits

Author SHA1 Message Date
Vinicius Lourenço
8e2da68fc6 test(api-monitoring): mock /fields/keys for quick filters settings stories (#12980)
Some checks are pending
build-staging / prepare (push) Waiting to run
build-staging / js-build (push) Blocked by required conditions
build-staging / go-build (push) Blocked by required conditions
build-staging / staging (push) Blocked by required conditions
cacheci / tests (push) Waiting to run
Release Drafter / update_release_draft (push) Waiting to run
<!--A few plain bullets saying what changed and why, for a reviewer
skimming it - not a wall of text, not a restatement of the diff, not
generated boilerplate.-->
#### Description

Add missing mocks for stories on api monitoring after
https://github.com/SigNoz/signoz/pull/12968
2026-09-24 14:24:56 +00:00
27 changed files with 515 additions and 1264 deletions

View File

@@ -13100,6 +13100,110 @@ paths:
tags:
- llmpricingrules
x-signoz-stability: alpha
/api/v1/logs/promote_paths:
get:
deprecated: false
description: This endpoints promotes and indexes paths
operationId: ListPromotedAndIndexedPaths
responses:
"200":
content:
application/json:
schema:
properties:
data:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Promote and index paths
tags:
- logs
x-signoz-stability: alpha
post:
deprecated: false
description: This endpoints promotes and indexes paths
operationId: HandlePromoteAndIndexPaths
requestBody:
content:
application/json:
schema:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
responses:
"201":
description: Created
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- EDITOR
- tokenizer:
- EDITOR
summary: Promote and index paths
tags:
- logs
x-signoz-stability: alpha
/api/v1/org/preferences:
get:
deprecated: false
@@ -13271,136 +13375,6 @@ paths:
tags:
- preferences
x-signoz-stability: alpha
/api/v1/promote_paths/{telemetry_signal}/{context}:
get:
deprecated: false
description: This endpoint lists the promoted paths of a JSON column. The promotion
domain is identified by the telemetry_signal and context path variables, e.g.
traces/attribute.
operationId: ListPromotedPaths
parameters:
- in: path
name: telemetry_signal
required: true
schema:
type: string
- in: path
name: context
required: true
schema:
type: string
responses:
"200":
content:
application/json:
schema:
properties:
data:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: List promoted paths
tags:
- promote
x-signoz-stability: alpha
post:
deprecated: false
description: This endpoint promotes paths of a JSON column to its promoted column.
The promotion domain is identified by the telemetry_signal and context path
variables, e.g. traces/attribute.
operationId: PromotePaths
parameters:
- in: path
name: telemetry_signal
required: true
schema:
type: string
- in: path
name: context
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
responses:
"201":
description: Created
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- EDITOR
- tokenizer:
- EDITOR
summary: Promote paths
tags:
- promote
x-signoz-stability: alpha
/api/v1/roles:
get:
deprecated: false

View File

@@ -4,15 +4,23 @@
* * regenerate with 'pnpm generate:api'
* SigNoz
*/
import { useMutation } from 'react-query';
import { useMutation, useQuery } from 'react-query';
import type {
InvalidateOptions,
MutationFunction,
QueryClient,
QueryFunction,
QueryKey,
UseMutationOptions,
UseMutationResult,
UseQueryOptions,
UseQueryResult,
} from 'react-query';
import type {
HandleExportRawDataPOSTParams,
ListPromotedAndIndexedPaths200,
PromotetypesPromotePathDTO,
Querybuildertypesv5QueryRangeRequestDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
@@ -20,6 +28,26 @@ import type {
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type { ErrorType, BodyType } from '../../../generatedAPIInstance';
const withQueryKey = <T extends object, K>(
query: T,
queryKey: K,
): T & { queryKey: K } => {
const result = { queryKey } as T & { queryKey: K };
for (const key of Object.keys(query)) {
// The explicit queryKey always wins, matching the previous
// `{ ...query, queryKey }` spread where it was set last.
if (key === 'queryKey') {
continue;
}
Object.defineProperty(result, key, {
enumerable: true,
configurable: true,
get: () => (query as Record<string, unknown>)[key],
});
}
return result;
};
/**
* This endpoints allows complex query exporting raw data for traces and logs
* @summary Export raw data
@@ -121,3 +149,175 @@ export const useHandleExportRawDataPOST = <
> => {
return useMutation(getHandleExportRawDataPOSTMutationOptions(options));
};
/**
* This endpoints promotes and indexes paths
* @summary Promote and index paths
*/
export const listPromotedAndIndexedPaths = (signal?: AbortSignal) => {
return GeneratedAPIInstance<ListPromotedAndIndexedPaths200>({
url: `/api/v1/logs/promote_paths`,
method: 'GET',
signal,
});
};
export const getListPromotedAndIndexedPathsQueryKey = () => {
return [`/api/v1/logs/promote_paths`] as const;
};
export const getListPromotedAndIndexedPathsQueryOptions = <
TData = Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError,
TData
>;
}) => {
const { query: queryOptions } = options ?? {};
const queryKey =
queryOptions?.queryKey ?? getListPromotedAndIndexedPathsQueryKey();
const queryFn: QueryFunction<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>
> = ({ signal }) => listPromotedAndIndexedPaths(signal);
return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError,
TData
> & { queryKey: QueryKey };
};
export type ListPromotedAndIndexedPathsQueryResult = NonNullable<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>
>;
export type ListPromotedAndIndexedPathsQueryError =
ErrorType<RenderErrorResponseDTO>;
/**
* @summary Promote and index paths
*/
export function useListPromotedAndIndexedPaths<
TData = Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError,
TData
>;
}): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
const queryOptions = getListPromotedAndIndexedPathsQueryOptions(options);
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
queryKey: QueryKey;
};
return withQueryKey(query, queryOptions.queryKey);
}
/**
* @summary Promote and index paths
*/
export const invalidateListPromotedAndIndexedPaths = async (
queryClient: QueryClient,
options?: InvalidateOptions,
): Promise<QueryClient> => {
await queryClient.invalidateQueries(
{ queryKey: getListPromotedAndIndexedPathsQueryKey() },
options,
);
return queryClient;
};
/**
* This endpoints promotes and indexes paths
* @summary Promote and index paths
*/
export const handlePromoteAndIndexPaths = (
promotetypesPromotePathDTONull?: BodyType<
PromotetypesPromotePathDTO[] | null
> | null,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<void>({
url: `/api/v1/logs/promote_paths`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: promotetypesPromotePathDTONull,
signal,
});
};
export const getHandlePromoteAndIndexPathsMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
> => {
const mutationKey = ['handlePromoteAndIndexPaths'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> }
> = (props) => {
const { data } = props ?? {};
return handlePromoteAndIndexPaths(data);
};
return { mutationFn, ...mutationOptions };
};
export type HandlePromoteAndIndexPathsMutationResult = NonNullable<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>
>;
export type HandlePromoteAndIndexPathsMutationBody =
| BodyType<PromotetypesPromotePathDTO[] | null>
| undefined;
export type HandlePromoteAndIndexPathsMutationError =
ErrorType<RenderErrorResponseDTO>;
/**
* @summary Promote and index paths
*/
export const useHandlePromoteAndIndexPaths = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
> => {
return useMutation(getHandlePromoteAndIndexPathsMutationOptions(options));
};

View File

@@ -1,262 +0,0 @@
/**
* ! Do not edit manually
* * The file has been auto-generated using Orval for SigNoz
* * regenerate with 'pnpm generate:api'
* SigNoz
*/
import { useMutation, useQuery } from 'react-query';
import type {
InvalidateOptions,
MutationFunction,
QueryClient,
QueryFunction,
QueryKey,
UseMutationOptions,
UseMutationResult,
UseQueryOptions,
UseQueryResult,
} from 'react-query';
import type {
ListPromotedPaths200,
ListPromotedPathsPathParameters,
PromotePathsPathParameters,
PromotetypesPromotePathDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type { ErrorType, BodyType } from '../../../generatedAPIInstance';
const withQueryKey = <T extends object, K>(
query: T,
queryKey: K,
): T & { queryKey: K } => {
const result = { queryKey } as T & { queryKey: K };
for (const key of Object.keys(query)) {
// The explicit queryKey always wins, matching the previous
// `{ ...query, queryKey }` spread where it was set last.
if (key === 'queryKey') {
continue;
}
Object.defineProperty(result, key, {
enumerable: true,
configurable: true,
get: () => (query as Record<string, unknown>)[key],
});
}
return result;
};
/**
* This endpoint lists the promoted paths of a JSON column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.
* @summary List promoted paths
*/
export const listPromotedPaths = (
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<ListPromotedPaths200>({
url: `/api/v1/promote_paths/${telemetrySignal}/${context}`,
method: 'GET',
signal,
});
};
export const getListPromotedPathsQueryKey = ({
telemetrySignal,
context,
}: ListPromotedPathsPathParameters) => {
return [`/api/v1/promote_paths/${telemetrySignal}/${context}`] as const;
};
export const getListPromotedPathsQueryOptions = <
TData = Awaited<ReturnType<typeof listPromotedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedPaths>>,
TError,
TData
>;
},
) => {
const { query: queryOptions } = options ?? {};
const queryKey =
queryOptions?.queryKey ??
getListPromotedPathsQueryKey({ telemetrySignal, context });
const queryFn: QueryFunction<
Awaited<ReturnType<typeof listPromotedPaths>>
> = ({ signal }) => listPromotedPaths({ telemetrySignal, context }, signal);
return {
queryKey,
queryFn,
enabled:
telemetrySignal !== null &&
telemetrySignal !== undefined &&
context !== null &&
context !== undefined,
...queryOptions,
} as UseQueryOptions<
Awaited<ReturnType<typeof listPromotedPaths>>,
TError,
TData
> & { queryKey: QueryKey };
};
export type ListPromotedPathsQueryResult = NonNullable<
Awaited<ReturnType<typeof listPromotedPaths>>
>;
export type ListPromotedPathsQueryError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary List promoted paths
*/
export function useListPromotedPaths<
TData = Awaited<ReturnType<typeof listPromotedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedPaths>>,
TError,
TData
>;
},
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
const queryOptions = getListPromotedPathsQueryOptions(
{ telemetrySignal, context },
options,
);
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
queryKey: QueryKey;
};
return withQueryKey(query, queryOptions.queryKey);
}
/**
* @summary List promoted paths
*/
export const invalidateListPromotedPaths = async (
queryClient: QueryClient,
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
options?: InvalidateOptions,
): Promise<QueryClient> => {
await queryClient.invalidateQueries(
{ queryKey: getListPromotedPathsQueryKey({ telemetrySignal, context }) },
options,
);
return queryClient;
};
/**
* This endpoint promotes paths of a JSON column to its promoted column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.
* @summary Promote paths
*/
export const promotePaths = (
{ telemetrySignal, context }: PromotePathsPathParameters,
promotetypesPromotePathDTONull?: BodyType<
PromotetypesPromotePathDTO[] | null
> | null,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<void>({
url: `/api/v1/promote_paths/${telemetrySignal}/${context}`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: promotetypesPromotePathDTONull,
signal,
});
};
export const getPromotePathsMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
> => {
const mutationKey = ['promotePaths'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof promotePaths>>,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
}
> = (props) => {
const { pathParams, data } = props ?? {};
return promotePaths(pathParams, data);
};
return { mutationFn, ...mutationOptions };
};
export type PromotePathsMutationResult = NonNullable<
Awaited<ReturnType<typeof promotePaths>>
>;
export type PromotePathsMutationBody =
| BodyType<PromotetypesPromotePathDTO[] | null>
| undefined;
export type PromotePathsMutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Promote paths
*/
export const usePromotePaths = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
> => {
return useMutation(getPromotePathsMutationOptions(options));
};

View File

@@ -12470,6 +12470,17 @@ export type ListUnmappedLLMModels200 = {
status: string;
};
export type ListPromotedAndIndexedPaths200 = {
/**
* @type array,null
*/
data: PromotetypesPromotePathDTO[] | null;
/**
* @type string
*/
status: string;
};
export type ListOrgPreferences200 = {
/**
* @type array
@@ -12495,25 +12506,6 @@ export type GetOrgPreference200 = {
export type UpdateOrgPreferencePathParameters = {
name: string;
};
export type ListPromotedPathsPathParameters = {
telemetrySignal: string;
context: string;
};
export type ListPromotedPaths200 = {
/**
* @type array,null
*/
data: PromotetypesPromotePathDTO[] | null;
/**
* @type string
*/
status: string;
};
export type PromotePathsPathParameters = {
telemetrySignal: string;
context: string;
};
export type ListRoles200 = {
/**
* @type array

View File

@@ -8,6 +8,7 @@ import {
QuickfiltertypesSourceDTO,
TelemetrytypesFieldContextDTO,
TelemetrytypesFieldDataTypeDTO,
TelemetrytypesSignalDTO,
} from 'api/generated/services/sigNoz.schemas';
import ROUTES from 'constants/routes';
import { VIEWS } from 'container/ApiMonitoring/Explorer/Domains/DomainDetails/constants';
@@ -24,7 +25,10 @@ import {
toggleControl,
} from '@/storybook/controls/controls';
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
import {
fieldKeysResponse,
fieldValuesResponse,
} from '@/storybook/msw/__story_mockdata__/fields';
import { quickFiltersResponse } from '@/storybook/msw/__story_mockdata__/quickFilters';
import {
@@ -317,6 +321,21 @@ export const apiMonitoringMocks = defineStoryMocks({
})),
),
rest.get(
'http://localhost/api/v1/fields/keys',
response.json((req) =>
fieldKeysResponse(
groupByAttributeKeys(req.url.searchParams.get('searchText') ?? '').map(
({ key }) => key,
),
{
signal: TelemetrytypesSignalDTO.traces,
fieldContext: TelemetrytypesFieldContextDTO.attribute,
},
),
),
),
rest.get(
'http://localhost/api/v1/fields/values',
response.json((req) =>

View File

@@ -10,11 +10,11 @@ import (
)
func (provider *provider) addPromoteRoutes(router *mux.Router) error {
if err := router.Handle("/api/v1/promote_paths/{telemetry_signal}/{context}", handler.New(provider.authzMiddleware.EditAccess(provider.promoteHandler.PromotePaths), handler.OpenAPIDef{
ID: "PromotePaths",
Tags: []string{"promote"},
Summary: "Promote paths",
Description: "This endpoint promotes paths of a JSON column to its promoted column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.",
if err := router.Handle("/api/v1/logs/promote_paths", handler.New(provider.authzMiddleware.EditAccess(provider.promoteHandler.HandlePromoteAndIndexPaths), handler.OpenAPIDef{
ID: "HandlePromoteAndIndexPaths",
Tags: []string{"logs"},
Summary: "Promote and index paths",
Description: "This endpoints promotes and indexes paths",
Request: new([]*promotetypes.PromotePath),
RequestContentType: "application/json",
Response: nil,
@@ -26,11 +26,11 @@ func (provider *provider) addPromoteRoutes(router *mux.Router) error {
return err
}
if err := router.Handle("/api/v1/promote_paths/{telemetry_signal}/{context}", handler.New(provider.authzMiddleware.ViewAccess(provider.promoteHandler.ListPromotedPaths), handler.OpenAPIDef{
ID: "ListPromotedPaths",
Tags: []string{"promote"},
Summary: "List promoted paths",
Description: "This endpoint lists the promoted paths of a JSON column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.",
if err := router.Handle("/api/v1/logs/promote_paths", handler.New(provider.authzMiddleware.ViewAccess(provider.promoteHandler.ListPromotedAndIndexedPaths), handler.OpenAPIDef{
ID: "ListPromotedAndIndexedPaths",
Tags: []string{"logs"},
Summary: "Promote and index paths",
Description: "This endpoints promotes and indexes paths",
Request: nil,
RequestContentType: "",
Response: new([]*promotetypes.PromotePath),

View File

@@ -11,7 +11,6 @@ var (
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
FeatureResolveSemconvFamilies = featuretypes.MustNewName("resolve_semconv_families")
FeatureUseTraceAttributesJSON = featuretypes.MustNewName("use_trace_attributes_json")
)
func MustNewRegistry() featuretypes.Registry {
@@ -80,14 +79,6 @@ func MustNewRegistry() featuretypes.Registry {
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
&featuretypes.Feature{
Name: FeatureUseTraceAttributesJSON,
Kind: featuretypes.KindBoolean,
Stage: featuretypes.StageExperimental,
Description: "Controls whether trace queries read span attributes from the JSON columns",
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
)
if err != nil {
panic(err)

View File

@@ -9,7 +9,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/gorilla/mux"
)
type handler struct {
@@ -20,16 +19,9 @@ func NewHandler(module promote.Module) promote.Handler {
return &handler{module: module}
}
func (h *handler) PromotePaths(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
target, err := promotetypes.NewTargetFromPath(vars["telemetry_signal"], vars["context"])
if err != nil {
render.Error(w, err)
return
}
func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Request) {
// TODO(Nitya): Use in multi tenant setup
_, err = authtypes.ClaimsFromContext(r.Context())
_, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, errors.NewInternalf(errors.CodeInternal, "failed to get org id from context"))
return
@@ -41,7 +33,7 @@ func (h *handler) PromotePaths(w http.ResponseWriter, r *http.Request) {
return
}
err = h.module.PromotePaths(r.Context(), target, req...)
err = h.module.PromoteAndIndexPaths(r.Context(), req...)
if err != nil {
render.Error(w, err)
return
@@ -50,22 +42,15 @@ func (h *handler) PromotePaths(w http.ResponseWriter, r *http.Request) {
render.Success(w, http.StatusCreated, nil)
}
func (h *handler) ListPromotedPaths(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
target, err := promotetypes.NewTargetFromPath(vars["telemetry_signal"], vars["context"])
if err != nil {
render.Error(w, err)
return
}
func (h *handler) ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request) {
// TODO(Nitya): Use in multi tenant setup
_, err = authtypes.ClaimsFromContext(r.Context())
_, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, errors.NewInternalf(errors.CodeInternal, "failed to get org id from context"))
return
}
paths, err := h.module.ListPromotedPaths(r.Context(), target)
paths, err := h.module.ListPromotedAndIndexedPaths(r.Context())
if err != nil {
render.Error(w, err)
return

View File

@@ -2,11 +2,14 @@ package implpromote
import (
"context"
"maps"
"slices"
"strings"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -28,57 +31,45 @@ func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore teleme
return &module{metadataStore: metadataStore, telemetryStore: telemetrystore}
}
// ListPromotedPaths lists the promoted paths of the target JSON column,
// merged with per-path index metadata where the target supports indexes.
func (m *module) ListPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error) {
promotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry)
if err != nil {
return nil, err
}
response := make([]promotetypes.PromotePath, 0, len(promotedPaths))
for path := range promotedPaths {
response = append(response, promotetypes.PromotePath{
Path: target.RequiredPathPrefix + path,
Promote: true,
})
}
// Index metadata is optional per target; merge it in only where supported.
if !target.IndexesSupported {
return response, nil
}
func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error) {
indexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
if err != nil {
return nil, err
}
// index.Name is the bare path and index.BaseColumn carries the column
// prefix, so the aggregate key is the full sub-column path.
aggr := map[string][]promotetypes.WrappedIndex{}
for _, index := range indexes {
fullPath := index.BaseColumn + index.Name
aggr[fullPath] = append(aggr[fullPath], promotetypes.WrappedIndex{
aggr[index.Name] = append(aggr[index.Name], promotetypes.WrappedIndex{
FieldDataType: index.FieldDataType,
Type: index.IndexType,
Granularity: index.Granularity,
})
}
promotedPaths, err := m.listPromotedPaths(ctx)
if err != nil {
return nil, err
}
for i := range response {
fullPath := target.PromotedColumnPrefix() + strings.TrimPrefix(response[i].Path, target.RequiredPathPrefix)
if indexes, ok := aggr[fullPath]; ok {
response[i].Indexes = indexes
response := []promotetypes.PromotePath{}
for _, path := range promotedPaths {
fullPath := logstelemetryschema.BodyPromotedColumnPrefix + path
path = telemetrytypes.BodyJSONStringSearchPrefix + path
item := promotetypes.PromotePath{
Path: path,
Promote: true,
}
indexes, ok := aggr[fullPath]
if ok {
item.Indexes = indexes
delete(aggr, fullPath)
}
response = append(response, item)
}
// add the paths that are not promoted but have indexes
for fullPath, indexes := range aggr {
path := strings.TrimPrefix(fullPath, target.BaseColumnPrefix())
path = strings.TrimPrefix(path, target.PromotedColumnPrefix())
path = target.RequiredPathPrefix + path
for path, indexes := range aggr {
path := strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
path = telemetrytypes.BodyJSONStringSearchPrefix + path
response = append(response, promotetypes.PromotePath{
Path: path,
Indexes: indexes,
@@ -87,10 +78,54 @@ func (m *module) ListPromotedPaths(ctx context.Context, target promotetypes.Targ
return response, nil
}
// PromotePaths records new promotions of the target JSON column in the column
// evolution table and, for targets with index support, creates the requested
// per-path skip indexes.
func (m *module) PromotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error {
func (m *module) listPromotedPaths(ctx context.Context) ([]string, error) {
paths, err := m.metadataStore.GetPromotedPaths(ctx)
if err != nil {
return nil, err
}
return slices.Collect(maps.Keys(paths)), nil
}
// PromotePaths inserts provided JSON paths into the promoted paths table for logs queries.
func (m *module) PromotePaths(ctx context.Context, paths []string) error {
if len(paths) == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
}
return m.metadataStore.PromotePaths(ctx, paths...)
}
// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Index) error {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.CodeNamespace: "promote",
instrumentationtypes.CodeFunctionName: "createIndexes",
})
if len(indexes) == 0 {
return nil
}
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
Database: logstelemetryschema.DBName,
Table: logstelemetryschema.LogsV2LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
if err := m.telemetryStore.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
}
}
return nil
}
// PromoteAndIndexPaths handles promoting paths and creating indexes in one call.
func (m *module) PromoteAndIndexPaths(
ctx context.Context,
paths ...*promotetypes.PromotePath,
) error {
if len(paths) == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
}
@@ -98,13 +133,13 @@ func (m *module) PromotePaths(ctx context.Context, target promotetypes.Target, p
pathsStr := []string{}
// validate the paths
for _, path := range paths {
if err := path.ValidateAndSetDefaults(target); err != nil {
if err := path.ValidateAndSetDefaults(); err != nil {
return err
}
pathsStr = append(pathsStr, path.Path)
}
existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry, pathsStr...)
existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, pathsStr...)
if err != nil {
return err
}
@@ -118,10 +153,10 @@ func (m *module) PromotePaths(ctx context.Context, target promotetypes.Target, p
}
}
if len(it.Indexes) > 0 {
parentColumn := target.BaseColumn
parentColumn := logstelemetryschema.LogsV2BodyV2Column
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
parentColumn = target.PromotedColumn()
parentColumn = logstelemetryschema.LogsV2BodyPromotedColumn
}
for _, index := range it.Indexes {
@@ -147,43 +182,17 @@ func (m *module) PromotePaths(ctx context.Context, target promotetypes.Target, p
}
if len(toInsert) > 0 {
err := m.metadataStore.PromotePaths(ctx, target.Entry, toInsert...)
err := m.PromotePaths(ctx, toInsert)
if err != nil {
return err
}
}
if len(indexes) > 0 {
if err := m.createIndexes(ctx, target, indexes); err != nil {
if err := m.createIndexes(ctx, indexes); err != nil {
return err
}
}
return nil
}
// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
func (m *module) createIndexes(ctx context.Context, target promotetypes.Target, indexes []schemamigrator.Index) error {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: target.Entry.Signal.StringValue(),
instrumentationtypes.CodeNamespace: "promote",
instrumentationtypes.CodeFunctionName: "createIndexes",
})
if len(indexes) == 0 {
return nil
}
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
Database: target.DBName,
Table: target.LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
if err := m.telemetryStore.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
}
}
return nil
}

View File

@@ -1,234 +0,0 @@
package implpromote
import (
"context"
"regexp"
"testing"
sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPromotePaths(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
target promotetypes.Target
paths []*promotetypes.PromotePath
promoteTwice bool
wantErr bool
wantPromoted []string
}{
{
name: "PromotesNewAttributes_Idempotent",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{
{Path: "http.method", Promote: true},
{Path: "span.operation", Promote: true},
},
promoteTwice: true,
wantPromoted: []string{"http.method", "span.operation"},
},
{
name: "NonPromoteEntries_NotRecorded",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{Path: "http.method"}},
},
{
name: "ColumnPrefixedPath_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{Path: "attributes.http.method", Promote: true}},
wantErr: true,
},
{
name: "EmptyPath_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{Path: "", Promote: true}},
wantErr: true,
},
{
name: "EmptyRequest_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
wantErr: true,
},
{
name: "PromotesBodyPath_PrefixStripped",
target: promotetypes.NewLogsBodyTarget(),
paths: []*promotetypes.PromotePath{{Path: "body.user.name", Promote: true}},
wantPromoted: []string{"user.name"},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
store := telemetrytypestest.NewMockMetadataStore()
m := NewModule(store, nil)
err := m.PromotePaths(ctx, testCase.target, testCase.paths...)
if testCase.wantErr {
assert.Error(t, err)
assert.Empty(t, store.PromotedPathsMap)
return
}
require.NoError(t, err)
require.Len(t, store.PromotedPathsMap, len(testCase.wantPromoted))
for _, path := range testCase.wantPromoted {
assert.True(t, store.PromotedPathsMap[path], path)
}
if testCase.promoteTwice {
// promoting again must not fail
require.NoError(t, m.PromotePaths(ctx, testCase.target, testCase.paths...))
assert.Len(t, store.PromotedPathsMap, len(testCase.wantPromoted))
}
})
}
}
func TestPromotePathsCreatesIndexes(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
promoted map[string]bool
path *promotetypes.PromotePath
wantDDLColumn string
}{
{
name: "NewPromotion_IndexesPromotedColumn",
path: &promotetypes.PromotePath{
Path: "body.user.name",
Promote: true,
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantDDLColumn: "dynamicElement(body_promoted.user.name",
},
{
name: "AlreadyPromoted_IndexesPromotedColumn",
promoted: map[string]bool{"user.name": true},
path: &promotetypes.PromotePath{
Path: "body.user.name",
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantDDLColumn: "dynamicElement(body_promoted.user.name",
},
{
name: "UnpromotedPath_IndexesBaseColumn",
path: &promotetypes.PromotePath{
Path: "body.user.name",
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantDDLColumn: "dynamicElement(body_v2.user.name",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
ts := telemetrystoretest.New(telemetrystore.Config{}, sqlmock.QueryMatcherRegexp)
store := telemetrytypestest.NewMockMetadataStore()
if testCase.promoted != nil {
store.PromotedPathsMap = testCase.promoted
}
m := NewModule(store, ts)
ts.Mock().ExpectExec("ADD INDEX (.+)" + regexp.QuoteMeta(testCase.wantDDLColumn)).WillReturnError(nil)
require.NoError(t, m.PromotePaths(ctx, promotetypes.NewLogsBodyTarget(), testCase.path))
assert.NoError(t, ts.Mock().ExpectationsWereMet())
})
}
}
func TestListPromotedPaths(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
target promotetypes.Target
promoted map[string]bool
indexes []telemetrytypes.TelemetryFieldKeySkipIndex
wantPaths []promotetypes.PromotePath
}{
{
name: "TracesAttributes_PromotedPaths",
target: promotetypes.NewTracesAttributesTarget(),
promoted: map[string]bool{"http.method": true},
wantPaths: []promotetypes.PromotePath{
{Path: "http.method", Promote: true},
},
},
{
name: "LogsBody_PromotedAndIndexedPaths",
target: promotetypes.NewLogsBodyTarget(),
promoted: map[string]bool{"user.name": true},
indexes: []telemetrytypes.TelemetryFieldKeySkipIndex{
{
Name: "user.name",
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: telemetrytypes.FieldDataTypeString,
BaseColumn: "body_promoted.",
IndexType: "ngrambf_v1(4, 1024, 2, 0)",
Granularity: 1,
},
{
Name: "request.duration",
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: telemetrytypes.FieldDataTypeFloat64,
BaseColumn: "body_v2.",
IndexType: "minmax",
Granularity: 1,
},
},
wantPaths: []promotetypes.PromotePath{
{
Path: "body.user.name",
Promote: true,
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
{
Path: "body.request.duration",
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeFloat64, Type: "minmax", Granularity: 1},
},
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
store := telemetrytypestest.NewMockMetadataStore()
store.PromotedPathsMap = testCase.promoted
store.LogsJSONIndexes = testCase.indexes
m := NewModule(store, nil)
paths, err := m.ListPromotedPaths(ctx, testCase.target)
require.NoError(t, err)
require.Len(t, paths, len(testCase.wantPaths))
byPath := map[string]promotetypes.PromotePath{}
for _, path := range paths {
byPath[path.Path] = path
}
for _, want := range testCase.wantPaths {
require.Contains(t, byPath, want.Path)
assert.Equal(t, want, byPath[want.Path])
}
})
}
}

View File

@@ -8,11 +8,11 @@ import (
)
type Module interface {
ListPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error)
PromotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error
ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error)
PromoteAndIndexPaths(ctx context.Context, paths ...*promotetypes.PromotePath) error
}
type Handler interface {
PromotePaths(w http.ResponseWriter, r *http.Request)
ListPromotedPaths(w http.ResponseWriter, r *http.Request)
HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Request)
ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request)
}

View File

@@ -24,9 +24,7 @@ func NewQueryInfo(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, si
FamiliesOn: semconvFamiliesEnabled(ctx, orgID, fl),
}
if fl != nil {
evalCtx := featuretypes.NewFlaggerEvaluationContext(orgID)
q.BodyJSONOn = fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, evalCtx)
q.TraceAttrsJSONOn = fl.BooleanOrEmpty(ctx, flagger.FeatureUseTraceAttributesJSON, evalCtx)
q.BodyJSONOn = fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))
}
return q
}

View File

@@ -7,7 +7,6 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -25,7 +24,7 @@ var jsonAttrColRe = regexp.MustCompile(`,\s*attributes\s*(,| FROM )`)
func newBulkTestBuilder(t *testing.T, releaseTime time.Time) *traceQueryStatementBuilder {
t.Helper()
fl := flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureUseTraceAttributesJSON.String(): true})
fl := flaggertest.New(t)
storage := tracestelemetryschema.NewStorage()
store := telemetrytypestest.NewMockMetadataStore()
store.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)

View File

@@ -16,7 +16,6 @@ import (
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
)
@@ -34,10 +33,6 @@ var (
CodeFailedToAppendPath = errors.MustNewCode("failed_to_append_path_promoted_paths")
)
// logsBodyPromotedEntry templates the column evolution rows recorded for
// logs body promotions.
var logsBodyPromotedEntry = promotetypes.NewLogsBodyTarget().Entry
// enrichJSONKeys enriches body-context keys with promoted path info, indexes,
// and JSON access plans. parentTypeCache contains parent array types (ArrayJSON/ArrayDynamic)
// pre-fetched in the main UNION query.
@@ -72,7 +67,7 @@ func (t *telemetryMetaStore) enrichJSONKeys(ctx context.Context, selectors []*te
}
// fetch promoted paths
promoted, err := t.GetPromotedPaths(ctx, logsBodyPromotedEntry, paths...)
promoted, err := t.GetPromotedPaths(ctx, paths...)
if err != nil {
return err
}
@@ -162,7 +157,7 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
}
func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
ctx = withTelemetryContext(ctx, telemetrytypes.SignalLogs, "ListLogsJSONIndexes")
ctx = withTelemetryContext(ctx, "ListLogsJSONIndexes")
query, args := buildListLogsJSONIndexesQuery(t.telemetrystore.Cluster(), filters...)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
@@ -220,14 +215,14 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
// TODO(Piyush): Remove this if not used in future.
func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, limit int) (*telemetrytypes.TelemetryFieldValues, bool, error) {
ctx = withTelemetryContext(ctx, telemetrytypes.SignalLogs, "ListJSONValues")
ctx = withTelemetryContext(ctx, "ListJSONValues")
path = CleanPathPrefixes(path)
if strings.Contains(path, telemetrytypes.ArraySep) || strings.Contains(path, telemetrytypes.ArrayAnyIndex) {
return nil, false, errors.NewInvalidInputf(errors.CodeInvalidInput, "array paths are not supported")
}
promoted, err := t.isPathPromoted(ctx, logsBodyPromotedEntry, path)
promoted, err := t.IsPathPromoted(ctx, path)
if err != nil {
return nil, false, err
}
@@ -381,13 +376,13 @@ func derefValue(v any) any {
return val.Interface()
}
// isPathPromoted checks if a specific path is promoted (Column Evolution table: field_name for the entry's column).
func (t *telemetryMetaStore) isPathPromoted(ctx context.Context, entry telemetrytypes.EvolutionEntry, path string) (bool, error) {
ctx = withTelemetryContext(ctx, entry.Signal, "isPathPromoted")
// IsPathPromoted checks if a specific path is promoted (Column Evolution table: field_name for logs body).
func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (bool, error) {
ctx = withTelemetryContext(ctx, "IsPathPromoted")
split := strings.Split(path, telemetrytypes.ArraySep)
pathSegment := split[0]
query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE signal = ? AND column_name = ? AND field_context = ? AND field_name = ? LIMIT 1", DBName, PromotedPathsTableName)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, entry.Signal, entry.ColumnName, entry.FieldContext, pathSegment)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
if err != nil {
return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
}
@@ -396,14 +391,14 @@ func (t *telemetryMetaStore) isPathPromoted(ctx context.Context, entry telemetry
return rows.Next(), nil
}
// GetPromotedPaths returns promoted paths from the Column Evolution table (field_name for the entry's column).
func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) (map[string]bool, error) {
ctx = withTelemetryContext(ctx, entry.Signal, "GetPromotedPaths")
// GetPromotedPaths returns promoted paths from the Column Evolution table (field_name for logs body).
func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error) {
ctx = withTelemetryContext(ctx, "GetPromotedPaths")
sb := sqlbuilder.Select("field_name").From(fmt.Sprintf("%s.%s", DBName, PromotedPathsTableName))
conditions := []string{
sb.Equal("signal", entry.Signal),
sb.Equal("column_name", entry.ColumnName),
sb.Equal("field_context", entry.FieldContext),
sb.Equal("signal", telemetrytypes.SignalLogs),
sb.Equal("column_name", logstelemetryschema.LogsV2BodyPromotedColumn),
sb.Equal("field_context", telemetrytypes.FieldContextBody),
sb.NotEqual("field_name", "__all__"),
}
if len(paths) > 0 {
@@ -443,10 +438,9 @@ func CleanPathPrefixes(path string) string {
return path
}
// PromotePaths inserts promoted paths into the Column Evolution table as rows templated by entry
// (same schema as signoz-otel-collector metadata_migrations); FieldName and ReleaseTime are set per path.
func (t *telemetryMetaStore) PromotePaths(ctx context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) error {
ctx = withTelemetryContext(ctx, entry.Signal, "PromotePaths")
// PromotePaths inserts promoted paths into the Column Evolution table (same schema as signoz-otel-collector metadata_migrations).
func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string) error {
ctx = withTelemetryContext(ctx, "PromotePaths")
batch, err := t.telemetrystore.ClickhouseDB().PrepareBatch(ctx,
fmt.Sprintf("INSERT INTO %s.%s (signal, column_name, column_type, field_context, field_name, version, release_time) VALUES", DBName,
PromotedPathsTableName))
@@ -460,7 +454,7 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, entry telemetryty
if trimmed == "" {
continue
}
if err := batch.Append(entry.Signal, entry.ColumnName, entry.ColumnType, entry.FieldContext, trimmed, entry.Version, releaseTime); err != nil {
if err := batch.Append(telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
_ = batch.Abort()
return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
}
@@ -472,9 +466,9 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, entry telemetryty
return nil
}
func withTelemetryContext(ctx context.Context, signal telemetrytypes.Signal, functionName string) context.Context {
func withTelemetryContext(ctx context.Context, functionName string) context.Context {
return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: signal.StringValue(),
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: functionName,
})

View File

@@ -172,7 +172,7 @@ func NewStorage() qbtypes.Storage {
func (m *storage) getColumn(
_ context.Context,
q qbtypes.QueryInfo,
_, _ uint64,
key *telemetrytypes.TelemetryFieldKey,
) ([]*schema.Column, error) {
switch key.FieldContext {
@@ -194,8 +194,8 @@ func (m *storage) getColumn(
default:
return nil, qbtypes.ErrColumnNotFound
}
// The use_trace_attributes_json flag and the `attributes` evolution entry are the rollout control.
if q.TraceAttrsJSONOn && attributeColumnEvolutionRegistered(key, SpanAttributesColumn) {
// The `attributes` evolution entry is the rollout control.
if attributeColumnEvolutionRegistered(key, SpanAttributesColumn) {
cols := make([]*schema.Column, 0, 3)
if attributeColumnEvolutionRegistered(key, SpanAttributesPromotedColumn) {
cols = append(cols, indexV3Columns["attributes_promoted"])
@@ -233,15 +233,15 @@ func (m *storage) getColumn(
// (after evolution selection); existExprs only carries guards for guardable column types.
func (m *storage) resolveColumnExprs(
ctx context.Context,
q qbtypes.QueryInfo,
startNs, endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
) (exprs []string, existExprs []string, columns []*schema.Column, err error) {
columns, err = m.getColumn(ctx, q, key)
columns, err = m.getColumn(ctx, startNs, endNs, key)
if err != nil {
return nil, nil, nil, err
}
newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, q.StartNs, q.EndNs)
newColumns, evolutionsEntries, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
if err != nil {
return nil, nil, nil, err
}
@@ -355,12 +355,12 @@ func attributeJSONValueExpr(path string, dataType telemetrytypes.FieldDataType)
// columnIsTemporal reports whether key resolves to a single time column, after evolution
// selection. Multiple columns mean an attribute-map union, which is never temporal.
func (m *storage) columnIsTemporal(ctx context.Context, q qbtypes.QueryInfo, key *telemetrytypes.TelemetryFieldKey) (bool, error) {
columns, err := m.getColumn(ctx, q, key)
func (m *storage) columnIsTemporal(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (bool, error) {
columns, err := m.getColumn(ctx, startNs, endNs, key)
if err != nil {
return false, err
}
newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, q.StartNs, q.EndNs)
newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
if err != nil {
return false, err
}
@@ -394,7 +394,7 @@ func (m *storage) read(ctx context.Context, q qbtypes.QueryInfo, key *telemetryt
return key.Name, nil
}
exprs, existExpr, columns, err := m.resolveColumnExprs(ctx, q, key)
exprs, existExpr, columns, err := m.resolveColumnExprs(ctx, q.StartNs, q.EndNs, key)
if err != nil {
return "", err
}
@@ -457,7 +457,7 @@ func (m *storage) Read(ctx context.Context, q qbtypes.QueryInfo, key *telemetryt
if isSpanSearchScopeField(key.Name) {
return qbtypes.Read{SQL: key.Name, Presence: "true", Absence: "false", WhenAbsent: qbtypes.AlwaysPresent}, nil
}
exprs, existExprs, columns, err := m.resolveColumnExprs(ctx, q, key)
exprs, existExprs, columns, err := m.resolveColumnExprs(ctx, q.StartNs, q.EndNs, key)
if err != nil {
return qbtypes.Read{}, err
}
@@ -469,7 +469,7 @@ func (m *storage) Read(ctx context.Context, q qbtypes.QueryInfo, key *telemetryt
if err != nil {
return qbtypes.Read{}, err
}
temporal, err := m.columnIsTemporal(ctx, q, key)
temporal, err := m.columnIsTemporal(ctx, q.StartNs, q.EndNs, key)
if err != nil {
return qbtypes.Read{}, err
}
@@ -532,18 +532,18 @@ func foldAbsentJSONReadToTypeDefault(key *telemetrytypes.TelemetryFieldKey, oper
// and corrects to the attribute maps when it names no column. A strict
// context synthesizes its type variants under the stripped and the literal
// spelling.
func (m *storage) Fallback(ctx context.Context, q qbtypes.QueryInfo, key *telemetrytypes.TelemetryFieldKey, _ qbtypes.FilterOperator, value any) ([]*telemetrytypes.LogicalField, error) {
func (m *storage) Fallback(ctx context.Context, _ qbtypes.QueryInfo, key *telemetrytypes.TelemetryFieldKey, _ qbtypes.FilterOperator, value any) ([]*telemetrytypes.LogicalField, error) {
var keys []*telemetrytypes.TelemetryFieldKey
switch key.FieldContext {
case telemetrytypes.FieldContextUnspecified:
probe := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextSpan, key.FieldDataType)
if columns, err := m.getColumn(ctx, q, probe); err == nil {
if columns, err := m.getColumn(ctx, 0, 0, probe); err == nil {
keys = []*telemetrytypes.TelemetryFieldKey{stampColumnType(probe, columns)}
} else {
keys = querybuilder.SynthesizeKeys(key, value)
}
case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextTrace:
if columns, err := m.getColumn(ctx, q, key); err == nil {
if columns, err := m.getColumn(ctx, 0, 0, key); err == nil {
column := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType)
keys = []*telemetrytypes.TelemetryFieldKey{stampColumnType(column, columns)}
} else {

View File

@@ -22,7 +22,7 @@ var (
)
func readSQL(ctx context.Context, storage qbtypes.Storage, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
read, err := storage.Read(ctx, qbtypes.QueryInfo{StartNs: startNs, EndNs: endNs, TraceAttrsJSONOn: true}, key)
read, err := storage.Read(ctx, qbtypes.QueryInfo{StartNs: startNs, EndNs: endNs}, key)
return read.SQL, err
}
@@ -96,40 +96,6 @@ func TestFieldForAttributeNoEvolutionParity(t *testing.T) {
}
}
// TestAttributeJSONFlagOffParity proves the evolution entry alone does not switch reads to the
// JSON column: with use_trace_attributes_json off, reads and conditions stay on the Map for every window.
func TestAttributeJSONFlagOffParity(t *testing.T) {
ctx := context.Background()
storage := NewStorage()
evo := MockAttributeEvolutionData(attrJSONRelease)
testCases := []struct {
name string
window [2]uint64
}{
{"BeforeRelease", attrWindowBefore},
{"AfterRelease", attrWindowAfter},
{"StraddlingRelease", attrWindowStraddle},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeNumber, evo)
q := qbtypes.QueryInfo{StartNs: testCase.window[0], EndNs: testCase.window[1]}
read, err := storage.Read(ctx, q, &key)
require.NoError(t, err)
assert.Equal(t, "attributes_number['user.id']", read.SQL)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, q, storage, &key, qbtypes.FilterOperatorNotEqual, float64(1), map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
require.Len(t, conds, 1)
assert.NotContains(t, conds[0], "attributes.`user.id`")
})
}
}
// TestConditionForAttributeJSON asserts the emitted WHERE fragment per operator against the JSON
// column (window fully after release). Positive operators carry the raw-path existence guard;
// numeric comparisons keep numeric semantics; existence never tests the ::String cast.
@@ -204,7 +170,7 @@ func TestConditionForAttributeJSON(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &tc.key, tc.operator, tc.value, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &tc.key, tc.operator, tc.value, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -223,7 +189,7 @@ func TestConditionForAttributeJSONNotExistsDualRead(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorNotExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1]}, storage, &key, qbtypes.FilterOperatorNotExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -243,14 +209,14 @@ func TestColumnExpressionForAttributeJSON(t *testing.T) {
t.Run("group by string", func(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, telemetrytypes.FieldDataTypeString, nil)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &key, telemetrytypes.FieldDataTypeString, nil)
require.NoError(t, err)
assert.Equal(t, "multiIf(attributes.`user.id` IS NOT NULL, attributes.`user.id`::String, mapContains(attributes_string, 'attribute.user.id'), attributes_string['attribute.user.id'], NULL)", got)
})
t.Run("aggregation numeric", func(t *testing.T) {
key := attrKey("latency", telemetrytypes.FieldDataTypeNumber, evo)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, telemetrytypes.FieldDataTypeFloat64, nil)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &key, telemetrytypes.FieldDataTypeFloat64, nil)
require.NoError(t, err)
assert.Equal(t, "multiIf(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL) IS NOT NULL, toFloat64(if(dynamicType(attributes.`latency`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`latency`, 'Float64'), NULL)), mapContains(attributes_number, 'attribute.latency'), toFloat64(attributes_number['attribute.latency']), NULL)", got)
})
@@ -266,7 +232,7 @@ func TestAttributeJSONNoAmbiguityWarning(t *testing.T) {
key := attrKey("user.id", telemetrytypes.FieldDataTypeString, evo)
sb := sqlbuilder.NewSelectBuilder()
_, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorEqual, "x", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
_, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &key, qbtypes.FilterOperatorEqual, "x", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
assert.Empty(t, warnings, "a plain attribute filter must not emit an ambiguity warning")
}
@@ -288,7 +254,7 @@ func TestConditionForAttributeJSONTypeCollision(t *testing.T) {
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
sb := sqlbuilder.NewSelectBuilder()
conds, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
conds, warnings, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
require.NoError(t, err)
require.Len(t, conds, 2, "a colliding name must build one condition per data type")
@@ -317,7 +283,7 @@ func TestColumnExpressionForAttributeJSONTypeCollision(t *testing.T) {
}
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, telemetrytypes.FieldDataTypeString, fieldKeys)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &ref, telemetrytypes.FieldDataTypeString, fieldKeys)
require.NoError(t, err)
assert.Equal(t,
"multiIf(attributes.`http.status_code` IS NOT NULL, attributes.`http.status_code`::String, if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, toString(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)), NULL)",
@@ -341,7 +307,7 @@ func TestColumnExpressionForAttributeJSONTypeCollisionNumericAgg(t *testing.T) {
}
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, storage, &ref, telemetrytypes.FieldDataTypeFloat64, fieldKeys)
got, err := querybuilder.ResolveColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1]}, storage, &ref, telemetrytypes.FieldDataTypeFloat64, fieldKeys)
require.NoError(t, err)
assert.Equal(t,
"multiIf(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL) IS NOT NULL, toFloat64(if(dynamicType(attributes.`http.status_code`) IN ('Int64', 'UInt64', 'Float64'), accurateCastOrNull(attributes.`http.status_code`, 'Float64'), NULL)), attributes.`http.status_code` IS NOT NULL, toFloat64OrNull(attributes.`http.status_code`::String), NULL)",
@@ -364,7 +330,7 @@ func TestConditionForAttributeMapTypeCollisionParity(t *testing.T) {
ref := attrKey("http.status_code", telemetrytypes.FieldDataTypeUnspecified, nil)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowBefore[0], EndNs: attrWindowBefore[1], TraceAttrsJSONOn: true}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowBefore[0], EndNs: attrWindowBefore[1]}, storage, &ref, qbtypes.FilterOperatorEqual, float64(200), fieldKeys, false, sb)
require.NoError(t, err)
require.Len(t, conds, 2, "a colliding name must build one condition per data type")
@@ -385,7 +351,7 @@ func TestColumnForUnspecifiedAttributeNoBranchFlip(t *testing.T) {
evo := MockAttributeEvolutionData(attrJSONRelease)
key := attrKey("user.id", telemetrytypes.FieldDataTypeUnspecified, evo)
_, err := (&storage{}).getColumn(ctx, qbtypes.QueryInfo{StartNs: attrWindowAfter[0], EndNs: attrWindowAfter[1], TraceAttrsJSONOn: true}, &key)
_, err := (&storage{}).getColumn(ctx, attrWindowAfter[0], attrWindowAfter[1], &key)
assert.ErrorIs(t, err, qbtypes.ErrColumnNotFound)
}
@@ -403,7 +369,7 @@ func TestConditionForAttributeJSONNegativeOperatorParity(t *testing.T) {
build := func(t *testing.T, key telemetrytypes.TelemetryFieldKey, window [2]uint64, op qbtypes.FilterOperator, value any) string {
t.Helper()
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: window[0], EndNs: window[1], TraceAttrsJSONOn: true}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: window[0], EndNs: window[1]}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -483,7 +449,7 @@ func TestConditionForAttributeJSONStraddleAbsentKeyExclusion(t *testing.T) {
build := func(t *testing.T, key telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) string {
t.Helper()
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1], TraceAttrsJSONOn: true}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: attrWindowStraddle[0], EndNs: attrWindowStraddle[1]}, storage, &key, op, value, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)

View File

@@ -81,7 +81,7 @@ func TestConditionForAttributePromoted(t *testing.T) {
t.Run("equal reads promoted column only", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorEqual, "GET", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1]}, storage, &key, qbtypes.FilterOperatorEqual, "GET", map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -91,7 +91,7 @@ func TestConditionForAttributePromoted(t *testing.T) {
t.Run("exists uses promoted raw path", func(t *testing.T) {
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1], TraceAttrsJSONOn: true}, storage, &key, qbtypes.FilterOperatorExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
conds, _, err := querybuilder.Conditions(ctx, qbtypes.QueryInfo{StartNs: afterPromo[0], EndNs: afterPromo[1]}, storage, &key, qbtypes.FilterOperatorExists, nil, map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, false, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)

View File

@@ -1,131 +0,0 @@
package promotetypes
import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// Target identifies a promotion domain: the column evolution record written
// per promoted path, the table per-path indexes are created on, and the API
// path rules.
type Target struct {
Entry telemetrytypes.EvolutionEntry // evolution row template; FieldName and ReleaseTime are set per write
DBName string // index DDL database, used only when IndexesSupported
LocalTableName string // index DDL local table, used only when IndexesSupported
BaseColumn string // column holding every path; indexes for unpromoted paths are created on it
RequiredPathPrefix string // prefix API paths must carry, stripped before storing; empty for bare names
IndexesSupported bool // whether per-path skip indexes can be created for this domain
}
func (t Target) PromotedColumn() string { return t.Entry.ColumnName }
func (t Target) BaseColumnPrefix() string { return t.BaseColumn + "." }
func (t Target) PromotedColumnPrefix() string { return t.PromotedColumn() + "." }
// NewTarget creates the Target for a promotion domain.
func NewTarget(entry telemetrytypes.EvolutionEntry, dbName, localTableName, baseColumn, requiredPathPrefix string, indexesSupported bool) Target {
return Target{
Entry: entry,
DBName: dbName,
LocalTableName: localTableName,
BaseColumn: baseColumn,
RequiredPathPrefix: requiredPathPrefix,
IndexesSupported: indexesSupported,
}
}
// NewLogsBodyTarget returns the domain for the logs body JSON column
// (body_v2 -> body_promoted), with per-path skip index support.
func NewLogsBodyTarget() Target {
return NewTarget(
telemetrytypes.EvolutionEntry{
Signal: telemetrytypes.SignalLogs,
ColumnName: logstelemetryschema.LogsV2BodyPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
},
logstelemetryschema.DBName,
logstelemetryschema.LogsV2LocalTableName,
logstelemetryschema.LogsV2BodyV2Column,
telemetrytypes.BodyJSONStringSearchPrefix,
true,
)
}
// NewTracesAttributesTarget returns the domain for the spans attributes JSON
// column (attributes -> attributes_promoted); promotion only for now.
func NewTracesAttributesTarget() Target {
return NewTarget(
telemetrytypes.EvolutionEntry{
Signal: telemetrytypes.SignalTraces,
ColumnName: tracestelemetryschema.SpanAttributesPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextAttribute,
},
tracestelemetryschema.DBName,
tracestelemetryschema.SpanIndexV3LocalTableName,
tracestelemetryschema.SpanAttributesColumn,
"",
false,
)
}
// NewTargetFromPath validates the {telemetry_signal} and {context} path
// variables and returns their promotion domain.
func NewTargetFromPath(signal, context string) (Target, error) {
params := &PathParams{Signal: signal, Context: context}
if err := params.Validate(); err != nil {
return Target{}, err
}
parsedSignal, _ := telemetrytypes.SignalFromText(params.Signal)
parsedContext, _ := telemetrytypes.FieldContextFromText(params.Context)
target, _ := TargetFor(parsedSignal, parsedContext)
return target, nil
}
// Targets returns every supported promotion domain.
func Targets() []Target {
return []Target{
NewLogsBodyTarget(),
NewTracesAttributesTarget(),
}
}
// TargetFor resolves the domain for a (signal, context) pair; ok is false
// when no domain exists for the pair.
func TargetFor(signal telemetrytypes.Signal, context telemetrytypes.FieldContext) (Target, bool) {
for _, target := range Targets() {
if target.Entry.Signal.StringValue() == signal.StringValue() &&
target.Entry.FieldContext.StringValue() == context.StringValue() {
return target, true
}
}
return Target{}, false
}
// PathParams carries the raw {telemetry_signal} and {context} path variables
// of the promote paths API.
type PathParams struct {
Signal string
Context string
}
// Validate ensures the path variables are known values naming a supported
// promotion domain.
func (p *PathParams) Validate() error {
signal, ok := telemetrytypes.SignalFromText(p.Signal)
if !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid signal: %s", p.Signal)
}
context, ok := telemetrytypes.FieldContextFromText(p.Context)
if !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid context: %s", p.Context)
}
if _, ok := TargetFor(signal, context); !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "promotion is not supported for %s %s", signal.StringValue(), context.StringValue())
}
return nil
}

View File

@@ -3,6 +3,7 @@ package promotetypes
import (
"strings"
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz-otel-collector/pkg/keycheck"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -22,7 +23,7 @@ type PromotePath struct {
Indexes []WrappedIndex `json:"indexes,omitempty"`
}
func (i *PromotePath) ValidateAndSetDefaults(target Target) error {
func (i *PromotePath) ValidateAndSetDefaults() error {
if i.Path == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path is required")
}
@@ -35,27 +36,22 @@ func (i *PromotePath) ValidateAndSetDefaults(target Target) error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "array paths can not be promoted or indexed")
}
if strings.HasPrefix(i.Path, target.BaseColumnPrefix()) || strings.HasPrefix(i.Path, target.PromotedColumnPrefix()) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", target.BaseColumnPrefix(), target.PromotedColumnPrefix())
if strings.HasPrefix(i.Path, constants.BodyV2ColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyV2ColumnPrefix, constants.BodyPromotedColumnPrefix)
}
if target.RequiredPathPrefix != "" {
if !strings.HasPrefix(i.Path, target.RequiredPathPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `%s`", target.RequiredPathPrefix)
}
// remove the required prefix from the path
i.Path = strings.TrimPrefix(i.Path, target.RequiredPathPrefix)
if !strings.HasPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `body.`")
}
// remove the "body." prefix from the path
i.Path = strings.TrimPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix)
isCardinal := keycheck.IsCardinal(i.Path)
if isCardinal {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cardinal paths can not be promoted or indexed")
}
if len(i.Indexes) > 0 && !target.IndexesSupported {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "indexes are not supported for %s %s", target.Entry.Signal.StringValue(), target.Entry.FieldContext.StringValue())
}
for idx, index := range i.Indexes {
if index.Type == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index type is required")

View File

@@ -1,172 +0,0 @@
package promotetypes
import (
"testing"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateAndSetDefaultsLogsBody(t *testing.T) {
target := NewLogsBodyTarget()
testCases := []struct {
name string
path *PromotePath
wantErr bool
wantPath string
wantJSONDataType telemetrytypes.JSONDataType
}{
{
name: "ValidPath_BodyPrefixStripped",
path: &PromotePath{Path: "body.user.name", Promote: true},
wantPath: "user.name",
},
{
name: "PathWithoutBodyPrefix_Rejected",
path: &PromotePath{Path: "user.name", Promote: true},
wantErr: true,
},
{
name: "BodyV2PrefixedPath_Rejected",
path: &PromotePath{Path: "body_v2.user.name", Promote: true},
wantErr: true,
},
{
name: "BodyPromotedPrefixedPath_Rejected",
path: &PromotePath{Path: "body_promoted.user.name", Promote: true},
wantErr: true,
},
{
name: "EmptyPath_Rejected",
path: &PromotePath{Path: "", Promote: true},
wantErr: true,
},
{
name: "SpacedPath_Rejected",
path: &PromotePath{Path: "body.my path", Promote: true},
wantErr: true,
},
{
name: "ArrayIndexPath_Rejected",
path: &PromotePath{Path: "body.users[].id", Promote: true},
wantErr: true,
},
{
name: "ArrayWildcardPath_Rejected",
path: &PromotePath{Path: "body.users[*].id", Promote: true},
wantErr: true,
},
{
name: "CardinalPath_Rejected",
path: &PromotePath{Path: "body.request.550e8400-e29b-41d4-a716-446655440000", Promote: true},
wantErr: true,
},
{
name: "ValidIndex_JSONDataTypeDefaulted",
path: &PromotePath{
Path: "body.user.name",
Indexes: []WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantPath: "user.name",
wantJSONDataType: telemetrytypes.String,
},
{
name: "UnsupportedColumnTypeIndex_Rejected",
path: &PromotePath{
Path: "body.user.active",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeBool, Type: "minmax", Granularity: 1}},
},
wantErr: true,
},
{
name: "IndexWithoutType_Rejected",
path: &PromotePath{
Path: "body.user.name",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Granularity: 1}},
},
wantErr: true,
},
{
name: "IndexWithoutGranularity_Rejected",
path: &PromotePath{
Path: "body.user.name",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "minmax"}},
},
wantErr: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := testCase.path.ValidateAndSetDefaults(target)
if testCase.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.wantPath, testCase.path.Path)
if testCase.wantJSONDataType != (telemetrytypes.JSONDataType{}) {
require.Len(t, testCase.path.Indexes, 1)
assert.Equal(t, testCase.wantJSONDataType, testCase.path.Indexes[0].JSONDataType)
}
})
}
}
func TestValidateAndSetDefaultsTracesAttributes(t *testing.T) {
target := NewTracesAttributesTarget()
testCases := []struct {
name string
path *PromotePath
wantErr bool
wantPath string
}{
{
name: "BareAttributeName_KeptAsIs",
path: &PromotePath{Path: "http.method", Promote: true},
wantPath: "http.method",
},
{
name: "AttributesPrefixedPath_Rejected",
path: &PromotePath{Path: "attributes.http.method", Promote: true},
wantErr: true,
},
{
name: "AttributesPromotedPrefixedPath_Rejected",
path: &PromotePath{Path: "attributes_promoted.http.method", Promote: true},
wantErr: true,
},
{
name: "EmptyPath_Rejected",
path: &PromotePath{Path: "", Promote: true},
wantErr: true,
},
{
name: "SpacedPath_Rejected",
path: &PromotePath{Path: "my attr", Promote: true},
wantErr: true,
},
{
name: "ArrayIndexPath_Rejected",
path: &PromotePath{Path: "tags[].id", Promote: true},
wantErr: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := testCase.path.ValidateAndSetDefaults(target)
if testCase.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.wantPath, testCase.path.Path)
})
}
}

View File

@@ -13,7 +13,6 @@ import (
// SelectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
// Logic:
// - Ignores evolutions of columns outside the candidate columns
// - Finds the latest base evolution (<= tsStartTime) across ALL columns
// - Rejects all evolutions before this latest base evolution
// - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
@@ -24,11 +23,6 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
return columns, nil, nil
}
columnLookUpMap := make(map[string]*schema.Column, len(columns))
for _, column := range columns {
columnLookUpMap[column.Name] = column
}
// Derive the base column from the candidate columns.
seen := make(map[string]struct{}, len(evolutions))
for _, e := range evolutions {
@@ -70,9 +64,6 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
if evolution.ReleaseTime.After(tsStartTime) {
break
}
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
continue
}
latestBaseEvolutionAcrossAll = evolution
}
@@ -81,6 +72,11 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
}
columnLookUpMap := make(map[string]*schema.Column)
for _, column := range columns {
columnLookUpMap[column.Name] = column
}
// Collect column-evolution pairs
type colEvoPair struct {
column *schema.Column
@@ -99,7 +95,7 @@ func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetr
}
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
continue
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
}
pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})

View File

@@ -208,7 +208,7 @@ func TestSelectEvolutionsForColumns(t *testing.T) {
expectedColumns: []string{},
expectedEvols: []string{},
expectedError: true,
errorStr: "no base evolution found",
errorStr: "column resources_string not found",
},
{
name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
@@ -441,26 +441,6 @@ func TestSelectEvolutionsForColumns(t *testing.T) {
expectedColumns: []string{"attributes"},
expectedEvols: []string{"attributes"},
},
{
name: "Non-candidate evolution ignored - JSON released before window keeps the map",
columns: []*schema.Column{
attributes_string,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalTraces,
ColumnName: "attributes",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldName: "__all__",
ReleaseTime: time.Date(2024, 2, 10, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"attributes_string"},
expectedEvols: []string{"attributes_string"},
},
}
for _, tc := range testCases {

View File

@@ -39,16 +39,14 @@ var (
// QueryInfo is the query's context as one value. It holds the time range
// every read needs, the signal and queried metric that family admission
// needs, and the query-path flags evaluated one time per request. The
// generic flows read FamiliesOn. Only the logs storage reads BodyJSONOn, and
// only the traces storage reads TraceAttrsJSONOn.
// generic flows read FamiliesOn. Only the logs storage reads BodyJSONOn.
type QueryInfo struct {
StartNs uint64
EndNs uint64
Signal telemetrytypes.Signal
Metric *telemetrytypes.MetricContext
FamiliesOn bool
BodyJSONOn bool
TraceAttrsJSONOn bool
StartNs uint64
EndNs uint64
Signal telemetrytypes.Signal
Metric *telemetrytypes.MetricContext
FamiliesOn bool
BodyJSONOn bool
}
// Absent is how a field key reads for a row that does not carry it, with

View File

@@ -22,18 +22,3 @@ func (Signal) Enum() []any {
SignalUnspecified,
}
}
// SignalFromText resolves a signal word to its Signal; ok is false for an
// unknown word.
func SignalFromText(text string) (Signal, bool) {
s := Signal{valuer.NewString(text)}
switch s {
case SignalTraces:
return SignalTraces, true
case SignalLogs:
return SignalLogs, true
case SignalMetrics:
return SignalMetrics, true
}
return Signal{}, false
}

View File

@@ -37,13 +37,11 @@ type MetadataStore interface {
// ListLogsJSONIndexes lists the JSON indexes for the logs table.
ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]TelemetryFieldKeySkipIndex, error)
// GetPromotedPaths lists the promoted paths recorded in the column
// evolution table for the entry's signal, column and field context.
GetPromotedPaths(ctx context.Context, entry EvolutionEntry, paths ...string) (map[string]bool, error)
// ListPromotedPaths lists the promoted paths.
GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error)
// PromotePaths records promoted paths in the column evolution table as
// rows templated by entry; FieldName and ReleaseTime are set per path.
PromotePaths(ctx context.Context, entry EvolutionEntry, paths ...string) error
// PromotePaths promotes the paths.
PromotePaths(ctx context.Context, paths ...string) error
// GetFirstSeenFromMetricMetadata gets the first seen timestamp for a metric metadata lookup key.
GetFirstSeenFromMetricMetadata(ctx context.Context, lookupKeys []MetricMetadataLookupKey) (map[MetricMetadataLookupKey]int64, error)

View File

@@ -361,7 +361,7 @@ func (m *MockMetadataStore) SetTemporality(metricName string, temporality metric
}
// PromotePaths promotes the paths.
func (m *MockMetadataStore) PromotePaths(_ context.Context, _ telemetrytypes.EvolutionEntry, paths ...string) error {
func (m *MockMetadataStore) PromotePaths(ctx context.Context, paths ...string) error {
for _, path := range paths {
m.PromotedPathsMap[path] = true
}
@@ -369,7 +369,7 @@ func (m *MockMetadataStore) PromotePaths(_ context.Context, _ telemetrytypes.Evo
}
// GetPromotedPaths returns the promoted paths.
func (m *MockMetadataStore) GetPromotedPaths(_ context.Context, _ telemetrytypes.EvolutionEntry, _ ...string) (map[string]bool, error) {
func (m *MockMetadataStore) GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error) {
return m.PromotedPathsMap, nil
}

View File

@@ -1,30 +0,0 @@
import pytest
from testcontainers.core.container import Network
from fixtures import types
from fixtures.signoz import create_signoz
@pytest.fixture(name="signoz", scope="package")
def signoz_trace_attributes_json(
network: Network,
zeus: types.TestContainerDocker,
gateway: types.TestContainerDocker,
sqlstore: types.TestContainerSQL,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.SigNoz:
return create_signoz(
network=network,
zeus=zeus,
gateway=gateway,
sqlstore=sqlstore,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
cache_key="signoz-trace-attributes-json",
env_overrides={
"SIGNOZ_FLAGGER_CONFIG_BOOLEAN_USE__TRACE__ATTRIBUTES__JSON": True,
},
)