Compare commits

...

12 Commits

Author SHA1 Message Date
Nikhil Soni
9dec58c1f6 test(promote): cover the per-path skip index creation of the logs body domain 2026-09-16 20:02:17 +05:30
Nikhil Soni
3856a3ba41 refactor(promote): move the path resolution to types with a validate method, table-drive the tests 2026-09-16 19:42:11 +05:30
Nikhil Soni
1d7489499f chore: regenerate openapi spec and api clients 2026-09-16 19:21:53 +05:30
Nikhil Soni
4390b754f2 test: align the subtest names with the table format rule 2026-09-16 19:20:01 +05:30
Nikhil Soni
8d00527156 fix(promote): rename the signal path variable to telemetry_signal
orval generates an AbortSignal parameter named signal for every client
method, so a {signal} path variable produced a duplicate identifier in
the generated client (tsc error). The URL itself is unchanged in
behavior: /api/v1/promote_paths/{telemetry_signal}/{context}.
2026-09-16 19:19:28 +05:30
Nikhil Soni
1855d1cf5e refactor(promote): inline the promote and list helpers into their sole callers 2026-09-16 19:19:28 +05:30
Nikhil Soni
306a3e81d6 refactor(promote)!: drop the legacy logs promote_paths routes
There are no consumers of /api/v1/logs/promote_paths, so no backward
compatibility is needed: the logs body domain is served by the generic
/api/v1/promote_paths/{signal}/{context} routes and the legacy routes
and handler methods are removed.
2026-09-16 19:19:27 +05:30
Nikhil Soni
03726711ac refactor(promote): move Target into target.go, enum-style SignalFromText, rename handler method
- Target type definition moves from types.go to target.go alongside its
  constructors, with inline comments
- SignalFromText follows the codebase enum pattern (switch over the
  declared values + Enum method) instead of a string-to-signal map
- generic route handler method renamed HandlePromotePaths -> PromotePaths
2026-09-16 19:19:27 +05:30
Nikhil Soni
ffc8b6b319 refactor(promote): centralize domain construction and generalize routes
- target construction moves to promotetypes: a generic NewTarget plus
  per-domain constructors (NewLogsBodyTarget, NewTracesAttributesTarget)
  and a TargetFor registry keyed by (signal, context); implpromote and
  telemetrymetadata no longer hand-roll domain literals
- routes generalize to /api/v1/promote_paths/{signal}/{context}: the
  legacy logs body route (/api/v1/logs/promote_paths) is kept for
  compatibility but the domain now travels in the path, so a future logs
  attribute domain does not collide with the logs body route; supersedes
  the /api/v1/traces/promote_paths routes
- add telemetrytypes.SignalFromText for parsing the signal path variable
2026-09-16 19:19:27 +05:30
Nikhil Soni
0b612652c2 refactor(promote): template the promotion record with EvolutionEntry
Target now carries an EvolutionEntry template (signal, promoted column
name and type, field context) instead of loose signal/context/column
fields, so the store write is exactly row template + field names +
release time and the hardcoded JSON() column type moves to the domain
definitions. DBName/LocalTableName stay on Target explicitly as index
DDL config, used only by targets with index support.
2026-09-16 19:19:27 +05:30
Nikhil Soni
bb9d23de7e refactor(promote): collapse module interface to target-parameterized methods
The per-domain methods were pure delegates; the promotion domain now
travels as promotetypes.Target through Module.ListPromotedPaths /
Module.PromotePaths, with the handler methods (one per route) passing
their domain's target.
2026-09-16 19:19:27 +05:30
Nikhil Soni
ec51a3b55b feat(promote): add traces attributes promotion API
Refactor the promote module into a target-parameterized core so the logs
body_v2 flow and future promotion domains share one implementation, and
add the spans attributes JSON column (attributes -> attributes_promoted)
as a second domain behind POST/GET /api/v1/traces/promote_paths.

- promotetypes.Target describes a promotion domain: signal, field
  context, db/table, base/promoted columns, path prefix rule and whether
  per-path skip indexes are supported
- index support is optional per target; traces starts promotion-only
  since the traces query builder does not consume per-path skip indexes
- metadata store GetPromotedPaths/PromotePaths take (signal, column,
  context) instead of being hardcoded to the logs body column
- fix the list response never attaching indexes to promoted entries
  (aggregated by unprefixed name but looked up by prefixed path) and
  reporting indexed+promoted paths twice
2026-09-16 19:19:27 +05:30
17 changed files with 1169 additions and 447 deletions

View File

@@ -12814,108 +12814,6 @@ paths:
summary: List unmapped models
tags:
- llmpricingrules
/api/v1/logs/promote_paths:
get:
deprecated: false
description: This endpoints promotes and indexes paths
operationId: ListPromotedAndIndexedPaths
responses:
"200":
content:
application/json:
schema:
properties:
data:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Promote and index paths
tags:
- logs
post:
deprecated: false
description: This endpoints promotes and indexes paths
operationId: HandlePromoteAndIndexPaths
requestBody:
content:
application/json:
schema:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
responses:
"201":
description: Created
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- EDITOR
- tokenizer:
- EDITOR
summary: Promote and index paths
tags:
- logs
/api/v1/org/preferences:
get:
deprecated: false
@@ -13084,6 +12982,134 @@ paths:
summary: Update org preference
tags:
- preferences
/api/v1/promote_paths/{telemetry_signal}/{context}:
get:
deprecated: false
description: This endpoint lists the promoted paths of a JSON column. The promotion
domain is identified by the telemetry_signal and context path variables, e.g.
traces/attribute.
operationId: ListPromotedPaths
parameters:
- in: path
name: telemetry_signal
required: true
schema:
type: string
- in: path
name: context
required: true
schema:
type: string
responses:
"200":
content:
application/json:
schema:
properties:
data:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: List promoted paths
tags:
- promote
post:
deprecated: false
description: This endpoint promotes paths of a JSON column to its promoted column.
The promotion domain is identified by the telemetry_signal and context path
variables, e.g. traces/attribute.
operationId: PromotePaths
parameters:
- in: path
name: telemetry_signal
required: true
schema:
type: string
- in: path
name: context
required: true
schema:
type: string
requestBody:
content:
application/json:
schema:
items:
$ref: '#/components/schemas/PromotetypesPromotePath'
nullable: true
type: array
responses:
"201":
description: Created
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- EDITOR
- tokenizer:
- EDITOR
summary: Promote paths
tags:
- promote
/api/v1/public/dashboards/{id}:
get:
deprecated: false

View File

@@ -4,23 +4,15 @@
* * regenerate with 'pnpm generate:api'
* SigNoz
*/
import { useMutation, useQuery } from 'react-query';
import { useMutation } from 'react-query';
import type {
InvalidateOptions,
MutationFunction,
QueryClient,
QueryFunction,
QueryKey,
UseMutationOptions,
UseMutationResult,
UseQueryOptions,
UseQueryResult,
} from 'react-query';
import type {
HandleExportRawDataPOSTParams,
ListPromotedAndIndexedPaths200,
PromotetypesPromotePathDTO,
Querybuildertypesv5QueryRangeRequestDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
@@ -28,26 +20,6 @@ import type {
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type { ErrorType, BodyType } from '../../../generatedAPIInstance';
const withQueryKey = <T extends object, K>(
query: T,
queryKey: K,
): T & { queryKey: K } => {
const result = { queryKey } as T & { queryKey: K };
for (const key of Object.keys(query)) {
// The explicit queryKey always wins, matching the previous
// `{ ...query, queryKey }` spread where it was set last.
if (key === 'queryKey') {
continue;
}
Object.defineProperty(result, key, {
enumerable: true,
configurable: true,
get: () => (query as Record<string, unknown>)[key],
});
}
return result;
};
/**
* This endpoints allows complex query exporting raw data for traces and logs
* @summary Export raw data
@@ -149,175 +121,3 @@ export const useHandleExportRawDataPOST = <
> => {
return useMutation(getHandleExportRawDataPOSTMutationOptions(options));
};
/**
* This endpoints promotes and indexes paths
* @summary Promote and index paths
*/
export const listPromotedAndIndexedPaths = (signal?: AbortSignal) => {
return GeneratedAPIInstance<ListPromotedAndIndexedPaths200>({
url: `/api/v1/logs/promote_paths`,
method: 'GET',
signal,
});
};
export const getListPromotedAndIndexedPathsQueryKey = () => {
return [`/api/v1/logs/promote_paths`] as const;
};
export const getListPromotedAndIndexedPathsQueryOptions = <
TData = Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError,
TData
>;
}) => {
const { query: queryOptions } = options ?? {};
const queryKey =
queryOptions?.queryKey ?? getListPromotedAndIndexedPathsQueryKey();
const queryFn: QueryFunction<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>
> = ({ signal }) => listPromotedAndIndexedPaths(signal);
return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError,
TData
> & { queryKey: QueryKey };
};
export type ListPromotedAndIndexedPathsQueryResult = NonNullable<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>
>;
export type ListPromotedAndIndexedPathsQueryError =
ErrorType<RenderErrorResponseDTO>;
/**
* @summary Promote and index paths
*/
export function useListPromotedAndIndexedPaths<
TData = Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedAndIndexedPaths>>,
TError,
TData
>;
}): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
const queryOptions = getListPromotedAndIndexedPathsQueryOptions(options);
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
queryKey: QueryKey;
};
return withQueryKey(query, queryOptions.queryKey);
}
/**
* @summary Promote and index paths
*/
export const invalidateListPromotedAndIndexedPaths = async (
queryClient: QueryClient,
options?: InvalidateOptions,
): Promise<QueryClient> => {
await queryClient.invalidateQueries(
{ queryKey: getListPromotedAndIndexedPathsQueryKey() },
options,
);
return queryClient;
};
/**
* This endpoints promotes and indexes paths
* @summary Promote and index paths
*/
export const handlePromoteAndIndexPaths = (
promotetypesPromotePathDTONull?: BodyType<
PromotetypesPromotePathDTO[] | null
> | null,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<void>({
url: `/api/v1/logs/promote_paths`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: promotetypesPromotePathDTONull,
signal,
});
};
export const getHandlePromoteAndIndexPathsMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
> => {
const mutationKey = ['handlePromoteAndIndexPaths'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> }
> = (props) => {
const { data } = props ?? {};
return handlePromoteAndIndexPaths(data);
};
return { mutationFn, ...mutationOptions };
};
export type HandlePromoteAndIndexPathsMutationResult = NonNullable<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>
>;
export type HandlePromoteAndIndexPathsMutationBody =
| BodyType<PromotetypesPromotePathDTO[] | null>
| undefined;
export type HandlePromoteAndIndexPathsMutationError =
ErrorType<RenderErrorResponseDTO>;
/**
* @summary Promote and index paths
*/
export const useHandlePromoteAndIndexPaths = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof handlePromoteAndIndexPaths>>,
TError,
{ data?: BodyType<PromotetypesPromotePathDTO[] | null> },
TContext
> => {
return useMutation(getHandlePromoteAndIndexPathsMutationOptions(options));
};

View File

@@ -0,0 +1,262 @@
/**
* ! Do not edit manually
* * The file has been auto-generated using Orval for SigNoz
* * regenerate with 'pnpm generate:api'
* SigNoz
*/
import { useMutation, useQuery } from 'react-query';
import type {
InvalidateOptions,
MutationFunction,
QueryClient,
QueryFunction,
QueryKey,
UseMutationOptions,
UseMutationResult,
UseQueryOptions,
UseQueryResult,
} from 'react-query';
import type {
ListPromotedPaths200,
ListPromotedPathsPathParameters,
PromotePathsPathParameters,
PromotetypesPromotePathDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type { ErrorType, BodyType } from '../../../generatedAPIInstance';
const withQueryKey = <T extends object, K>(
query: T,
queryKey: K,
): T & { queryKey: K } => {
const result = { queryKey } as T & { queryKey: K };
for (const key of Object.keys(query)) {
// The explicit queryKey always wins, matching the previous
// `{ ...query, queryKey }` spread where it was set last.
if (key === 'queryKey') {
continue;
}
Object.defineProperty(result, key, {
enumerable: true,
configurable: true,
get: () => (query as Record<string, unknown>)[key],
});
}
return result;
};
/**
* This endpoint lists the promoted paths of a JSON column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.
* @summary List promoted paths
*/
export const listPromotedPaths = (
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<ListPromotedPaths200>({
url: `/api/v1/promote_paths/${telemetrySignal}/${context}`,
method: 'GET',
signal,
});
};
export const getListPromotedPathsQueryKey = ({
telemetrySignal,
context,
}: ListPromotedPathsPathParameters) => {
return [`/api/v1/promote_paths/${telemetrySignal}/${context}`] as const;
};
export const getListPromotedPathsQueryOptions = <
TData = Awaited<ReturnType<typeof listPromotedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedPaths>>,
TError,
TData
>;
},
) => {
const { query: queryOptions } = options ?? {};
const queryKey =
queryOptions?.queryKey ??
getListPromotedPathsQueryKey({ telemetrySignal, context });
const queryFn: QueryFunction<
Awaited<ReturnType<typeof listPromotedPaths>>
> = ({ signal }) => listPromotedPaths({ telemetrySignal, context }, signal);
return {
queryKey,
queryFn,
enabled:
telemetrySignal !== null &&
telemetrySignal !== undefined &&
context !== null &&
context !== undefined,
...queryOptions,
} as UseQueryOptions<
Awaited<ReturnType<typeof listPromotedPaths>>,
TError,
TData
> & { queryKey: QueryKey };
};
export type ListPromotedPathsQueryResult = NonNullable<
Awaited<ReturnType<typeof listPromotedPaths>>
>;
export type ListPromotedPathsQueryError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary List promoted paths
*/
export function useListPromotedPaths<
TData = Awaited<ReturnType<typeof listPromotedPaths>>,
TError = ErrorType<RenderErrorResponseDTO>,
>(
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof listPromotedPaths>>,
TError,
TData
>;
},
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
const queryOptions = getListPromotedPathsQueryOptions(
{ telemetrySignal, context },
options,
);
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
queryKey: QueryKey;
};
return withQueryKey(query, queryOptions.queryKey);
}
/**
* @summary List promoted paths
*/
export const invalidateListPromotedPaths = async (
queryClient: QueryClient,
{ telemetrySignal, context }: ListPromotedPathsPathParameters,
options?: InvalidateOptions,
): Promise<QueryClient> => {
await queryClient.invalidateQueries(
{ queryKey: getListPromotedPathsQueryKey({ telemetrySignal, context }) },
options,
);
return queryClient;
};
/**
* This endpoint promotes paths of a JSON column to its promoted column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.
* @summary Promote paths
*/
export const promotePaths = (
{ telemetrySignal, context }: PromotePathsPathParameters,
promotetypesPromotePathDTONull?: BodyType<
PromotetypesPromotePathDTO[] | null
> | null,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<void>({
url: `/api/v1/promote_paths/${telemetrySignal}/${context}`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: promotetypesPromotePathDTONull,
signal,
});
};
export const getPromotePathsMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
> => {
const mutationKey = ['promotePaths'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof promotePaths>>,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
}
> = (props) => {
const { pathParams, data } = props ?? {};
return promotePaths(pathParams, data);
};
return { mutationFn, ...mutationOptions };
};
export type PromotePathsMutationResult = NonNullable<
Awaited<ReturnType<typeof promotePaths>>
>;
export type PromotePathsMutationBody =
| BodyType<PromotetypesPromotePathDTO[] | null>
| undefined;
export type PromotePathsMutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Promote paths
*/
export const usePromotePaths = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof promotePaths>>,
TError,
{
pathParams: PromotePathsPathParameters;
data?: BodyType<PromotetypesPromotePathDTO[] | null>;
},
TContext
> => {
return useMutation(getPromotePathsMutationOptions(options));
};

View File

@@ -12227,17 +12227,6 @@ export type ListUnmappedLLMModels200 = {
status: string;
};
export type ListPromotedAndIndexedPaths200 = {
/**
* @type array,null
*/
data: PromotetypesPromotePathDTO[] | null;
/**
* @type string
*/
status: string;
};
export type ListOrgPreferences200 = {
/**
* @type array
@@ -12263,6 +12252,25 @@ export type GetOrgPreference200 = {
export type UpdateOrgPreferencePathParameters = {
name: string;
};
export type ListPromotedPathsPathParameters = {
telemetrySignal: string;
context: string;
};
export type ListPromotedPaths200 = {
/**
* @type array,null
*/
data: PromotetypesPromotePathDTO[] | null;
/**
* @type string
*/
status: string;
};
export type PromotePathsPathParameters = {
telemetrySignal: string;
context: string;
};
export type GetPublicDashboardDataPathParameters = {
id: string;
};

View File

@@ -10,11 +10,11 @@ import (
)
func (provider *provider) addPromoteRoutes(router *mux.Router) error {
if err := router.Handle("/api/v1/logs/promote_paths", handler.New(provider.authzMiddleware.EditAccess(provider.promoteHandler.HandlePromoteAndIndexPaths), handler.OpenAPIDef{
ID: "HandlePromoteAndIndexPaths",
Tags: []string{"logs"},
Summary: "Promote and index paths",
Description: "This endpoints promotes and indexes paths",
if err := router.Handle("/api/v1/promote_paths/{telemetry_signal}/{context}", handler.New(provider.authzMiddleware.EditAccess(provider.promoteHandler.PromotePaths), handler.OpenAPIDef{
ID: "PromotePaths",
Tags: []string{"promote"},
Summary: "Promote paths",
Description: "This endpoint promotes paths of a JSON column to its promoted column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.",
Request: new([]*promotetypes.PromotePath),
RequestContentType: "application/json",
Response: nil,
@@ -26,11 +26,11 @@ func (provider *provider) addPromoteRoutes(router *mux.Router) error {
return err
}
if err := router.Handle("/api/v1/logs/promote_paths", handler.New(provider.authzMiddleware.ViewAccess(provider.promoteHandler.ListPromotedAndIndexedPaths), handler.OpenAPIDef{
ID: "ListPromotedAndIndexedPaths",
Tags: []string{"logs"},
Summary: "Promote and index paths",
Description: "This endpoints promotes and indexes paths",
if err := router.Handle("/api/v1/promote_paths/{telemetry_signal}/{context}", handler.New(provider.authzMiddleware.ViewAccess(provider.promoteHandler.ListPromotedPaths), handler.OpenAPIDef{
ID: "ListPromotedPaths",
Tags: []string{"promote"},
Summary: "List promoted paths",
Description: "This endpoint lists the promoted paths of a JSON column. The promotion domain is identified by the telemetry_signal and context path variables, e.g. traces/attribute.",
Request: nil,
RequestContentType: "",
Response: new([]*promotetypes.PromotePath),

View File

@@ -9,6 +9,7 @@ import (
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/gorilla/mux"
)
type handler struct {
@@ -19,9 +20,16 @@ func NewHandler(module promote.Module) promote.Handler {
return &handler{module: module}
}
func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Request) {
func (h *handler) PromotePaths(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
target, err := promotetypes.NewTargetFromPath(vars["telemetry_signal"], vars["context"])
if err != nil {
render.Error(w, err)
return
}
// TODO(Nitya): Use in multi tenant setup
_, err := authtypes.ClaimsFromContext(r.Context())
_, err = authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, errors.NewInternalf(errors.CodeInternal, "failed to get org id from context"))
return
@@ -33,7 +41,7 @@ func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Requ
return
}
err = h.module.PromoteAndIndexPaths(r.Context(), req...)
err = h.module.PromotePaths(r.Context(), target, req...)
if err != nil {
render.Error(w, err)
return
@@ -42,15 +50,22 @@ func (h *handler) HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Requ
render.Success(w, http.StatusCreated, nil)
}
func (h *handler) ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request) {
func (h *handler) ListPromotedPaths(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
target, err := promotetypes.NewTargetFromPath(vars["telemetry_signal"], vars["context"])
if err != nil {
render.Error(w, err)
return
}
// TODO(Nitya): Use in multi tenant setup
_, err := authtypes.ClaimsFromContext(r.Context())
_, err = authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, errors.NewInternalf(errors.CodeInternal, "failed to get org id from context"))
return
}
paths, err := h.module.ListPromotedAndIndexedPaths(r.Context())
paths, err := h.module.ListPromotedPaths(r.Context(), target)
if err != nil {
render.Error(w, err)
return

View File

@@ -2,14 +2,11 @@ package implpromote
import (
"context"
"maps"
"slices"
"strings"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
@@ -31,45 +28,57 @@ func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore teleme
return &module{metadataStore: metadataStore, telemetryStore: telemetrystore}
}
func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error) {
// ListPromotedPaths lists the promoted paths of the target JSON column,
// merged with per-path index metadata where the target supports indexes.
func (m *module) ListPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error) {
promotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry)
if err != nil {
return nil, err
}
response := make([]promotetypes.PromotePath, 0, len(promotedPaths))
for path := range promotedPaths {
response = append(response, promotetypes.PromotePath{
Path: target.RequiredPathPrefix + path,
Promote: true,
})
}
// Index metadata is optional per target; merge it in only where supported.
if !target.IndexesSupported {
return response, nil
}
indexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
if err != nil {
return nil, err
}
// index.Name is the bare path and index.BaseColumn carries the column
// prefix, so the aggregate key is the full sub-column path.
aggr := map[string][]promotetypes.WrappedIndex{}
for _, index := range indexes {
aggr[index.Name] = append(aggr[index.Name], promotetypes.WrappedIndex{
fullPath := index.BaseColumn + index.Name
aggr[fullPath] = append(aggr[fullPath], promotetypes.WrappedIndex{
FieldDataType: index.FieldDataType,
Type: index.IndexType,
Granularity: index.Granularity,
})
}
promotedPaths, err := m.listPromotedPaths(ctx)
if err != nil {
return nil, err
}
response := []promotetypes.PromotePath{}
for _, path := range promotedPaths {
fullPath := logstelemetryschema.BodyPromotedColumnPrefix + path
path = telemetrytypes.BodyJSONStringSearchPrefix + path
item := promotetypes.PromotePath{
Path: path,
Promote: true,
}
indexes, ok := aggr[fullPath]
if ok {
item.Indexes = indexes
for i := range response {
fullPath := target.PromotedColumnPrefix() + strings.TrimPrefix(response[i].Path, target.RequiredPathPrefix)
if indexes, ok := aggr[fullPath]; ok {
response[i].Indexes = indexes
delete(aggr, fullPath)
}
response = append(response, item)
}
// add the paths that are not promoted but have indexes
for path, indexes := range aggr {
path := strings.TrimPrefix(path, logstelemetryschema.BodyV2ColumnPrefix)
path = telemetrytypes.BodyJSONStringSearchPrefix + path
for fullPath, indexes := range aggr {
path := strings.TrimPrefix(fullPath, target.BaseColumnPrefix())
path = strings.TrimPrefix(path, target.PromotedColumnPrefix())
path = target.RequiredPathPrefix + path
response = append(response, promotetypes.PromotePath{
Path: path,
Indexes: indexes,
@@ -78,54 +87,10 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
return response, nil
}
func (m *module) listPromotedPaths(ctx context.Context) ([]string, error) {
paths, err := m.metadataStore.GetPromotedPaths(ctx)
if err != nil {
return nil, err
}
return slices.Collect(maps.Keys(paths)), nil
}
// PromotePaths inserts provided JSON paths into the promoted paths table for logs queries.
func (m *module) PromotePaths(ctx context.Context, paths []string) error {
if len(paths) == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
}
return m.metadataStore.PromotePaths(ctx, paths...)
}
// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Index) error {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.CodeNamespace: "promote",
instrumentationtypes.CodeFunctionName: "createIndexes",
})
if len(indexes) == 0 {
return nil
}
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
Database: logstelemetryschema.DBName,
Table: logstelemetryschema.LogsV2LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
if err := m.telemetryStore.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
}
}
return nil
}
// PromoteAndIndexPaths handles promoting paths and creating indexes in one call.
func (m *module) PromoteAndIndexPaths(
ctx context.Context,
paths ...*promotetypes.PromotePath,
) error {
// PromotePaths records new promotions of the target JSON column in the column
// evolution table and, for targets with index support, creates the requested
// per-path skip indexes.
func (m *module) PromotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error {
if len(paths) == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
}
@@ -133,13 +98,13 @@ func (m *module) PromoteAndIndexPaths(
pathsStr := []string{}
// validate the paths
for _, path := range paths {
if err := path.ValidateAndSetDefaults(); err != nil {
if err := path.ValidateAndSetDefaults(target); err != nil {
return err
}
pathsStr = append(pathsStr, path.Path)
}
existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, pathsStr...)
existingPromotedPaths, err := m.metadataStore.GetPromotedPaths(ctx, target.Entry, pathsStr...)
if err != nil {
return err
}
@@ -153,10 +118,10 @@ func (m *module) PromoteAndIndexPaths(
}
}
if len(it.Indexes) > 0 {
parentColumn := logstelemetryschema.LogsV2BodyV2Column
parentColumn := target.BaseColumn
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
parentColumn = logstelemetryschema.LogsV2BodyPromotedColumn
parentColumn = target.PromotedColumn()
}
for _, index := range it.Indexes {
@@ -182,17 +147,43 @@ func (m *module) PromoteAndIndexPaths(
}
if len(toInsert) > 0 {
err := m.PromotePaths(ctx, toInsert)
err := m.metadataStore.PromotePaths(ctx, target.Entry, toInsert...)
if err != nil {
return err
}
}
if len(indexes) > 0 {
if err := m.createIndexes(ctx, indexes); err != nil {
if err := m.createIndexes(ctx, target, indexes); err != nil {
return err
}
}
return nil
}
// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
func (m *module) createIndexes(ctx context.Context, target promotetypes.Target, indexes []schemamigrator.Index) error {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: target.Entry.Signal.StringValue(),
instrumentationtypes.CodeNamespace: "promote",
instrumentationtypes.CodeFunctionName: "createIndexes",
})
if len(indexes) == 0 {
return nil
}
for _, index := range indexes {
alterStmt := schemamigrator.AlterTableAddIndex{
Database: target.DBName,
Table: target.LocalTableName,
Index: index,
}
op := alterStmt.OnCluster(m.telemetryStore.Cluster())
if err := m.telemetryStore.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
}
}
return nil
}

View File

@@ -0,0 +1,246 @@
package implpromote
import (
"context"
"regexp"
"testing"
sqlmock "github.com/DATA-DOG/go-sqlmock"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPromotePaths(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
target promotetypes.Target
paths []*promotetypes.PromotePath
promoteTwice bool
wantErr bool
wantPromoted []string
}{
{
name: "PromotesNewAttributes_Idempotent",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{
{Path: "http.method", Promote: true},
{Path: "span.operation", Promote: true},
},
promoteTwice: true,
wantPromoted: []string{"http.method", "span.operation"},
},
{
name: "NonPromoteEntries_NotRecorded",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{Path: "http.method"}},
},
{
name: "IndexesOnTraceAttributes_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{
Path: "http.method",
Promote: true,
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
}},
wantErr: true,
},
{
name: "ColumnPrefixedPath_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{Path: "attributes.http.method", Promote: true}},
wantErr: true,
},
{
name: "EmptyPath_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
paths: []*promotetypes.PromotePath{{Path: "", Promote: true}},
wantErr: true,
},
{
name: "EmptyRequest_Rejected",
target: promotetypes.NewTracesAttributesTarget(),
wantErr: true,
},
{
name: "PromotesBodyPath_PrefixStripped",
target: promotetypes.NewLogsBodyTarget(),
paths: []*promotetypes.PromotePath{{Path: "body.user.name", Promote: true}},
wantPromoted: []string{"user.name"},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
store := telemetrytypestest.NewMockMetadataStore()
m := NewModule(store, nil)
err := m.PromotePaths(ctx, testCase.target, testCase.paths...)
if testCase.wantErr {
assert.Error(t, err)
assert.Empty(t, store.PromotedPathsMap)
return
}
require.NoError(t, err)
require.Len(t, store.PromotedPathsMap, len(testCase.wantPromoted))
for _, path := range testCase.wantPromoted {
assert.True(t, store.PromotedPathsMap[path], path)
}
if testCase.promoteTwice {
// promoting again must not fail
require.NoError(t, m.PromotePaths(ctx, testCase.target, testCase.paths...))
assert.Len(t, store.PromotedPathsMap, len(testCase.wantPromoted))
}
})
}
}
func TestPromotePathsCreatesIndexes(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
promoted map[string]bool
path *promotetypes.PromotePath
wantDDLColumn string
}{
{
name: "NewPromotion_IndexesPromotedColumn",
path: &promotetypes.PromotePath{
Path: "body.user.name",
Promote: true,
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantDDLColumn: "dynamicElement(body_promoted.user.name",
},
{
name: "AlreadyPromoted_IndexesPromotedColumn",
promoted: map[string]bool{"user.name": true},
path: &promotetypes.PromotePath{
Path: "body.user.name",
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantDDLColumn: "dynamicElement(body_promoted.user.name",
},
{
name: "UnpromotedPath_IndexesBaseColumn",
path: &promotetypes.PromotePath{
Path: "body.user.name",
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantDDLColumn: "dynamicElement(body_v2.user.name",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
ts := telemetrystoretest.New(telemetrystore.Config{}, sqlmock.QueryMatcherRegexp)
store := telemetrytypestest.NewMockMetadataStore()
if testCase.promoted != nil {
store.PromotedPathsMap = testCase.promoted
}
m := NewModule(store, ts)
ts.Mock().ExpectExec("ADD INDEX (.+)" + regexp.QuoteMeta(testCase.wantDDLColumn)).WillReturnError(nil)
require.NoError(t, m.PromotePaths(ctx, promotetypes.NewLogsBodyTarget(), testCase.path))
assert.NoError(t, ts.Mock().ExpectationsWereMet())
})
}
}
func TestListPromotedPaths(t *testing.T) {
ctx := context.Background()
testCases := []struct {
name string
target promotetypes.Target
promoted map[string]bool
indexes []telemetrytypes.TelemetryFieldKeySkipIndex
wantPaths []promotetypes.PromotePath
}{
{
name: "TracesAttributes_PromotedPaths",
target: promotetypes.NewTracesAttributesTarget(),
promoted: map[string]bool{"http.method": true},
wantPaths: []promotetypes.PromotePath{
{Path: "http.method", Promote: true},
},
},
{
name: "LogsBody_PromotedAndIndexedPaths",
target: promotetypes.NewLogsBodyTarget(),
promoted: map[string]bool{"user.name": true},
indexes: []telemetrytypes.TelemetryFieldKeySkipIndex{
{
Name: "user.name",
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: telemetrytypes.FieldDataTypeString,
BaseColumn: "body_promoted.",
IndexType: "ngrambf_v1(4, 1024, 2, 0)",
Granularity: 1,
},
{
Name: "request.duration",
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: telemetrytypes.FieldDataTypeFloat64,
BaseColumn: "body_v2.",
IndexType: "minmax",
Granularity: 1,
},
},
wantPaths: []promotetypes.PromotePath{
{
Path: "body.user.name",
Promote: true,
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
{
Path: "body.request.duration",
Indexes: []promotetypes.WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeFloat64, Type: "minmax", Granularity: 1},
},
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
store := telemetrytypestest.NewMockMetadataStore()
store.PromotedPathsMap = testCase.promoted
store.LogsJSONIndexes = testCase.indexes
m := NewModule(store, nil)
paths, err := m.ListPromotedPaths(ctx, testCase.target)
require.NoError(t, err)
require.Len(t, paths, len(testCase.wantPaths))
byPath := map[string]promotetypes.PromotePath{}
for _, path := range paths {
byPath[path.Path] = path
}
for _, want := range testCase.wantPaths {
require.Contains(t, byPath, want.Path)
assert.Equal(t, want, byPath[want.Path])
}
})
}
}

View File

@@ -8,11 +8,11 @@ import (
)
type Module interface {
ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error)
PromoteAndIndexPaths(ctx context.Context, paths ...*promotetypes.PromotePath) error
ListPromotedPaths(ctx context.Context, target promotetypes.Target) ([]promotetypes.PromotePath, error)
PromotePaths(ctx context.Context, target promotetypes.Target, paths ...*promotetypes.PromotePath) error
}
type Handler interface {
HandlePromoteAndIndexPaths(w http.ResponseWriter, r *http.Request)
ListPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request)
PromotePaths(w http.ResponseWriter, r *http.Request)
ListPromotedPaths(w http.ResponseWriter, r *http.Request)
}

View File

@@ -16,6 +16,7 @@ import (
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
"github.com/SigNoz/signoz/pkg/types/promotetypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
)
@@ -33,6 +34,10 @@ var (
CodeFailedToAppendPath = errors.MustNewCode("failed_to_append_path_promoted_paths")
)
// logsBodyPromotedEntry templates the column evolution rows recorded for
// logs body promotions.
var logsBodyPromotedEntry = promotetypes.NewLogsBodyTarget().Entry
// enrichJSONKeys enriches body-context keys with promoted path info, indexes,
// and JSON access plans. parentTypeCache contains parent array types (ArrayJSON/ArrayDynamic)
// pre-fetched in the main UNION query.
@@ -67,7 +72,7 @@ func (t *telemetryMetaStore) enrichJSONKeys(ctx context.Context, selectors []*te
}
// fetch promoted paths
promoted, err := t.GetPromotedPaths(ctx, paths...)
promoted, err := t.GetPromotedPaths(ctx, logsBodyPromotedEntry, paths...)
if err != nil {
return err
}
@@ -157,7 +162,7 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
}
func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
ctx = withTelemetryContext(ctx, "ListLogsJSONIndexes")
ctx = withTelemetryContext(ctx, telemetrytypes.SignalLogs, "ListLogsJSONIndexes")
query, args := buildListLogsJSONIndexesQuery(t.telemetrystore.Cluster(), filters...)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
@@ -215,14 +220,14 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
// TODO(Piyush): Remove this if not used in future.
func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, limit int) (*telemetrytypes.TelemetryFieldValues, bool, error) {
ctx = withTelemetryContext(ctx, "ListJSONValues")
ctx = withTelemetryContext(ctx, telemetrytypes.SignalLogs, "ListJSONValues")
path = CleanPathPrefixes(path)
if strings.Contains(path, telemetrytypes.ArraySep) || strings.Contains(path, telemetrytypes.ArrayAnyIndex) {
return nil, false, errors.NewInvalidInputf(errors.CodeInvalidInput, "array paths are not supported")
}
promoted, err := t.IsPathPromoted(ctx, path)
promoted, err := t.isPathPromoted(ctx, logsBodyPromotedEntry, path)
if err != nil {
return nil, false, err
}
@@ -376,13 +381,13 @@ func derefValue(v any) any {
return val.Interface()
}
// IsPathPromoted checks if a specific path is promoted (Column Evolution table: field_name for logs body).
func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (bool, error) {
ctx = withTelemetryContext(ctx, "IsPathPromoted")
// isPathPromoted checks if a specific path is promoted (Column Evolution table: field_name for the entry's column).
func (t *telemetryMetaStore) isPathPromoted(ctx context.Context, entry telemetrytypes.EvolutionEntry, path string) (bool, error) {
ctx = withTelemetryContext(ctx, entry.Signal, "isPathPromoted")
split := strings.Split(path, telemetrytypes.ArraySep)
pathSegment := split[0]
query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE signal = ? AND column_name = ? AND field_context = ? AND field_name = ? LIMIT 1", DBName, PromotedPathsTableName)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, telemetrytypes.FieldContextBody, pathSegment)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, entry.Signal, entry.ColumnName, entry.FieldContext, pathSegment)
if err != nil {
return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
}
@@ -391,14 +396,14 @@ func (t *telemetryMetaStore) IsPathPromoted(ctx context.Context, path string) (b
return rows.Next(), nil
}
// GetPromotedPaths returns promoted paths from the Column Evolution table (field_name for logs body).
func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error) {
ctx = withTelemetryContext(ctx, "GetPromotedPaths")
// GetPromotedPaths returns promoted paths from the Column Evolution table (field_name for the entry's column).
func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) (map[string]bool, error) {
ctx = withTelemetryContext(ctx, entry.Signal, "GetPromotedPaths")
sb := sqlbuilder.Select("field_name").From(fmt.Sprintf("%s.%s", DBName, PromotedPathsTableName))
conditions := []string{
sb.Equal("signal", telemetrytypes.SignalLogs),
sb.Equal("column_name", logstelemetryschema.LogsV2BodyPromotedColumn),
sb.Equal("field_context", telemetrytypes.FieldContextBody),
sb.Equal("signal", entry.Signal),
sb.Equal("column_name", entry.ColumnName),
sb.Equal("field_context", entry.FieldContext),
sb.NotEqual("field_name", "__all__"),
}
if len(paths) > 0 {
@@ -438,9 +443,10 @@ func CleanPathPrefixes(path string) string {
return path
}
// PromotePaths inserts promoted paths into the Column Evolution table (same schema as signoz-otel-collector metadata_migrations).
func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string) error {
ctx = withTelemetryContext(ctx, "PromotePaths")
// PromotePaths inserts promoted paths into the Column Evolution table as rows templated by entry
// (same schema as signoz-otel-collector metadata_migrations); FieldName and ReleaseTime are set per path.
func (t *telemetryMetaStore) PromotePaths(ctx context.Context, entry telemetrytypes.EvolutionEntry, paths ...string) error {
ctx = withTelemetryContext(ctx, entry.Signal, "PromotePaths")
batch, err := t.telemetrystore.ClickhouseDB().PrepareBatch(ctx,
fmt.Sprintf("INSERT INTO %s.%s (signal, column_name, column_type, field_context, field_name, version, release_time) VALUES", DBName,
PromotedPathsTableName))
@@ -454,7 +460,7 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
if trimmed == "" {
continue
}
if err := batch.Append(telemetrytypes.SignalLogs, logstelemetryschema.LogsV2BodyPromotedColumn, "JSON()", telemetrytypes.FieldContextBody, trimmed, 0, releaseTime); err != nil {
if err := batch.Append(entry.Signal, entry.ColumnName, entry.ColumnType, entry.FieldContext, trimmed, entry.Version, releaseTime); err != nil {
_ = batch.Abort()
return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
}
@@ -466,9 +472,9 @@ func (t *telemetryMetaStore) PromotePaths(ctx context.Context, paths ...string)
return nil
}
func withTelemetryContext(ctx context.Context, functionName string) context.Context {
func withTelemetryContext(ctx context.Context, signal telemetrytypes.Signal, functionName string) context.Context {
return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.TelemetrySignal: signal.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: functionName,
})

View File

@@ -0,0 +1,131 @@
package promotetypes
import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// Target identifies a promotion domain: the column evolution record written
// per promoted path, the table per-path indexes are created on, and the API
// path rules.
type Target struct {
Entry telemetrytypes.EvolutionEntry // evolution row template; FieldName and ReleaseTime are set per write
DBName string // index DDL database, used only when IndexesSupported
LocalTableName string // index DDL local table, used only when IndexesSupported
BaseColumn string // column holding every path; indexes for unpromoted paths are created on it
RequiredPathPrefix string // prefix API paths must carry, stripped before storing; empty for bare names
IndexesSupported bool // whether per-path skip indexes can be created for this domain
}
func (t Target) PromotedColumn() string { return t.Entry.ColumnName }
func (t Target) BaseColumnPrefix() string { return t.BaseColumn + "." }
func (t Target) PromotedColumnPrefix() string { return t.PromotedColumn() + "." }
// NewTarget creates the Target for a promotion domain.
func NewTarget(entry telemetrytypes.EvolutionEntry, dbName, localTableName, baseColumn, requiredPathPrefix string, indexesSupported bool) Target {
return Target{
Entry: entry,
DBName: dbName,
LocalTableName: localTableName,
BaseColumn: baseColumn,
RequiredPathPrefix: requiredPathPrefix,
IndexesSupported: indexesSupported,
}
}
// NewLogsBodyTarget returns the domain for the logs body JSON column
// (body_v2 -> body_promoted), with per-path skip index support.
func NewLogsBodyTarget() Target {
return NewTarget(
telemetrytypes.EvolutionEntry{
Signal: telemetrytypes.SignalLogs,
ColumnName: logstelemetryschema.LogsV2BodyPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
},
logstelemetryschema.DBName,
logstelemetryschema.LogsV2LocalTableName,
logstelemetryschema.LogsV2BodyV2Column,
telemetrytypes.BodyJSONStringSearchPrefix,
true,
)
}
// NewTracesAttributesTarget returns the domain for the spans attributes JSON
// column (attributes -> attributes_promoted); promotion only for now.
func NewTracesAttributesTarget() Target {
return NewTarget(
telemetrytypes.EvolutionEntry{
Signal: telemetrytypes.SignalTraces,
ColumnName: tracestelemetryschema.SpanAttributesPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextAttribute,
},
tracestelemetryschema.DBName,
tracestelemetryschema.SpanIndexV3LocalTableName,
tracestelemetryschema.SpanAttributesColumn,
"",
false,
)
}
// Targets returns every supported promotion domain.
func Targets() []Target {
return []Target{
NewLogsBodyTarget(),
NewTracesAttributesTarget(),
}
}
// TargetFor resolves the domain for a (signal, context) pair; ok is false
// when no domain exists for the pair.
func TargetFor(signal telemetrytypes.Signal, context telemetrytypes.FieldContext) (Target, bool) {
for _, target := range Targets() {
if target.Entry.Signal.StringValue() == signal.StringValue() &&
target.Entry.FieldContext.StringValue() == context.StringValue() {
return target, true
}
}
return Target{}, false
}
// PathParams carries the raw {telemetry_signal} and {context} path variables
// of the promote paths API.
type PathParams struct {
Signal string
Context string
}
// Validate ensures the path variables are known values naming a supported
// promotion domain.
func (p *PathParams) Validate() error {
signal, ok := telemetrytypes.SignalFromText(p.Signal)
if !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid signal: %s", p.Signal)
}
context, ok := telemetrytypes.FieldContextFromText(p.Context)
if !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid context: %s", p.Context)
}
if _, ok := TargetFor(signal, context); !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "promotion is not supported for %s %s", signal.StringValue(), context.StringValue())
}
return nil
}
// NewTargetFromPath validates the {telemetry_signal} and {context} path
// variables and returns their promotion domain.
func NewTargetFromPath(signal, context string) (Target, error) {
params := &PathParams{Signal: signal, Context: context}
if err := params.Validate(); err != nil {
return Target{}, err
}
parsedSignal, _ := telemetrytypes.SignalFromText(params.Signal)
parsedContext, _ := telemetrytypes.FieldContextFromText(params.Context)
target, _ := TargetFor(parsedSignal, parsedContext)
return target, nil
}

View File

@@ -0,0 +1,36 @@
package promotetypes
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewTargetFromPath(t *testing.T) {
testCases := []struct {
name string
signal string
context string
wantErr bool
wantTarget Target
}{
{name: "LogsBody_Resolved", signal: "logs", context: "body", wantTarget: NewLogsBodyTarget()},
{name: "TracesAttributes_Resolved", signal: "traces", context: "attribute", wantTarget: NewTracesAttributesTarget()},
{name: "InvalidSignal_Rejected", signal: "span", context: "attribute", wantErr: true},
{name: "InvalidContext_Rejected", signal: "traces", context: "header", wantErr: true},
{name: "UnsupportedPair_Rejected", signal: "metrics", context: "attribute", wantErr: true},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
target, err := NewTargetFromPath(testCase.signal, testCase.context)
if testCase.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.wantTarget, target)
})
}
}

View File

@@ -3,7 +3,6 @@ package promotetypes
import (
"strings"
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz-otel-collector/pkg/keycheck"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -23,7 +22,7 @@ type PromotePath struct {
Indexes []WrappedIndex `json:"indexes,omitempty"`
}
func (i *PromotePath) ValidateAndSetDefaults() error {
func (i *PromotePath) ValidateAndSetDefaults(target Target) error {
if i.Path == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path is required")
}
@@ -36,22 +35,27 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "array paths can not be promoted or indexed")
}
if strings.HasPrefix(i.Path, constants.BodyV2ColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyV2ColumnPrefix, constants.BodyPromotedColumnPrefix)
if strings.HasPrefix(i.Path, target.BaseColumnPrefix()) || strings.HasPrefix(i.Path, target.PromotedColumnPrefix()) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", target.BaseColumnPrefix(), target.PromotedColumnPrefix())
}
if !strings.HasPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `body.`")
if target.RequiredPathPrefix != "" {
if !strings.HasPrefix(i.Path, target.RequiredPathPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `%s`", target.RequiredPathPrefix)
}
// remove the required prefix from the path
i.Path = strings.TrimPrefix(i.Path, target.RequiredPathPrefix)
}
// remove the "body." prefix from the path
i.Path = strings.TrimPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix)
isCardinal := keycheck.IsCardinal(i.Path)
if isCardinal {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cardinal paths can not be promoted or indexed")
}
if len(i.Indexes) > 0 && !target.IndexesSupported {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "indexes are not supported for %s %s", target.Entry.Signal.StringValue(), target.Entry.FieldContext.StringValue())
}
for idx, index := range i.Indexes {
if index.Type == "" {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index type is required")

View File

@@ -0,0 +1,180 @@
package promotetypes
import (
"testing"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestValidateAndSetDefaultsLogsBody(t *testing.T) {
target := NewLogsBodyTarget()
testCases := []struct {
name string
path *PromotePath
wantErr bool
wantPath string
wantJSONDataType telemetrytypes.JSONDataType
}{
{
name: "ValidPath_BodyPrefixStripped",
path: &PromotePath{Path: "body.user.name", Promote: true},
wantPath: "user.name",
},
{
name: "PathWithoutBodyPrefix_Rejected",
path: &PromotePath{Path: "user.name", Promote: true},
wantErr: true,
},
{
name: "BodyV2PrefixedPath_Rejected",
path: &PromotePath{Path: "body_v2.user.name", Promote: true},
wantErr: true,
},
{
name: "BodyPromotedPrefixedPath_Rejected",
path: &PromotePath{Path: "body_promoted.user.name", Promote: true},
wantErr: true,
},
{
name: "EmptyPath_Rejected",
path: &PromotePath{Path: "", Promote: true},
wantErr: true,
},
{
name: "SpacedPath_Rejected",
path: &PromotePath{Path: "body.my path", Promote: true},
wantErr: true,
},
{
name: "ArrayIndexPath_Rejected",
path: &PromotePath{Path: "body.users[].id", Promote: true},
wantErr: true,
},
{
name: "ArrayWildcardPath_Rejected",
path: &PromotePath{Path: "body.users[*].id", Promote: true},
wantErr: true,
},
{
name: "CardinalPath_Rejected",
path: &PromotePath{Path: "body.request.550e8400-e29b-41d4-a716-446655440000", Promote: true},
wantErr: true,
},
{
name: "ValidIndex_JSONDataTypeDefaulted",
path: &PromotePath{
Path: "body.user.name",
Indexes: []WrappedIndex{
{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1},
},
},
wantPath: "user.name",
wantJSONDataType: telemetrytypes.String,
},
{
name: "UnsupportedColumnTypeIndex_Rejected",
path: &PromotePath{
Path: "body.user.active",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeBool, Type: "minmax", Granularity: 1}},
},
wantErr: true,
},
{
name: "IndexWithoutType_Rejected",
path: &PromotePath{
Path: "body.user.name",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Granularity: 1}},
},
wantErr: true,
},
{
name: "IndexWithoutGranularity_Rejected",
path: &PromotePath{
Path: "body.user.name",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "minmax"}},
},
wantErr: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := testCase.path.ValidateAndSetDefaults(target)
if testCase.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.wantPath, testCase.path.Path)
if testCase.wantJSONDataType != (telemetrytypes.JSONDataType{}) {
require.Len(t, testCase.path.Indexes, 1)
assert.Equal(t, testCase.wantJSONDataType, testCase.path.Indexes[0].JSONDataType)
}
})
}
}
func TestValidateAndSetDefaultsTracesAttributes(t *testing.T) {
target := NewTracesAttributesTarget()
testCases := []struct {
name string
path *PromotePath
wantErr bool
wantPath string
}{
{
name: "BareAttributeName_KeptAsIs",
path: &PromotePath{Path: "http.method", Promote: true},
wantPath: "http.method",
},
{
name: "AttributesPrefixedPath_Rejected",
path: &PromotePath{Path: "attributes.http.method", Promote: true},
wantErr: true,
},
{
name: "AttributesPromotedPrefixedPath_Rejected",
path: &PromotePath{Path: "attributes_promoted.http.method", Promote: true},
wantErr: true,
},
{
name: "EmptyPath_Rejected",
path: &PromotePath{Path: "", Promote: true},
wantErr: true,
},
{
name: "SpacedPath_Rejected",
path: &PromotePath{Path: "my attr", Promote: true},
wantErr: true,
},
{
name: "ArrayIndexPath_Rejected",
path: &PromotePath{Path: "tags[].id", Promote: true},
wantErr: true,
},
{
name: "IndexesWithoutSupport_Rejected",
path: &PromotePath{
Path: "http.method",
Indexes: []WrappedIndex{{FieldDataType: telemetrytypes.FieldDataTypeString, Type: "ngrambf_v1(4, 1024, 2, 0)", Granularity: 1}},
},
wantErr: true,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
err := testCase.path.ValidateAndSetDefaults(target)
if testCase.wantErr {
assert.Error(t, err)
return
}
require.NoError(t, err)
assert.Equal(t, testCase.wantPath, testCase.path.Path)
})
}
}

View File

@@ -22,3 +22,18 @@ func (Signal) Enum() []any {
SignalUnspecified,
}
}
// SignalFromText resolves a signal word to its Signal; ok is false for an
// unknown word.
func SignalFromText(text string) (Signal, bool) {
s := Signal{valuer.NewString(text)}
switch s {
case SignalTraces:
return SignalTraces, true
case SignalLogs:
return SignalLogs, true
case SignalMetrics:
return SignalMetrics, true
}
return Signal{}, false
}

View File

@@ -37,11 +37,13 @@ type MetadataStore interface {
// ListLogsJSONIndexes lists the JSON indexes for the logs table.
ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]TelemetryFieldKeySkipIndex, error)
// ListPromotedPaths lists the promoted paths.
GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error)
// GetPromotedPaths lists the promoted paths recorded in the column
// evolution table for the entry's signal, column and field context.
GetPromotedPaths(ctx context.Context, entry EvolutionEntry, paths ...string) (map[string]bool, error)
// PromotePaths promotes the paths.
PromotePaths(ctx context.Context, paths ...string) error
// PromotePaths records promoted paths in the column evolution table as
// rows templated by entry; FieldName and ReleaseTime are set per path.
PromotePaths(ctx context.Context, entry EvolutionEntry, paths ...string) error
// GetFirstSeenFromMetricMetadata gets the first seen timestamp for a metric metadata lookup key.
GetFirstSeenFromMetricMetadata(ctx context.Context, lookupKeys []MetricMetadataLookupKey) (map[MetricMetadataLookupKey]int64, error)

View File

@@ -361,7 +361,7 @@ func (m *MockMetadataStore) SetTemporality(metricName string, temporality metric
}
// PromotePaths promotes the paths.
func (m *MockMetadataStore) PromotePaths(ctx context.Context, paths ...string) error {
func (m *MockMetadataStore) PromotePaths(_ context.Context, _ telemetrytypes.EvolutionEntry, paths ...string) error {
for _, path := range paths {
m.PromotedPathsMap[path] = true
}
@@ -369,7 +369,7 @@ func (m *MockMetadataStore) PromotePaths(ctx context.Context, paths ...string) e
}
// GetPromotedPaths returns the promoted paths.
func (m *MockMetadataStore) GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error) {
func (m *MockMetadataStore) GetPromotedPaths(_ context.Context, _ telemetrytypes.EvolutionEntry, _ ...string) (map[string]bool, error) {
return m.PromotedPathsMap, nil
}