Compare commits

..

1 Commits

Author SHA1 Message Date
srikanthccv
f20aabc3ba feat: resolve semantic convention names in trace queries
A referenced name now resolves to logical fields (LogicalField, #12499).
A slice shows ambiguity: each logical field gets its own condition. A
group shows a semantic-convention family: one field with more than one
spelling. Compilers merge a family with current-wins precedence, and
the response keeps the spelling that the request used.

- FieldMapper gets one new method: ExistsFor, the per-key existence
  primitive. Each mapper already had this knowledge; two rename a
  private method and five expose logic from their condition builders.
- LogicalValueExpr and LogicalExistsExpr build all family SQL in one
  place, only from FieldFor and ExistsFor. No signal implements family
  logic.
- Statement builders prefetch sibling spellings with
  ExpandKeySelectorsForFamilies. The metadata store stays family-blind,
  and autocomplete stays literal.
- The traces and resource-filter condition builders compile per logical
  field. The traces group-by path upgrades legacy candidates to their
  family in a post-pass; the legacy candidate flow does not change.
- Logs, metrics, audit, metadata, and rule-state-history flatten to
  single keys; their SQL does not change.

The resolve_semconv_families feature flag (default: disabled) gates all
family behavior. The components that resolve names hold the flagger and
evaluate the flag at the point of use, with the org from the call, the
same way the logs mapper uses use_json_body. A nil flagger means off,
so resolution stays literal on any path without one. With the flag off,
the generated SQL is the same as main, and tests pin this. The legacy
compatibility-key map stays in place for the same reason.

Assisted-by: Claude Fable 5
2026-08-11 15:36:08 +05:30
70 changed files with 12076 additions and 2914 deletions

View File

@@ -376,19 +376,7 @@ function App(): JSX.Element {
tracesSampleRate: 0, // Ref: https://github.com/SigNoz/platform-pod/issues/2393#issuecomment-4603658055
replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production.
replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur.
beforeSend(event, hint) {
const error = hint?.originalException as
| { name?: string; code?: string | number }
| undefined;
// Ignore benign aborted/cancelled requests (axios + fetch).
if (error?.code === 'ERR_CANCELED' || error?.code === 'ECONNABORTED') {
return null;
}
if (error?.name === 'AbortError') {
return null;
}
beforeSend(event) {
// Drop the event if its level is 'warning' or 'info'
if (event.level === 'warning' || event.level === 'info') {
return null;

View File

@@ -0,0 +1,31 @@
import axios from 'api';
import { ErrorResponseHandlerV2 } from 'api/ErrorResponseHandlerV2';
import { AxiosError } from 'axios';
import { ErrorV2Resp, SuccessResponseV2 } from 'types/api';
import { PayloadProps, Props } from 'types/api/user/resetPassword';
/**
* @deprecated Use the generated `useResetPassword` hook (or `resetPassword` fetcher) from
* `api/generated/services/users` instead. This hand-written client targets the
* same endpoint and will be removed once call sites migrate.
*
* Part of https://github.com/SigNoz/engineering-pod/issues/5289, add a comment or update when removing this method.
*/
const resetPassword = async (
props: Props,
): Promise<SuccessResponseV2<PayloadProps>> => {
try {
const response = await axios.post<PayloadProps>(`/resetPassword`, {
...props,
});
return {
httpStatusCode: response.status,
data: response.data,
};
} catch (error) {
ErrorResponseHandlerV2(error as AxiosError<ErrorV2Resp>);
}
};
export default resetPassword;

View File

@@ -5,9 +5,10 @@ import {
useCreateResetPasswordToken,
useDeleteUser,
useGetResetPasswordToken,
useCreateUserRole,
useDeleteUserRole,
useGetRolesByUserID,
useGetUser,
useRemoveUserRoleByUserIDAndRoleID,
useSetRoleByUserID,
useUpdateMyUserV2,
useUpdateUser,
} from 'api/generated/services/users';
@@ -24,14 +25,15 @@ import EditMemberDrawer, { EditMemberDrawerProps } from '../EditMemberDrawer';
jest.mock('api/generated/services/users', () => ({
useDeleteUser: jest.fn(),
useGetUser: jest.fn(),
useDeleteUserRole: jest.fn(),
useGetRolesByUserID: jest.fn(),
useRemoveUserRoleByUserIDAndRoleID: jest.fn(),
useUpdateUser: jest.fn(),
useUpdateMyUserV2: jest.fn(),
useCreateUserRole: jest.fn(),
useSetRoleByUserID: jest.fn(),
useGetResetPasswordToken: jest.fn(),
useCreateResetPasswordToken: jest.fn(),
getGetUserQueryKey: ({ id }: { id: string }): string[] => [
`/api/v2/users/${id}`,
getGetRolesByUserIDQueryKey: ({ id }: { id: string }): string[] => [
`/api/v2/users/${id}/roles`,
],
}));
@@ -192,7 +194,11 @@ describe('EditMemberDrawer', () => {
isLoading: false,
refetch: jest.fn(),
});
(useDeleteUserRole as jest.Mock).mockReturnValue({
(useGetRolesByUserID as jest.Mock).mockReturnValue({
data: { data: [managedRoles[0]] },
isLoading: false,
});
(useRemoveUserRoleByUserIDAndRoleID as jest.Mock).mockReturnValue({
mutateAsync: mockRemoveMutateAsync.mockResolvedValue({}),
isLoading: false,
});
@@ -204,7 +210,7 @@ describe('EditMemberDrawer', () => {
mutateAsync: jest.fn().mockResolvedValue({}),
isLoading: false,
});
(useCreateUserRole as jest.Mock).mockReturnValue({
(useSetRoleByUserID as jest.Mock).mockReturnValue({
mutateAsync: jest.fn().mockResolvedValue({}),
isLoading: false,
});
@@ -306,12 +312,12 @@ describe('EditMemberDrawer', () => {
expect(onClose).not.toHaveBeenCalled();
});
it('adding a new role creates a user role without removing existing ones', async () => {
it('adding a new role calls setRole without removing existing ones', async () => {
const onComplete = jest.fn();
const user = userEvent.setup({ pointerEventsCheck: 0 });
const mockSet = jest.fn().mockResolvedValue({});
(useCreateUserRole as jest.Mock).mockReturnValue({
(useSetRoleByUserID as jest.Mock).mockReturnValue({
mutateAsync: mockSet,
isLoading: false,
});
@@ -328,14 +334,15 @@ describe('EditMemberDrawer', () => {
await waitFor(() => {
expect(mockSet).toHaveBeenCalledWith({
data: { userId: 'user-1', roleId: managedRoles[1].id },
pathParams: { id: 'user-1' },
data: { name: 'signoz-editor' },
});
expect(mockRemoveMutateAsync).not.toHaveBeenCalled();
expect(onComplete).toHaveBeenCalled();
});
});
it('deselecting a role deletes the user role by its assignment id', async () => {
it('deselecting a role calls removeRole with the role id', async () => {
const onComplete = jest.fn();
const user = userEvent.setup({ pointerEventsCheck: 0 });
@@ -354,7 +361,7 @@ describe('EditMemberDrawer', () => {
await waitFor(() => {
expect(mockRemoveMutateAsync).toHaveBeenCalledWith({
pathParams: { id: 'ur-1' },
pathParams: { id: 'user-1', roleId: managedRoles[0].id },
});
expect(onComplete).toHaveBeenCalled();
});

View File

@@ -183,14 +183,15 @@ function QuerySearch({
isProgrammaticChangeRef.current = true;
}
const changes = view.state.changes({
from: 0,
to: currentValue.length,
insert: value,
});
view.dispatch({
changes,
selection: { anchor: changes.newLength },
changes: {
from: 0,
to: currentValue.length,
insert: value,
},
selection: {
anchor: value.length,
},
});
},
[],

View File

@@ -301,66 +301,6 @@ describe('QuerySearch (Integration with Real CodeMirror)', () => {
dispatchSpy.mockRestore();
});
it('does not crash when the expression contains CRLF line breaks (issue #5869)', async () => {
const dispatchSpy = jest.spyOn(EditorView.prototype, 'dispatch');
const onChange = jest.fn() as jest.MockedFunction<(v: string) => void>;
const initialExpression = "service.name = 'frontend'";
// Filtering on a multi-line log value (CRLF) used to throw
// "RangeError: Selection points outside of document".
const crlfExpression = "body CONTAINS 'line1\r\nline2\r\nline3'";
const baseQueryData = {
...initialQueriesMap.logs.builder.queryData[0],
filter: { expression: initialExpression },
};
const { rerender } = render(
<QuerySearch
onChange={onChange}
queryData={baseQueryData}
dataSource={DataSource.LOGS}
/>,
);
await waitFor(
() => {
const editorContent = document.querySelector(
CM_EDITOR_SELECTOR,
) as HTMLElement;
expect(editorContent.textContent || '').toBe(initialExpression);
},
{ timeout: 3000 },
);
rerender(
<QuerySearch
onChange={onChange}
queryData={{ ...baseQueryData, filter: { expression: crlfExpression } }}
dataSource={DataSource.LOGS}
/>,
);
// The programmatic replace dispatched without throwing, and the selection anchor
// stayed within the CRLF-normalized document (the bug set it past the end).
await waitFor(() => {
const spec = dispatchSpy.mock.calls
.map(
(call) =>
call[0] as {
selection?: { anchor?: number };
changes?: { newLength?: number };
},
)
.find((s) => s?.selection?.anchor != null && s?.changes?.newLength != null);
expect(spec).toBeDefined();
expect(spec?.selection?.anchor).toBeLessThanOrEqual(
spec?.changes?.newLength as number,
);
});
dispatchSpy.mockRestore();
});
it('fetches key suggestions for metrics even without aggregateAttribute.key when showFilterSuggestionsWithoutMetric is true', async () => {
const mockedGetKeys = getKeySuggestions as jest.MockedFunction<
typeof getKeySuggestions

View File

@@ -22,7 +22,6 @@ import { InfraMonitoringEntity } from 'container/InfraMonitoringK8sV2/constants'
import LoadingContainer from 'container/InfraMonitoringK8sV2/LoadingContainer';
import RunQueryBtn from 'container/QueryBuilder/components/RunQueryBtn/RunQueryBtn';
import { ChevronDown, ChevronRight } from '@signozhq/icons';
import { saveRecentQueryByExpression } from 'lib/recentQueries/saveRecentQuery';
import { useQueryState } from 'nuqs';
import { DataSource } from 'types/common/queryBuilder';
import { parseAsJsonNoValidate } from 'utils/nuqsParsers';
@@ -117,7 +116,6 @@ function EntityEventsContent({
: newUserExpression || '',
);
if (validation.isValid) {
saveRecentQueryByExpression(DataSource.LOGS, newUserExpression);
querySearchOnRun(newUserExpression || '');
void logEvent(InfraMonitoringEvents.FilterApplied, {

View File

@@ -29,7 +29,6 @@ import { getOldLogsOperatorFromNew } from 'hooks/logs/useActiveLog';
import useLogDetailHandlers from 'hooks/logs/useLogDetailHandlers';
import useScrollToLog from 'hooks/logs/useScrollToLog';
import { generateFilterQuery } from 'lib/logs/generateFilterQuery';
import { saveRecentQueryByExpression } from 'lib/recentQueries/saveRecentQuery';
import { ILog } from 'types/api/logs/log';
import { DataSource } from 'types/common/queryBuilder';
import { validateQuery } from 'utils/queryValidationUtils';
@@ -133,7 +132,6 @@ function EntityLogsContent({
);
if (validation.isValid) {
saveRecentQueryByExpression(DataSource.LOGS, newUserExpression);
querySearchOnRun(newUserExpression);
void logEvent(InfraMonitoringEvents.FilterApplied, {

View File

@@ -22,7 +22,6 @@ import { InfraMonitoringEntity } from 'container/InfraMonitoringK8sV2/constants'
import RunQueryBtn from 'container/QueryBuilder/components/RunQueryBtn/RunQueryBtn';
import { PER_PAGE_OPTIONS } from 'container/TracesExplorer/ListView/configs';
import { TracesLoading } from 'container/TracesExplorer/TraceLoading/TraceLoading';
import { saveRecentQueryByExpression } from 'lib/recentQueries/saveRecentQuery';
import { useQueryState } from 'nuqs';
import { DataSource } from 'types/common/queryBuilder';
import { parseAsJsonNoValidate } from 'utils/nuqsParsers';
@@ -99,7 +98,6 @@ function EntityTracesContent({
: newUserExpression || '',
);
if (validation.isValid) {
saveRecentQueryByExpression(DataSource.TRACES, newUserExpression);
querySearchOnRun(newUserExpression || '');
void logEvent(InfraMonitoringEvents.FilterApplied, {

View File

@@ -2,7 +2,6 @@ import { useCallback } from 'react';
import QuerySearch from 'components/QueryBuilderV2/QueryV2/QuerySearch/QuerySearch';
import RunQueryBtn from 'container/QueryBuilder/components/RunQueryBtn/RunQueryBtn';
import DateTimeSelectionV2 from 'container/TopNav/DateTimeSelectionV2';
import { saveRecentQueryByExpression } from 'lib/recentQueries/saveRecentQuery';
import { DataSource } from 'types/common/queryBuilder';
import { MetricsSearchProps } from './types';
@@ -24,14 +23,12 @@ function MetricsSearch({
);
const handleStageAndRunQuery = useCallback(() => {
saveRecentQueryByExpression(DataSource.METRICS, currentQueryFilterExpression);
onChange(currentQueryFilterExpression);
onRunQuery?.();
}, [currentQueryFilterExpression, onChange, onRunQuery]);
const handleRunQuery = useCallback(
(expression: string): void => {
saveRecentQueryByExpression(DataSource.METRICS, expression);
setCurrentQueryFilterExpression(expression);
onChange(expression);
},

View File

@@ -33,7 +33,7 @@ jest.mock('hooks/useNotifications', () => ({
}),
}));
const RESET_PASSWORD_ENDPOINT = '*/api/v2/factor_password/reset';
const RESET_PASSWORD_ENDPOINT = '*/resetPassword';
const mockHistoryPush = history.push as jest.MockedFunction<
typeof history.push

View File

@@ -1,12 +1,11 @@
import { useMemo, useState } from 'react';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useLocation } from 'react-use';
import { Button } from '@signozhq/ui/button';
import { Callout } from '@signozhq/ui/callout';
import { Form, Input as AntdInput } from 'antd';
import { Typography } from '@signozhq/ui/typography';
import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
import { useResetPassword } from 'api/generated/services/users';
import resetPasswordApi from 'api/v1/factor_password/resetPassword';
import AuthError from 'components/AuthError/AuthError';
import AuthPageContainer from 'components/AuthPageContainer';
import ROUTES from 'constants/routes';
@@ -15,6 +14,7 @@ import { useNotifications } from 'hooks/useNotifications';
import history from 'lib/history';
import { ArrowRight, CircleAlert, KeyRound } from '@signozhq/icons';
import { Label } from 'pages/SignUp/styles';
import APIError from 'types/api/error';
import { FormContainer } from './styles';
@@ -26,41 +26,40 @@ function ResetPassword({ version }: ResetPasswordProps): JSX.Element {
const [confirmPasswordError, setConfirmPasswordError] =
useState<boolean>(false);
const [errorMessage, setErrorMessage] = useState<APIError | null>();
const [isValidPassword, setIsValidPassword] = useState(false);
const [loading, setLoading] = useState(false);
const { t } = useTranslation(['common']);
const { search } = useLocation();
const params = new URLSearchParams(search);
const token = params.get('token');
const { notifications } = useNotifications();
const {
mutate: resetPassword,
isLoading,
error: mutationError,
} = useResetPassword();
const errorMessage = useMemo(
() => convertToApiError(mutationError),
[mutationError],
);
const [form] = Form.useForm<FormValues>();
const handleFormSubmit = (): void => {
const { password } = form.getFieldsValue();
const handleFormSubmit: () => Promise<void> = async () => {
try {
setLoading(true);
setErrorMessage(null);
const { password } = form.getFieldsValue();
resetPassword(
{ data: { password, token: token || '' } },
{
onSuccess: (): void => {
notifications.success({
message: t('success', {
ns: 'common',
}),
});
history.push(ROUTES.LOGIN);
},
},
);
await resetPasswordApi({
password,
token: token || '',
});
notifications.success({
message: t('success', {
ns: 'common',
}),
});
history.push(ROUTES.LOGIN);
setLoading(false);
} catch (error) {
setLoading(false);
setErrorMessage(error as APIError);
}
};
const validatePassword = (): boolean => {
@@ -223,7 +222,7 @@ function ResetPassword({ version }: ResetPasswordProps): JSX.Element {
color="primary"
type="submit"
data-attr="reset-password"
disabled={!isValidPassword || isLoading}
disabled={!isValidPassword || loading}
className="reset-password-submit-button"
suffix={<ArrowRight size={16} />}
>

View File

@@ -1,22 +1,19 @@
import { useCallback, useMemo } from 'react';
import type {
AuthtypesGettableRoleDTO,
AuthtypesUserRoleDTO,
} from 'api/generated/services/sigNoz.schemas';
import { useQueryClient } from 'react-query';
import type { AuthtypesGettableRoleDTO } from 'api/generated/services/sigNoz.schemas';
import {
useCreateUserRole,
useDeleteUserRole,
useGetUser,
getGetRolesByUserIDQueryKey,
useGetRolesByUserID,
useRemoveUserRoleByUserIDAndRoleID,
useSetRoleByUserID,
} from 'api/generated/services/users';
import { retryOn429 } from 'utils/errorUtils';
const enum PromiseStatus {
Fulfilled = 'fulfilled',
Rejected = 'rejected',
}
// Stable identity so the memos below do not recompute on every render.
const EMPTY_USER_ROLES: AuthtypesUserRoleDTO[] = [];
export interface MemberRoleUpdateFailure {
roleName: string;
error: unknown;
@@ -36,30 +33,30 @@ export function useMemberRoleManager(
userId: string,
enabled: boolean,
): UseMemberRoleManagerResult {
const { data, isLoading } = useGetUser(
const queryClient = useQueryClient();
const { data, isLoading } = useGetRolesByUserID(
{ id: userId },
{ query: { enabled: !!userId && enabled } },
);
const userRoles = data?.data?.userRoles ?? EMPTY_USER_ROLES;
const currentRoles = useMemo<AuthtypesGettableRoleDTO[]>(
() => userRoles.map((userRole) => userRole.role),
[userRoles],
() => data?.data ?? [],
[data?.data],
);
// DELETE /api/v2/user_roles/{id} is keyed by the user_role join row, not the role.
const assignmentIdByRoleId = useMemo(
() => new Map(userRoles.map((userRole) => [userRole.roleId, userRole.id])),
[userRoles],
);
const { mutateAsync: setRole } = useSetRoleByUserID({
mutation: { retry: retryOn429 },
});
const { mutateAsync: removeRole } = useRemoveUserRoleByUserIDAndRoleID({
mutation: { retry: retryOn429 },
});
const { mutateAsync: createUserRole } = useCreateUserRole({
mutation: { retry: retryOn429 },
});
const { mutateAsync: deleteUserRole } = useDeleteUserRole({
mutation: { retry: retryOn429 },
});
const invalidateRoles = useCallback(
() =>
queryClient.invalidateQueries(getGetRolesByUserIDQueryKey({ id: userId })),
[userId, queryClient],
);
const applyDiff = useCallback(
async (
@@ -83,33 +80,30 @@ export function useMemberRoleManager(
const allOperations = [
...addedRoles.map((role) => ({
role,
run: (): ReturnType<typeof createUserRole> =>
createUserRole({ data: { userId, roleId: role.id ?? '' } }),
run: (): ReturnType<typeof setRole> =>
setRole({
pathParams: { id: userId },
data: { name: role.name ?? '' },
}),
})),
...removedRoles.map((role) => ({
role,
run: (): ReturnType<typeof removeRole> =>
removeRole({ pathParams: { id: userId, roleId: role.id ?? '' } }),
})),
...removedRoles
.map((role) => ({
role,
assignmentId: assignmentIdByRoleId.get(role.id ?? ''),
}))
.filter(
(
entry,
): entry is {
role: AuthtypesGettableRoleDTO;
assignmentId: string;
} => !!entry.assignmentId,
)
.map(({ role, assignmentId }) => ({
role,
run: (): ReturnType<typeof deleteUserRole> =>
deleteUserRole({ pathParams: { id: assignmentId } }),
})),
];
const results = await Promise.allSettled(
allOperations.map((op) => op.run()),
);
const successCount = results.filter(
(r) => r.status === PromiseStatus.Fulfilled,
).length;
if (successCount > 0) {
await invalidateRoles();
}
const failures: MemberRoleUpdateFailure[] = [];
results.forEach((result, index) => {
if (result.status === PromiseStatus.Rejected) {
@@ -119,6 +113,7 @@ export function useMemberRoleManager(
error: result.reason,
onRetry: async (): Promise<void> => {
await run();
await invalidateRoles();
},
});
}
@@ -126,7 +121,7 @@ export function useMemberRoleManager(
return failures;
},
[userId, currentRoles, assignmentIdByRoleId, createUserRole, deleteUserRole],
[userId, currentRoles, setRole, removeRole, invalidateRoles],
);
return { currentRoles, isLoading, applyDiff };

View File

@@ -19,30 +19,6 @@ type CompositeWithBuilder = {
builder?: { queryData?: IBuilderQuery[] };
};
export function saveRecentQueryByExpression(
dataSource: IBuilderQuery['dataSource'],
expression: string | null | undefined,
source = '',
): void {
const trimmed = expression?.trim();
if (!trimmed) {
return;
}
const validation = validateQuery(trimmed);
if (!validation.isValid) {
return;
}
const signal = toSignal(dataSource);
if (!signal) {
return;
}
store.save({
signal,
source,
filter: { expression: trimmed },
});
}
// Persists each builder query in the composite as a recent entry. Call this
// only from explicit user-driven Run triggers — reacting to stagedQuery or any
// other derived state pollutes recents with navigation/refresh/go-to traffic.
@@ -55,10 +31,22 @@ export function saveRecentQuery(
}
queryData.forEach((q) => {
saveRecentQueryByExpression(
q.dataSource,
q.filter?.expression,
q.source ?? '',
);
const expression = q.filter?.expression?.trim();
if (!expression) {
return;
}
const validation = validateQuery(expression);
if (!validation.isValid) {
return;
}
const signal = toSignal(q.dataSource);
if (!signal) {
return;
}
store.save({
signal,
source: q.source ?? '',
filter: q.filter ?? { expression: '' },
});
});
}

View File

@@ -0,0 +1,9 @@
export interface Props {
token: string;
password: string;
}
export interface PayloadProps {
data: string;
status: string;
}

View File

@@ -3,16 +3,17 @@ package flagger
import "github.com/SigNoz/signoz/pkg/types/featuretypes"
var (
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
FeatureEnableAIObservability = featuretypes.MustNewName("enable_ai_observability")
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
FeatureEnableAIObservability = featuretypes.MustNewName("enable_ai_observability")
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
FeatureUsePrometheusClickhouseV2 = featuretypes.MustNewName("use_prometheus_clickhouse_v2")
FeatureResolveSemconvFamilies = featuretypes.MustNewName("resolve_semconv_families")
)
func MustNewRegistry() featuretypes.Registry {
@@ -97,6 +98,14 @@ func MustNewRegistry() featuretypes.Registry {
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
&featuretypes.Feature{
Name: FeatureResolveSemconvFamilies,
Kind: featuretypes.KindBoolean,
Stage: featuretypes.StageExperimental,
Description: "Controls whether trace queries resolve a semantic-convention name to all the spellings of its family",
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
)
if err != nil {
panic(err)

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="24px" height="24px" viewBox="0 0 24 24"><defs><style>.cls-1{fill:#aecbfa;}.cls-1,.cls-2,.cls-3{fill-rule:evenodd;}.cls-2{fill:#669df6;}.cls-3{fill:#4285f4;}</style></defs><title>Icon_24px_SQL_Color</title><g data-name="Product Icons"><g ><polygon class="cls-1" points="4.67 10.44 4.67 13.45 12 17.35 12 14.34 4.67 10.44"/><polygon class="cls-1" points="4.67 15.09 4.67 18.1 12 22 12 18.99 4.67 15.09"/><polygon class="cls-2" points="12 17.35 19.33 13.45 19.33 10.44 12 14.34 12 17.35"/><polygon class="cls-2" points="12 22 19.33 18.1 19.33 15.09 12 18.99 12 22"/><polygon class="cls-3" points="19.33 8.91 19.33 5.9 12 2 12 5.01 19.33 8.91"/><polygon class="cls-2" points="12 2 4.67 5.9 4.67 8.91 12 5.01 12 2"/><polygon class="cls-1" points="4.67 5.87 4.67 8.89 12 12.79 12 9.77 4.67 5.87"/><polygon class="cls-2" points="12 12.79 19.33 8.89 19.33 5.87 12 9.77 12 12.79"/></g></g></svg>

Before

Width:  |  Height:  |  Size: 933 B

View File

@@ -1,136 +0,0 @@
{
"id": "cloudsql_mysql",
"title": "GCP Cloud SQL for MySQL",
"icon": "file://icon.svg",
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
"logs": true
},
"dataCollected": {
"metrics": [
{
"name": "cloudsql.googleapis.com/database/up",
"unit": "Count",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/instance_state",
"unit": "Count",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/replication/replica_lag",
"unit": "Seconds",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/cpu/utilization",
"unit": "Percent",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/memory/utilization",
"unit": "Percent",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/disk/utilization",
"unit": "Percent",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/network/connections",
"unit": "Count",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/queries",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/dml_operations_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/threads",
"unit": "Count",
"type": "Gauge",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_reads_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_read_requests_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/slow_queries_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/aborted_connects_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/innodb/deadlocks_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/mysql/innodb/row_lock_waits_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/disk/read_ops_count",
"unit": "Count",
"type": "Sum",
"description": ""
},
{
"name": "cloudsql.googleapis.com/database/disk/write_ops_count",
"unit": "Count",
"type": "Sum",
"description": ""
}
],
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
},
"assets": {
"dashboards": [
{
"id": "overview",
"title": "GCP Cloud SQL for MySQL Overview",
"description": "Overview of GCP Cloud SQL for MySQL metrics",
"definition": "file://assets/dashboards/overview.json"
}
]
}
}

View File

@@ -1,3 +0,0 @@
### Monitor GCP Cloud SQL for MySQL with SigNoz
Collect key GCP Cloud SQL for MySQL metrics and view them with an out of the box dashboard.

View File

@@ -784,57 +784,40 @@
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/CompositeQuery",
"kind": "signoz/BuilderQuery",
"spec": {
"queries": [
"name": "A",
"signal": "metrics",
"source": "",
"aggregations": [
{
"type": "builder_query",
"spec": {
"name": "A",
"stepInterval": 0,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "cloudsql.googleapis.com/database/cpu/utilization",
"temporality": "",
"timeAggregation": "max",
"spaceAggregation": "max",
"reduceTo": ""
}
],
"disabled": false,
"filter": {
"expression": "project_id = $project_id AND database_id in $database_id AND gcp.resource_type = 'cloudsql_database' "
},
"groupBy": [
{
"name": "database_id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": null,
"selectFields": null,
"secondaryAggregations": null,
"functions": null,
"legend": "{{database_id}}"
}
},
{
"type": "builder_formula",
"spec": {
"name": "F1",
"expression": "100 * A",
"disabled": false,
"order": null,
"functions": null,
"legend": "{{database_id}}"
}
"metricName": "cloudsql.googleapis.com/database/cpu/utilization",
"temporality": "",
"timeAggregation": "max",
"spaceAggregation": "max",
"reduceTo": "avg"
}
]
],
"disabled": false,
"filter": {
"expression": "project_id = $project_id AND database_id in $database_id AND gcp.resource_type = 'cloudsql_database' "
},
"groupBy": [
{
"name": "database_id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{database_id}}"
}
}
}
@@ -1434,4 +1417,4 @@
"refreshInterval": "",
"links": []
}
}
}

View File

@@ -245,13 +245,11 @@ func (module *module) PatchV2(ctx context.Context, orgID valuer.UUID, id valuer.
}
func (module *module) DeleteV2(ctx context.Context, orgID valuer.UUID, id valuer.UUID) error {
// Read the storable, not the decoded v2 dashboard: deleting must work even
// when the stored data is corrupt or never migrated off the v1 schema.
storable, err := module.store.Get(ctx, orgID, id)
existing, err := module.GetV2(ctx, orgID, id)
if err != nil {
return err
}
if err := storable.ErrIfNotDeletable(); err != nil {
if err := existing.ErrIfNotDeletable(); err != nil {
return err
}

View File

@@ -40,7 +40,10 @@ func (c *conditionBuilder) ConditionFor(
return nil, nil, err
}
keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys))
// Rule state history fields have no family support, so every logical field
// is single-member and flattens losslessly to its physical key.
resolved, warning := querybuilder.ResolveLogicalFields(key, querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
keys := querybuilder.SingleKeys(resolved)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)

View File

@@ -6,6 +6,7 @@ import (
"strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -51,7 +52,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
return "", err
}
if col.Name == "labels" && key.Name != "labels" {
return fmt.Sprintf("JSONExtractString(labels, '%s')", strings.ReplaceAll(key.Name, "'", "\\'")), nil
return fmt.Sprintf("JSONExtractString(labels, %s)", querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return col.Name, nil
}
@@ -64,6 +65,23 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
return []*schema.Column{col}, nil
}
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
// Real columns always exist; labels are checked for key membership.
func (m *fieldMapper) ExistsFor(ctx context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
col, err := m.getColumn(ctx, key)
if err != nil {
return "", err
}
if col.Name != "labels" || key.Name == "labels" {
return "true", nil
}
pred := fmt.Sprintf("has(JSONExtractKeys(labels), %s)", querybuilder.ClickHouseStringLiteral(key.Name))
if exists {
return pred, nil
}
return "not " + pred, nil
}
func (m *fieldMapper) ColumnExpressionFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, field *telemetrytypes.TelemetryFieldKey, _ telemetrytypes.FieldDataType, _ map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) {
colName, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, field)
if err != nil {

View File

@@ -0,0 +1,17 @@
package querybuilder
import "strings"
// ClickHouseStringLiteral quotes a value for a ClickHouse string literal.
func ClickHouseStringLiteral(value string) string {
escaped := strings.ReplaceAll(value, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, `'`, `\'`)
return "'" + escaped + "'"
}
// ClickHouseIdentifier quotes a value for a ClickHouse identifier.
func ClickHouseIdentifier(value string) string {
escaped := strings.ReplaceAll(value, `\`, `\\`)
escaped = strings.ReplaceAll(escaped, "`", "\\`")
return "`" + escaped + "`"
}

View File

@@ -0,0 +1,17 @@
package querybuilder
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestClickHouseQuoting(t *testing.T) {
t.Run("string literal", func(t *testing.T) {
assert.Equal(t, `'name\'\\); SELECT 1 --'`, ClickHouseStringLiteral(`name'\); SELECT 1 --`))
})
t.Run("identifier", func(t *testing.T) {
assert.Equal(t, "`name\\`\\\\); SELECT 1 --`", ClickHouseIdentifier("name`\\); SELECT 1 --"))
})
}

View File

@@ -0,0 +1,64 @@
package querybuilder
import (
"context"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
// semconvFamiliesEnabled evaluates the resolve_semconv_families flag for the
// org. A nil flagger means off, so a caller without family support stays
// literal by default.
func semconvFamiliesEnabled(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger) bool {
if fl == nil {
return false
}
return fl.BooleanOrEmpty(ctx, flagger.FeatureResolveSemconvFamilies, featuretypes.NewFlaggerEvaluationContext(orgID))
}
// ExpandKeySelectorsForFamilies adds selectors for the other members of each
// semantic-convention family that a selector names. The metadata fetched for
// a query then contains each spelling that MatchingLogicalFields can group.
// This function is the prefetch of the resolution layer: statement builders
// call it after they derive the selectors, and the metadata store stays
// family-blind (autocomplete responses keep the literal spelling that the
// user typed). It does nothing when the resolve_semconv_families flag is off
// for the org. Only trace selectors expand today, because that matches the
// family support. Fuzzy (search-style) selectors never expand.
func ExpandKeySelectorsForFamilies(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, selectors []*telemetrytypes.FieldKeySelector) []*telemetrytypes.FieldKeySelector {
if !semconvFamiliesEnabled(ctx, orgID, fl) {
return selectors
}
out := selectors
seen := make(map[string]bool, len(selectors))
for _, selector := range selectors {
seen[selector.Name] = true
}
for _, selector := range selectors {
if selector.Signal != telemetrytypes.SignalTraces ||
selector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeFuzzy {
continue
}
members := semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: selector.Name,
Signal: selector.Signal,
FieldContext: selector.FieldContext,
})
for _, member := range members {
if seen[member] {
continue
}
seen[member] = true
expanded := *selector
expanded.Name = member
out = append(out, &expanded)
}
}
return out
}

View File

@@ -21,24 +21,25 @@ const (
hasTokenFunctionDocURL = "https://signoz.io/docs/userguide/functions-reference/#hastoken-function"
)
// ResolveKeys picks which matching field keys a filter term builds conditions for.
// With 0 or 1 match it returns the input unchanged and no warning. When a name is
// ambiguous it returns a warning; a resource+attribute mix defaults to the resource
// keys (the common intent), noted in the warning.
func ResolveKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeysForName []*telemetrytypes.TelemetryFieldKey) ([]*telemetrytypes.TelemetryFieldKey, string) {
if len(fieldKeysForName) <= 1 {
return fieldKeysForName, ""
// ResolveLogicalFields picks which logical fields a filter term builds conditions
// for. With 0 or 1 field it returns the input unchanged and no warning. When a
// name is ambiguous (several logical fields — a family is one field and never
// ambiguous with itself) it returns a warning; a resource+attribute mix defaults
// to the resource fields (the common intent), noted in the warning.
func ResolveLogicalFields(field *telemetrytypes.TelemetryFieldKey, logicalFields []*telemetrytypes.LogicalField) ([]*telemetrytypes.LogicalField, string) {
if len(logicalFields) <= 1 {
return logicalFields, ""
}
warning := fmt.Sprintf(
"Key `%s` is ambiguous, found %d different combinations of field context / data type: %v.",
field.Name,
len(fieldKeysForName),
fieldKeysForName,
len(logicalFields),
logicalFields,
)
hasResource, hasAttribute := false, false
for _, item := range fieldKeysForName {
for _, item := range logicalFields {
switch item.FieldContext {
case telemetrytypes.FieldContextResource:
hasResource = true
@@ -49,18 +50,40 @@ func ResolveKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeysForName []*te
// when there is both resource and attribute context, default to resource only
if hasResource && hasAttribute {
filteredKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(fieldKeysForName))
for _, item := range fieldKeysForName {
filtered := make([]*telemetrytypes.LogicalField, 0, len(logicalFields))
for _, item := range logicalFields {
if item.FieldContext == telemetrytypes.FieldContextResource {
filteredKeys = append(filteredKeys, item)
filtered = append(filtered, item)
}
}
fieldKeysForName = filteredKeys
logicalFields = filtered
warning += " " + "Using `resource` context by default. To query attributes explicitly, " +
fmt.Sprintf("use the fully qualified name (e.g., 'attribute.%s')", field.Name)
}
return fieldKeysForName, warning
return logicalFields, warning
}
// WrapAsLogicalFields wraps physical keys (candidate or synthesized) as
// single-member logical fields addressed by the requested spelling.
func WrapAsLogicalFields(requestedName string, keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.LogicalField {
fields := make([]*telemetrytypes.LogicalField, 0, len(keys))
for _, key := range keys {
fields = append(fields, telemetrytypes.SingleLogicalField(requestedName, key))
}
return fields
}
// SingleKeys flattens logical fields to their single members. It is the
// adapter for signals whose fields are single-member by construction (every
// signal without family support); their condition builders keep compiling per
// physical key.
func SingleKeys(fields []*telemetrytypes.LogicalField) []*telemetrytypes.TelemetryFieldKey {
keys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(fields))
for _, field := range fields {
keys = append(keys, field.Single())
}
return keys
}
// NewKeyNotFoundError builds the error a condition builder returns when a filter term

View File

@@ -0,0 +1,94 @@
package querybuilder
import (
"context"
"fmt"
"strings"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
// The two functions below are the only place family expressions are built.
// They compose exclusively from the mapper's per-key primitives (FieldFor,
// ExistsFor), so every member honors its own storage: materialized columns,
// evolutions, and JSON plans ride the member keys, and a signal supports
// families the moment its primitives are correct.
// LogicalValueExpr returns the value expression for a resolved logical field:
// the member's own expression for a single-member field, and a current-first
// merge across the members' expressions for a family.
func LogicalValueExpr(
ctx context.Context,
orgID valuer.UUID,
tsStart, tsEnd uint64,
fm qbtypes.FieldMapper,
logical *telemetrytypes.LogicalField,
) (string, error) {
if !logical.IsFamily() {
return fm.FieldFor(ctx, orgID, tsStart, tsEnd, logical.Single())
}
memberExprs := make([]string, 0, len(logical.Members))
for _, member := range logical.Members {
expr, err := fm.FieldFor(ctx, orgID, tsStart, tsEnd, member)
if err != nil {
return "", err
}
memberExprs = append(memberExprs, expr)
}
if logical.FieldDataType == telemetrytypes.FieldDataTypeString {
// The trailing '' keeps single-key semantics for rows without any
// member: string maps read '' for an absent key, and negative
// operators must keep including such rows (see AddDefaultExistsFilter).
values := make([]string, 0, len(memberExprs))
for _, expr := range memberExprs {
values = append(values, fmt.Sprintf("NULLIF(%s, '')", expr))
}
return "COALESCE(" + strings.Join(values, ", ") + ", '')", nil
}
// Numeric and boolean maps return zero for an absent key. If a family of
// either type is enabled, this tail must become zero too.
branches := make([]string, 0, len(logical.Members)*2)
for i, member := range logical.Members {
guard, err := fm.ExistsFor(ctx, orgID, tsStart, tsEnd, member, true)
if err != nil {
return "", err
}
branches = append(branches, guard, memberExprs[i])
}
return "multiIf(" + strings.Join(branches, ", ") + ", NULL)", nil
}
// LogicalExistsExpr returns the existence predicate for a resolved logical
// field: the member's own predicate for a single-member field, presence of
// any member for a family.
func LogicalExistsExpr(
ctx context.Context,
orgID valuer.UUID,
tsStart, tsEnd uint64,
fm qbtypes.FieldMapper,
logical *telemetrytypes.LogicalField,
exists bool,
) (string, error) {
if !logical.IsFamily() {
return fm.ExistsFor(ctx, orgID, tsStart, tsEnd, logical.Single(), exists)
}
guards := make([]string, 0, len(logical.Members))
for _, member := range logical.Members {
guard, err := fm.ExistsFor(ctx, orgID, tsStart, tsEnd, member, true)
if err != nil {
return "", err
}
guards = append(guards, guard)
}
combined := "(" + strings.Join(guards, " OR ") + ")"
if exists {
return combined, nil
}
return "NOT " + combined, nil
}

View File

@@ -0,0 +1,95 @@
package querybuilder
import (
"context"
"testing"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// stubFieldMapper provides just the two per-key primitives the shared
// composition builds on; the remaining FieldMapper methods are unused here.
type stubFieldMapper struct{}
func (stubFieldMapper) FieldFor(_ context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
return "value(" + key.Name + ")", nil
}
func (stubFieldMapper) ExistsFor(_ context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
if exists {
return "has(" + key.Name + ")", nil
}
return "NOT has(" + key.Name + ")", nil
}
func (stubFieldMapper) ColumnFor(context.Context, valuer.UUID, uint64, uint64, *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
return nil, qbtypes.ErrColumnNotFound
}
func (stubFieldMapper) ColumnExpressionFor(context.Context, valuer.UUID, uint64, uint64, *telemetrytypes.TelemetryFieldKey, telemetrytypes.FieldDataType, map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) {
return "", qbtypes.ErrColumnNotFound
}
func (stubFieldMapper) CandidateKeys(context.Context, valuer.UUID, *telemetrytypes.TelemetryFieldKey, any, map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
return nil
}
func stringFamily(names ...string) *telemetrytypes.LogicalField {
members := make([]*telemetrytypes.TelemetryFieldKey, 0, len(names))
for _, name := range names {
members = append(members, &telemetrytypes.TelemetryFieldKey{Name: name, FieldDataType: telemetrytypes.FieldDataTypeString})
}
return &telemetrytypes.LogicalField{Name: names[0], FieldDataType: telemetrytypes.FieldDataTypeString, Members: members}
}
func TestLogicalValueExprSingleMemberDelegatesToFieldFor(t *testing.T) {
logical := telemetrytypes.SingleLogicalField("a", &telemetrytypes.TelemetryFieldKey{Name: "a"})
expr, err := LogicalValueExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, logical)
require.NoError(t, err)
assert.Equal(t, "value(a)", expr)
}
func TestLogicalValueExprStringFamilyMergesCurrentFirst(t *testing.T) {
expr, err := LogicalValueExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, stringFamily("current", "old"))
require.NoError(t, err)
// The trailing '' preserves keyless-row semantics for negative operators.
assert.Equal(t, "COALESCE(NULLIF(value(current), ''), NULLIF(value(old), ''), '')", expr)
}
func TestLogicalValueExprNumericFamilyGuardsEveryMember(t *testing.T) {
logical := &telemetrytypes.LogicalField{
Name: "current",
FieldDataType: telemetrytypes.FieldDataTypeNumber,
Members: []*telemetrytypes.TelemetryFieldKey{
{Name: "current", FieldDataType: telemetrytypes.FieldDataTypeNumber},
{Name: "old", FieldDataType: telemetrytypes.FieldDataTypeNumber},
},
}
expr, err := LogicalValueExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, logical)
require.NoError(t, err)
assert.Equal(t, "multiIf(has(current), value(current), has(old), value(old), NULL)", expr)
}
func TestLogicalExistsExprSingleMemberDelegatesToExistsFor(t *testing.T) {
logical := telemetrytypes.SingleLogicalField("a", &telemetrytypes.TelemetryFieldKey{Name: "a"})
expr, err := LogicalExistsExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, logical, false)
require.NoError(t, err)
assert.Equal(t, "NOT has(a)", expr)
}
func TestLogicalExistsExprFamilyIsAnyMemberPresence(t *testing.T) {
family := stringFamily("current", "old")
expr, err := LogicalExistsExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, family, true)
require.NoError(t, err)
assert.Equal(t, "(has(current) OR has(old))", expr)
expr, err = LogicalExistsExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, family, false)
require.NoError(t, err)
assert.Equal(t, "NOT (has(current) OR has(old))", expr)
}

View File

@@ -0,0 +1,239 @@
package querybuilder
import (
"context"
"testing"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// familiesOn returns a flagger with resolve_semconv_families on.
func familiesOn(t *testing.T) flagger.Flagger {
return flaggertest.WithBooleanFlags(t, map[string]bool{
flagger.FeatureResolveSemconvFamilies.String(): true,
})
}
func memberNames(logical *telemetrytypes.LogicalField) []string {
names := make([]string, 0, len(logical.Members))
for _, member := range logical.Members {
names = append(names, member.Name)
}
return names
}
// The deployment.environment(.name) family (enabled in pkg/semconv) drives the
// grouping tests below.
// With the resolve_semconv_families flag off, matches stay single-member and
// selectors stay literal, even when the metadata map has both spellings.
func TestFamiliesOffByDefault(t *testing.T) {
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, flaggertest.New(t), &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, fieldKeys)
require.Len(t, fields, 1)
assert.False(t, fields[0].IsFamily())
assert.Equal(t, []string{"deployment.environment.name"}, memberNames(fields[0]))
selectors := []*telemetrytypes.FieldKeySelector{
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
}
assert.Len(t, ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, flaggertest.New(t), selectors), 1)
}
func TestMatchingLogicalFieldsGroupsFamilyMembers(t *testing.T) {
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
for _, requested := range []string{"deployment.environment.name", "deployment.environment"} {
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{Name: requested}, fieldKeys)
require.Len(t, fields, 1, "a family is one logical field, requested via %s", requested)
logical := fields[0]
assert.Equal(t, requested, logical.Name, "response identity is the requested spelling")
assert.Equal(t, telemetrytypes.FieldContextResource, logical.FieldContext)
assert.True(t, logical.IsFamily())
assert.Equal(t, []string{"deployment.environment.name", "deployment.environment"}, memberNames(logical),
"members are current-first regardless of the requested spelling")
}
}
// Member precedence is the family's current-first order, not lookup arrival
// order: a current-name key found only under its context-prefixed spelling
// arrives in the second lookup pass yet must still sort first.
func TestMatchingLogicalFieldsOrdersMembersByFamilyRank(t *testing.T) {
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"resource.deployment.environment.name": {{
Name: "resource.deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment.name",
FieldContext: telemetrytypes.FieldContextResource,
}, fieldKeys)
require.Len(t, fields, 1)
assert.Equal(t, []string{"resource.deployment.environment.name", "deployment.environment"}, memberNames(fields[0]))
}
// Non-trace signals have no family support: the requested spelling stays
// literal, and a family member name never pulls in its siblings.
func TestMatchingLogicalFieldsKeepsLogsLiteral(t *testing.T) {
logsKey := func(name string) *telemetrytypes.TelemetryFieldKey {
return &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
}
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {logsKey("deployment.environment.name")},
"deployment.environment": {logsKey("deployment.environment")},
}
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, fieldKeys)
require.Len(t, fields, 1)
assert.False(t, fields[0].IsFamily())
assert.Equal(t, []string{"deployment.environment.name"}, memberNames(fields[0]))
}
// A family and a genuine same-name collision stack cleanly: the family stays
// one logical field, the collision adds another, and resource preference keeps
// the family as a unit.
func TestResolveLogicalFieldsKeepsFamilyThroughAmbiguity(t *testing.T) {
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {
{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
requested := &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), requested, fieldKeys)
require.Len(t, fields, 2, "resource family + attribute collision")
resolved, warning := ResolveLogicalFields(requested, fields)
assert.NotEmpty(t, warning)
require.Len(t, resolved, 1)
assert.Equal(t, telemetrytypes.FieldContextResource, resolved[0].FieldContext)
assert.Equal(t, []string{"deployment.environment.name", "deployment.environment"}, memberNames(resolved[0]))
}
// Members of a family with different data types never merge: the identity
// (signal, context, data type) separates them into distinct logical fields.
func TestMatchingLogicalFieldsNeverMergesAcrossDataTypes(t *testing.T) {
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
}},
}
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, fieldKeys)
require.Len(t, fields, 2)
for _, logical := range fields {
assert.False(t, logical.IsFamily())
}
}
func TestExpandKeySelectorsForFamilies(t *testing.T) {
selectors := []*telemetrytypes.FieldKeySelector{
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
{Name: "service.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalLogs, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
}
expanded := ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, familiesOn(t), selectors)
names := make([]string, 0, len(expanded))
for _, selector := range expanded {
names = append(names, selector.Name)
}
assert.Equal(t, []string{
"deployment.environment.name",
"service.name",
"deployment.environment.name",
"deployment.environment",
}, names, "one sibling selector for the trace family member; logs and non-family names untouched")
sibling := expanded[len(expanded)-1]
assert.Equal(t, telemetrytypes.SignalTraces, sibling.Signal)
assert.Equal(t, telemetrytypes.FieldSelectorMatchTypeExact, sibling.SelectorMatchType)
}
func TestExpandKeySelectorsForFamiliesDeduplicatesAndSkipsFuzzy(t *testing.T) {
both := []*telemetrytypes.FieldKeySelector{
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
{Name: "deployment.environment", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
}
assert.Len(t, ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, familiesOn(t), both), 2, "both spellings already referenced")
fuzzy := []*telemetrytypes.FieldKeySelector{
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy},
}
assert.Len(t, ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, familiesOn(t), fuzzy), 1, "fuzzy (search-style) selectors never expand")
}

View File

@@ -9,7 +9,9 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/semconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -27,6 +29,7 @@ const stringMatchingOperatorDocURL = "https://signoz.io/docs/userguide/operators
type filterExpressionVisitor struct {
context context.Context
orgID valuer.UUID
fl flagger.Flagger
fieldMapper qbtypes.FieldMapper
conditionBuilder qbtypes.ConditionBuilder
warnings []string
@@ -48,8 +51,11 @@ type filterExpressionVisitor struct {
}
type FilterExprVisitorOpts struct {
Context context.Context
OrgID valuer.UUID
Context context.Context
OrgID valuer.UUID
// Flagger evaluates the resolve_semconv_families flag during resolution.
// A nil Flagger keeps resolution literal.
Flagger flagger.Flagger
Logger *slog.Logger
FieldMapper qbtypes.FieldMapper
ConditionBuilder qbtypes.ConditionBuilder
@@ -68,6 +74,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
return &filterExpressionVisitor{
context: opts.Context,
orgID: opts.OrgID,
fl: opts.Flagger,
fieldMapper: opts.FieldMapper,
conditionBuilder: opts.ConditionBuilder,
fieldKeys: opts.FieldKeys,
@@ -360,7 +367,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
return ErrorConditionLiteral
}
}
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.TelemetryFieldKey{v.fullTextColumn}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText))
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.LogicalField{telemetrytypes.SingleLogicalField(v.fullTextColumn.Name, v.fullTextColumn)}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText))
if !ok {
return ErrorConditionLiteral
}
@@ -379,7 +386,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
// VisitComparison handles all comparison operators.
func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := v.Visit(ctx.Key()).(*telemetrytypes.TelemetryFieldKey)
matching := MatchingFieldKeys(key, v.fieldKeys)
matching := MatchingLogicalFields(v.context, v.orgID, v.fl, key, v.fieldKeys)
// Handle EXISTS specially
if ctx.EXISTS() != nil {
@@ -675,7 +682,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
v.errors = append(v.errors, "full text search is not supported")
return ErrorConditionLiteral
}
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.TelemetryFieldKey{v.fullTextColumn}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text))
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.LogicalField{telemetrytypes.SingleLogicalField(v.fullTextColumn.Name, v.fullTextColumn)}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text))
if !ok {
return ErrorConditionLiteral
}
@@ -730,7 +737,7 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
return ErrorConditionLiteral
}
conds, ok := v.buildConditions(key, MatchingFieldKeys(key, v.fieldKeys), operator, value)
conds, ok := v.buildConditions(key, MatchingLogicalFields(v.context, v.orgID, v.fl, key, v.fieldKeys), operator, value)
if !ok {
return ErrorConditionLiteral
}
@@ -922,7 +929,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
// buildConditions invokes the condition builder for a filter term, folding its
// warnings/errors into visitor state; returns false if an error was recorded.
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) ([]string, bool) {
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.LogicalField, op qbtypes.FilterOperator, value any) ([]string, bool) {
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder)
if err != nil {
_, _, _, _, errURL, _ := errors.Unwrapb(err)
@@ -979,30 +986,161 @@ func assignIfEmpty(s *string, value string) {
}
}
// MatchingFieldKeys returns the field keys from the map that match the given key,
// honoring any context/data type the user specified.
func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
fieldKeysForName := []*telemetrytypes.TelemetryFieldKey{}
// match by name; keep items whose context and data type match (unspecified matches any)
for _, item := range fieldKeys[field.Name] {
if (field.FieldContext == telemetrytypes.FieldContextUnspecified || field.FieldContext == item.FieldContext) &&
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || field.FieldDataType == item.FieldDataType) {
fieldKeysForName = append(fieldKeysForName, item)
}
// familyMemberNames returns the physical spellings to look up for the
// referenced key: the semantic-convention family members (current-first) when
// the resolve_semconv_families flag is on for the org and the key can resolve
// to traces, else just the requested name. Only trace field mappers understand
// families today; logs and metrics keep the requested spelling until theirs
// land.
func familyMemberNames(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, field *telemetrytypes.TelemetryFieldKey) []string {
if !semconvFamiliesEnabled(ctx, orgID, fl) {
return []string{field.Name}
}
// A context may have been split off a name that legitimately contained it (e.g.
// `attribute.key`); also look up the context-prefixed name so both readings resolve.
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
contextPrefixedFieldName := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)
for _, item := range fieldKeys[contextPrefixedFieldName] {
// Context already matched via the lookup key; only data type needs checking.
if field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || item.FieldDataType == field.FieldDataType {
fieldKeysForName = append(fieldKeysForName, item)
}
}
if field.Signal != telemetrytypes.SignalUnspecified && field.Signal != telemetrytypes.SignalTraces {
return []string{field.Name}
}
return fieldKeysForName
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: field.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: field.FieldContext,
})
}
// MatchingLogicalFields resolves the referenced key against the metadata map
// into logical fields, honoring any context/data type the user specified.
//
// Physical keys that are members of one semantic-convention family (traces
// only today) group into one logical field per (signal, context, data type)
// identity, members ordered current-first. Every other matching key becomes
// its own single-member logical field. Ambiguity is the length of the
// returned slice: one family is one element and is never ambiguous with
// itself, but the slice can hold several logical fields — including several
// family fields, one per identity, when the family exists under more than
// one context or data type. Members alias the metadata map entries; nothing
// is copied or mutated.
//
// Family grouping only happens when the resolve_semconv_families flag is on
// for the org. A nil flagger means off: every match then stays a
// single-member logical field.
func MatchingLogicalFields(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.LogicalField {
members := familyMemberNames(ctx, orgID, fl, field)
matches := collectMemberMatches(field, members, fieldKeys)
return groupIntoLogicalFields(field.Name, len(members) > 1, matches)
}
// memberMatch pairs a metadata entry with the family rank of the member name
// it matched under. The stored name of a context-prefixed match differs from
// the member name, so the rank must travel with the match.
type memberMatch struct {
key *telemetrytypes.TelemetryFieldKey
rank int
}
// matchesRequestedIdentity reports whether the entry fits the context and data
// type that the request specified; unspecified matches any. A context-prefixed
// lookup already matched the context through the lookup key itself.
func matchesRequestedIdentity(field, item *telemetrytypes.TelemetryFieldKey, contextMatched bool) bool {
if !contextMatched && field.FieldContext != telemetrytypes.FieldContextUnspecified && field.FieldContext != item.FieldContext {
return false
}
if field.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && field.FieldDataType != item.FieldDataType {
return false
}
return true
}
// inFamilyScope reports whether a match found under a sibling member name is
// legitimate: the entry must be trace metadata, and the member must be in the
// family of the requested name for the entry's context. A member lookup can
// otherwise find a same-named field in a scope where the family does not
// apply.
func inFamilyScope(field, item *telemetrytypes.TelemetryFieldKey, memberName string) bool {
if item.Signal != telemetrytypes.SignalTraces {
return false
}
return slices.Contains(semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: field.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: item.FieldContext,
}), memberName)
}
// collectMemberMatches finds the metadata entries for every member spelling:
// first under the member names, then under their context-prefixed spellings
// (a context can be a legitimate part of a stored name, e.g. `attribute.key`).
func collectMemberMatches(field *telemetrytypes.TelemetryFieldKey, members []string, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []memberMatch {
matches := make([]memberMatch, 0)
collect := func(lookupName string, rank int, memberName string, contextMatched bool) {
for _, item := range fieldKeys[lookupName] {
if !matchesRequestedIdentity(field, item, contextMatched) {
continue
}
if memberName != field.Name && !inFamilyScope(field, item, memberName) {
continue
}
matches = append(matches, memberMatch{key: item, rank: rank})
}
}
for rank, member := range members {
collect(member, rank, member, false)
}
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
for rank, member := range members {
collect(fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), member), rank, member, true)
}
}
return matches
}
// groupIntoLogicalFields turns matches into logical fields. Trace entries in
// family mode group by their (signal, context, data type) identity; every
// other entry becomes its own single-member field. Members sort by family
// rank at the end: precedence is a property of the family, not of the order
// in which the lookups found the members.
func groupIntoLogicalFields(requestedName string, familyMode bool, matches []memberMatch) []*telemetrytypes.LogicalField {
fields := make([]*telemetrytypes.LogicalField, 0, len(matches))
groups := make(map[string]*telemetrytypes.LogicalField)
ranks := make(map[*telemetrytypes.TelemetryFieldKey]int)
for _, match := range matches {
if !familyMode || match.key.Signal != telemetrytypes.SignalTraces {
fields = append(fields, telemetrytypes.SingleLogicalField(requestedName, match.key))
continue
}
identity := match.key.Signal.StringValue() + ";" + match.key.FieldContext.StringValue() + ";" + match.key.FieldDataType.StringValue()
group, ok := groups[identity]
if !ok {
group = &telemetrytypes.LogicalField{
Name: requestedName,
Signal: match.key.Signal,
FieldContext: match.key.FieldContext,
FieldDataType: match.key.FieldDataType,
}
groups[identity] = group
fields = append(fields, group)
}
if groupHasMemberNamed(group, match.key.Name) {
continue
}
ranks[match.key] = match.rank
group.Members = append(group.Members, match.key)
}
for _, logical := range fields {
slices.SortStableFunc(logical.Members, func(a, b *telemetrytypes.TelemetryFieldKey) int {
return ranks[a] - ranks[b]
})
}
return fields
}
func groupHasMemberNamed(group *telemetrytypes.LogicalField, name string) bool {
for _, member := range group.Members {
if member.Name == name {
return true
}
}
return false
}

View File

@@ -588,9 +588,11 @@ func TestVisitKey(t *testing.T) {
// VisitKey only parses; the condition builder matches, resolves ambiguity
// and decides not-found handling. Replay that here against the generic
// builder behavior (error unless the key is ignored).
matching := MatchingFieldKeys(key, tt.fieldKeys)
keys, warning := ResolveKeys(key, matching)
// builder behavior (error unless the key is ignored). The test maps carry
// no signal, so every logical field is single-member and flattens losslessly.
matching := MatchingLogicalFields(context.Background(), valuer.UUID{}, nil, key, tt.fieldKeys)
resolved, warning := ResolveLogicalFields(key, matching)
keys := SingleKeys(resolved)
var gotErrors []string
var gotMainErrURL, gotMainWrnURL string
@@ -766,7 +768,8 @@ func (b *resourceConditionBuilder) ConditionFor(
return nil, nil, nil
}
keys, warning := ResolveKeys(key, MatchingFieldKeys(key, fieldKeys))
resolved, warning := ResolveLogicalFields(key, MatchingLogicalFields(context.Background(), valuer.UUID{}, nil, key, fieldKeys))
keys := SingleKeys(resolved)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)
@@ -808,7 +811,8 @@ func (b *conditionBuilder) ConditionFor(
return []string{fmt.Sprintf("%s_cond", key.Name)}, nil, nil
}
keys, warning := ResolveKeys(key, MatchingFieldKeys(key, fieldKeys))
resolved, warning := ResolveLogicalFields(key, MatchingLogicalFields(context.Background(), valuer.UUID{}, nil, key, fieldKeys))
keys := SingleKeys(resolved)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)

View File

@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -14,12 +15,15 @@ import (
type defaultConditionBuilder struct {
fm qbtypes.FieldMapper
// fl evaluates the resolve_semconv_families flag during resolution.
// A nil flagger keeps resolution literal.
fl flagger.Flagger
}
var _ qbtypes.ConditionBuilder = (*defaultConditionBuilder)(nil)
func NewConditionBuilder(fm qbtypes.FieldMapper) *defaultConditionBuilder {
return &defaultConditionBuilder{fm: fm}
func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *defaultConditionBuilder {
return &defaultConditionBuilder{fm: fm, fl: fl}
}
func valueForIndexFilter(op qbtypes.FilterOperator, key *telemetrytypes.TelemetryFieldKey, value any) any {
@@ -44,10 +48,74 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
return fmt.Sprintf(`%%%s%%`, key.Name)
}
// The three helpers below take the members of one logical field. With a single
// member they render exactly the pre-family shapes; a family widens key/value
// index hints to any-member and presence to any-member (all-absent when negated).
func keyIndexCondition(sb *sqlbuilder.SelectBuilder, column string, members []*telemetrytypes.TelemetryFieldKey) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
conditions = append(conditions, sb.Like(column, keyIndexFilter(member)))
}
if len(conditions) == 1 {
return conditions[0]
}
return sb.Or(conditions...)
}
func valueIndexCondition(
sb *sqlbuilder.SelectBuilder,
column string,
members []*telemetrytypes.TelemetryFieldKey,
op qbtypes.FilterOperator,
value any,
caseInsensitive bool,
) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
patterns := valueForIndexFilter(op, member, value)
switch values := patterns.(type) {
case []string:
for _, pattern := range values {
conditions = append(conditions, sb.Like(column, pattern))
}
default:
if caseInsensitive {
conditions = append(conditions, sb.ILike(column, values))
} else {
conditions = append(conditions, sb.Like(column, values))
}
}
}
if len(conditions) == 1 {
return conditions[0]
}
return sb.Or(conditions...)
}
func memberPresenceCondition(sb *sqlbuilder.SelectBuilder, column string, members []*telemetrytypes.TelemetryFieldKey, exists bool) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
field := fmt.Sprintf("simpleJSONHas(%s, '%s')", column, member.Name)
if exists {
conditions = append(conditions, sb.E(field, true))
} else {
conditions = append(conditions, sb.NE(field, true))
}
}
if exists {
if len(conditions) == 1 {
return conditions[0]
}
return sb.Or(conditions...)
}
return sb.And(conditions...)
}
// SkipResourceFilter is not applicable here: the fingerprint table only stores resource attributes.
func (b *defaultConditionBuilder) ConditionFor(
ctx context.Context,
_ valuer.UUID,
orgID valuer.UUID,
startNs uint64,
endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
@@ -57,7 +125,7 @@ func (b *defaultConditionBuilder) ConditionFor(
value any,
sb *sqlbuilder.SelectBuilder,
) ([]string, []string, error) {
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
matches := querybuilder.MatchingLogicalFields(ctx, orgID, b.fl, key, fieldKeys)
// has/hasAny/hasAll/hasToken are logs-body-only functions; they never apply to the
// resource fingerprint table, so skip them (the main query still evaluates them).
@@ -65,21 +133,21 @@ func (b *defaultConditionBuilder) ConditionFor(
return nil, nil, nil
}
keys, warning := querybuilder.ResolveKeys(key, matches)
logicalFields, warning := querybuilder.ResolveLogicalFields(key, matches)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)
}
conds := make([]string, 0, len(keys))
for _, k := range keys {
// the resource fingerprint table only stores resource attributes; keys from
conds := make([]string, 0, len(logicalFields))
for _, logical := range logicalFields {
// the resource fingerprint table only stores resource attributes; fields from
// any other context contribute no condition and are omitted. An empty result
// (including an unknown key) lets the caller skip this filter entirely.
if k.FieldContext != telemetrytypes.FieldContextResource {
if logical.FieldContext != telemetrytypes.FieldContextResource {
continue
}
cond, err := b.conditionForKey(ctx, startNs, endNs, k, op, value, sb)
cond, err := b.conditionForLogicalField(ctx, orgID, startNs, endNs, logical, op, value, sb)
if err != nil {
return nil, nil, err
}
@@ -88,11 +156,12 @@ func (b *defaultConditionBuilder) ConditionFor(
return conds, warnings, nil
}
func (b *defaultConditionBuilder) conditionForKey(
func (b *defaultConditionBuilder) conditionForLogicalField(
ctx context.Context,
orgID valuer.UUID,
startNs uint64,
endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
logical *telemetrytypes.LogicalField,
op qbtypes.FilterOperator,
value any,
sb *sqlbuilder.SelectBuilder,
@@ -102,7 +171,9 @@ func (b *defaultConditionBuilder) conditionForKey(
// as we store resource values as string
formattedValue := querybuilder.FormatValueForContains(value)
columns, err := b.fm.ColumnFor(ctx, valuer.UUID{}, startNs, endNs, key)
// Every resource-context key maps to the labels column, so any member
// resolves the column for the whole field.
columns, err := b.fm.ColumnFor(ctx, orgID, startNs, endNs, logical.Single())
if err != nil {
return "", err
}
@@ -115,10 +186,12 @@ func (b *defaultConditionBuilder) conditionForKey(
// as we have not changed the resource column in the resource fingerprint table.
column := columns[0]
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
valueForIndexFilter := valueForIndexFilter(op, key, value)
members := logical.Members
isFamily := logical.IsFamily()
keyIdxFilter := keyIndexCondition(sb, column.Name, members)
singleValueIndexFilter := valueForIndexFilter(op, members[0], value)
fieldName, err := b.fm.FieldFor(ctx, valuer.UUID{}, startNs, endNs, key)
fieldName, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, b.fm, logical)
if err != nil {
return "", err
}
@@ -128,12 +201,17 @@ func (b *defaultConditionBuilder) conditionForKey(
return sb.And(
sb.E(fieldName, formattedValue),
keyIdxFilter,
sb.Like(column.Name, valueForIndexFilter),
valueIndexCondition(sb, column.Name, members, op, value, false),
), nil
case qbtypes.FilterOperatorNotEqual:
if isFamily {
// A negated value-index hint would drop rows where another member
// holds the value; the fingerprint scan is small enough without it.
return sb.NE(fieldName, formattedValue), nil
}
return sb.And(
sb.NE(fieldName, formattedValue),
sb.NotLike(column.Name, valueForIndexFilter),
sb.NotLike(column.Name, singleValueIndexFilter),
), nil
case qbtypes.FilterOperatorGreaterThan:
return sb.And(sb.GT(fieldName, formattedValue), keyIdxFilter), nil
@@ -148,7 +226,7 @@ func (b *defaultConditionBuilder) conditionForKey(
return sb.And(
sb.ILike(fieldName, formattedValue),
keyIdxFilter,
sb.ILike(column.Name, valueForIndexFilter),
valueIndexCondition(sb, column.Name, members, op, value, true),
), nil
case qbtypes.FilterOperatorNotLike, qbtypes.FilterOperatorNotILike:
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
@@ -185,13 +263,11 @@ func (b *defaultConditionBuilder) conditionForKey(
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
}
mainCondition := sb.Or(inConditions...)
valConditions := make([]string, 0, len(values))
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
for _, v := range valuesForIndexFilter {
valConditions = append(valConditions, sb.Like(column.Name, v))
}
}
mainCondition = sb.And(mainCondition, keyIdxFilter, sb.Or(valConditions...))
mainCondition = sb.And(
mainCondition,
keyIdxFilter,
valueIndexCondition(sb, column.Name, members, op, value, false),
)
return mainCondition, nil
case qbtypes.FilterOperatorNotIn:
@@ -204,8 +280,13 @@ func (b *defaultConditionBuilder) conditionForKey(
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
}
mainCondition := sb.And(notInConditions...)
if isFamily {
// A negated value-index hint would drop rows where another member
// holds the value; the fingerprint scan is small enough without it.
return mainCondition, nil
}
valConditions := make([]string, 0, len(values))
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
if valuesForIndexFilter, ok := singleValueIndexFilter.([]string); ok {
for _, v := range valuesForIndexFilter {
valConditions = append(valConditions, sb.NotLike(column.Name, v))
}
@@ -215,13 +296,11 @@ func (b *defaultConditionBuilder) conditionForKey(
case qbtypes.FilterOperatorExists:
return sb.And(
sb.E(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
memberPresenceCondition(sb, column.Name, members, true),
keyIdxFilter,
), nil
case qbtypes.FilterOperatorNotExists:
return sb.And(
sb.NE(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
), nil
return memberPresenceCondition(sb, column.Name, members, false), nil
case qbtypes.FilterOperatorRegexp:
return sb.And(
@@ -237,7 +316,7 @@ func (b *defaultConditionBuilder) conditionForKey(
return sb.And(
sb.ILike(fieldName, fmt.Sprintf(`%%%s%%`, formattedValue)),
keyIdxFilter,
sb.ILike(column.Name, valueForIndexFilter),
valueIndexCondition(sb, column.Name, members, op, value, true),
), nil
case qbtypes.FilterOperatorNotContains:
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else

View File

@@ -2,6 +2,7 @@ package resourcefilter
import (
"context"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"testing"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -201,7 +202,7 @@ func TestConditionBuilder(t *testing.T) {
}
fm := NewFieldMapper()
conditionBuilder := NewConditionBuilder(fm)
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
for _, tc := range testCases {
sb := sqlbuilder.NewSelectBuilder()

View File

@@ -0,0 +1,141 @@
package resourcefilter
import (
"context"
"testing"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/huandu/go-sqlbuilder"
"github.com/stretchr/testify/require"
)
// The tests below run against the deployment.environment(.name) family with
// the resolve_semconv_families flag on, and with both spellings present in
// the metadata map as trace resource attributes.
func TestFamilyEqualWidensIndexHintsToAnyMember(t *testing.T) {
cb := NewConditionBuilder(NewFieldMapper(), flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureResolveSemconvFamilies.String(): true}))
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE (COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') = ? AND (labels LIKE ? OR labels LIKE ?) AND (labels LIKE ? OR labels LIKE ?))", sql)
require.Equal(t, []any{"production", "%deployment.environment.name%", "%deployment.environment%", "%deployment.environment.name\":\"production%", "%deployment.environment\":\"production%"}, args)
}
func TestFamilyNotEqualDropsNegatedValueHint(t *testing.T) {
cb := NewConditionBuilder(NewFieldMapper(), flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureResolveSemconvFamilies.String(): true}))
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotEqual, "production", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') <> ?", sql)
require.Equal(t, []any{"production"}, args)
}
func TestFamilyExistsIsAnyMemberPresence(t *testing.T) {
cb := NewConditionBuilder(NewFieldMapper(), flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureResolveSemconvFamilies.String(): true}))
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorExists, nil, sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE ((simpleJSONHas(labels, 'deployment.environment.name') = ? OR simpleJSONHas(labels, 'deployment.environment') = ?) AND (labels LIKE ? OR labels LIKE ?))", sql)
require.Equal(t, []any{true, true, "%deployment.environment.name%", "%deployment.environment%"}, args)
sb = sqlbuilder.NewSelectBuilder()
conds, _, err = cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotExists, nil, sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args = sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE (simpleJSONHas(labels, 'deployment.environment.name') <> ? AND simpleJSONHas(labels, 'deployment.environment') <> ?)", sql)
require.Equal(t, []any{true, true}, args)
}
// With only one member in metadata the SQL keeps the exact pre-family shape,
// including the negated value hint on !=.
func TestSingleMemberShapesUnchanged(t *testing.T) {
cb := NewConditionBuilder(NewFieldMapper(), flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureResolveSemconvFamilies.String(): true}))
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}},
}
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotEqual, "production", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE (simpleJSONExtractString(labels, 'deployment.environment.name') <> ? AND labels NOT LIKE ?)", sql)
require.Equal(t, []any{"production", "%deployment.environment.name\":\"production%"}, args)
}

View File

@@ -71,6 +71,33 @@ func (m *defaultFieldMapper) FieldFor(
return columns[0].Name, nil
}
// ExistsFor reports key presence in the fingerprint labels JSON. Only resource
// context keys have a presence notion here; anything else is a real column and
// always present.
func (m *defaultFieldMapper) ExistsFor(
ctx context.Context,
_ valuer.UUID,
tsStart, tsEnd uint64,
key *telemetrytypes.TelemetryFieldKey,
exists bool,
) (string, error) {
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
if err != nil {
return "", err
}
if key.FieldContext != telemetrytypes.FieldContextResource {
if exists {
return "true", nil
}
return "false", nil
}
pred := fmt.Sprintf("simpleJSONHas(%s, '%s')", columns[0].Name, key.Name)
if exists {
return pred, nil
}
return "NOT " + pred, nil
}
func (m *defaultFieldMapper) ColumnExpressionFor(
ctx context.Context,
orgID valuer.UUID,

View File

@@ -47,7 +47,7 @@ func New[T any](
) *resourceFilterStatementBuilder[T] {
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter")
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
cb := NewConditionBuilder(fm, fl)
return &resourceFilterStatementBuilder[T]{
logger: set.Logger(),
dbName: dbName,
@@ -99,7 +99,7 @@ func (b *resourceFilterStatementBuilder[T]) Build(
q.Select("fingerprint")
q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
keySelectors := b.getKeySelectors(query)
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.flagger, b.getKeySelectors(query))
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
if err != nil {
return nil, err
@@ -164,6 +164,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
filterWhereClause, err := querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
Context: ctx,
OrgID: orgID,
Flagger: b.flagger,
Logger: b.logger,
FieldMapper: b.fieldMapper,
ConditionBuilder: b.conditionBuilder,

View File

@@ -38,6 +38,7 @@ type scopedTraceStatementBuilder struct {
scope TraceScope
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
fl flagger.Flagger
}
var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*scopedTraceStatementBuilder)(nil)
@@ -59,8 +60,8 @@ func NewFactory(
if err != nil {
return nil, err
}
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
return NewScopedTraceStatementBuilder(settings, metadataStore, fm, cb, scope, traceStmtBuilder, fl), nil
},
)
@@ -98,6 +99,7 @@ func NewScopedTraceStatementBuilder(
scope: scope,
traceStmtBuilder: traceStmtBuilder,
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
fl: fl,
}
}
@@ -266,7 +268,7 @@ func (b *scopedTraceStatementBuilder) fetchKeys(ctx context.Context, orgID value
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
})
}
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.fl, selectors))
return keys, err
}
@@ -442,13 +444,14 @@ func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context,
for i := range selectors {
selectors[i].Signal = telemetrytypes.SignalTraces
}
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.fl, selectors))
if err != nil {
return "", nil, "", err
}
prepared, err := querybuilder.PrepareWhereClause(expr, querybuilder.FilterExprVisitorOpts{
Context: ctx,
OrgID: orgID,
Flagger: b.fl,
Logger: b.logger,
FieldMapper: b.fm,
ConditionBuilder: b.cb,

View File

@@ -31,6 +31,7 @@ type traceQueryStatementBuilder struct {
cb qbtypes.ConditionBuilder
resourceFilterResolver *resourcefilter.ResourceFingerprintResolver[qbtypes.TraceAggregation]
aggExprRewriter qbtypes.AggExprRewriter
fl flagger.Flagger
skipResourceFingerprintEnabled bool
}
@@ -47,8 +48,8 @@ func NewFactory(
return factory.NewProviderFactory(
factory.MustNewName("traces"),
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
return NewTraceQueryStatementBuilder(
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
@@ -91,6 +92,7 @@ func NewTraceQueryStatementBuilder(
cb: conditionBuilder,
resourceFilterResolver: resourceFilterResolver,
aggExprRewriter: aggExprRewriter,
fl: flagger,
skipResourceFingerprintEnabled: skipResourceFingerprintEnable,
}
}
@@ -119,7 +121,7 @@ func (b *traceQueryStatementBuilder) Build(
// We modify SelectFields above (injecting default fields), and those default
// fields can carry keys that need evolutions, so fetch keys after that.
keySelectors := getKeySelectors(query)
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.fl, getKeySelectors(query))
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
if err != nil {
@@ -795,6 +797,7 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
Context: ctx,
OrgID: orgID,
Flagger: b.fl,
Logger: b.logger,
FieldMapper: b.fm,
ConditionBuilder: b.cb,

View File

@@ -8,6 +8,7 @@ import (
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -376,8 +377,8 @@ func TestStatementBuilder(t *testing.T) {
}
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -677,8 +678,8 @@ func TestStatementBuilderListQuery(t *testing.T) {
}
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -804,8 +805,8 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = c.keysMap
if mockMetadataStore.KeysMap == nil {
@@ -879,8 +880,8 @@ func TestStatementBuilderGroupByResourceEvolution(t *testing.T) {
}
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -1046,8 +1047,8 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
}
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -1684,8 +1685,8 @@ func newSkipResourceFingerprintBuilder(
t.Helper()
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
@@ -1714,8 +1715,8 @@ func TestStatementBuilderGroupByUnseenKey(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -1756,8 +1757,8 @@ func TestStatementBuilderAggregationUnseenKey(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
@@ -1787,3 +1788,72 @@ func TestStatementBuilderAggregationUnseenKey(t *testing.T) {
assert.Contains(t, q.Query, "attributes_number['error.type']")
assert.Contains(t, q.Query, "attributes_bool['error.type']")
}
// TestStatementBuilderSemconvFamilies builds the same family-member filter
// with the resolve_semconv_families flag on and off. On: the resource filter
// merges both spellings and widens the index hints to any member. Off: the
// query uses only the requested spelling, so users see no change.
func TestStatementBuilderSemconvFamilies(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "count()"}},
Filter: &qbtypes.Filter{
Expression: "deployment.environment.name = 'production'",
},
}
cases := []struct {
name string
flag bool
expected qbtypes.Statement
}{
{
name: "flag on merges both spellings",
flag: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') = ? AND (labels LIKE ? OR labels LIKE ?) AND (labels LIKE ? OR labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC",
Args: []any{"production", "%deployment.environment.name%", "%deployment.environment%", "%deployment.environment.name\":\"production%", "%deployment.environment\":\"production%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
{
name: "flag off keeps the literal spelling",
flag: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'deployment.environment.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC",
Args: []any{"production", "%deployment.environment.name%", "%deployment.environment.name\":\"production%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
fl := flaggertest.WithBooleanFlags(t, map[string]bool{
flagger.FeatureResolveSemconvFamilies.String(): c.flag,
})
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
statementBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
nil,
fl,
false,
100000,
)
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeScalar, query, nil)
require.NoError(t, err)
require.Equal(t, c.expected.Query, q.Query)
require.Equal(t, c.expected.Args, q.Args)
})
}
}

View File

@@ -212,7 +212,7 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
return cteName, nil
}
keySelectors := getKeySelectors(*query)
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, b.orgID, b.stmtBuilder.fl, getKeySelectors(*query))
b.stmtBuilder.logger.DebugContext(ctx, "Key selectors for query", slog.String("query_name", queryName), slog.Any("key_selectors", keySelectors))
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, b.orgID, keySelectors)
if err != nil {
@@ -265,6 +265,7 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
querybuilder.FilterExprVisitorOpts{
Context: ctx,
OrgID: b.orgID,
Flagger: b.stmtBuilder.fl,
Logger: b.stmtBuilder.logger,
FieldMapper: b.stmtBuilder.fm,
ConditionBuilder: b.stmtBuilder.cb,
@@ -442,7 +443,7 @@ func (b *traceOperatorCTEBuilder) buildFinalQuery(ctx context.Context, selectFro
}
}
keySelectors := b.getKeySelectors()
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, b.orgID, b.stmtBuilder.fl, b.getKeySelectors())
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, b.orgID, keySelectors)
if err != nil {
return nil, err

View File

@@ -20,8 +20,8 @@ func newTestTraceOperatorStatementBuilder(t *testing.T) *traceOperatorStatementB
t.Helper()
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)

View File

@@ -25,6 +25,7 @@ type traceOperatorStatementBuilder struct {
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
aggExprRewriter qbtypes.AggExprRewriter
fl flagger.Flagger
}
var _ qbtypes.TraceOperatorStatementBuilder = (*traceOperatorStatementBuilder)(nil)
@@ -41,8 +42,8 @@ func NewOperatorFactory(
return factory.NewProviderFactory(
factory.MustNewName("traceoperator"),
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.TraceOperatorStatementBuilder, error) {
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
traceStmtBuilder := NewTraceQueryStatementBuilder(
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
@@ -85,6 +86,7 @@ func NewTraceOperatorStatementBuilder(
traceStmtBuilder: traceStmtBuilder,
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
aggExprRewriter: aggExprRewriter,
fl: flagger,
}
}

View File

@@ -21,8 +21,9 @@ import (
func TestTraceTimeRangeOptimization(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
fm := tracestelemetryschema.NewFieldMapper()
cb := tracestelemetryschema.NewConditionBuilder(fm)
fl := flaggertest.New(t)
fm := tracestelemetryschema.NewFieldMapper(fl)
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
@@ -39,7 +40,6 @@ func TestTraceTimeRangeOptimization(t *testing.T) {
Signal: telemetrytypes.SignalTraces,
}}
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
statementBuilder := NewTraceQueryStatementBuilder(

View File

@@ -38,8 +38,11 @@ func (c *conditionBuilder) ConditionFor(
return nil, nil, err
}
// an unknown key simply yields no condition rather than an error.
keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys))
// an unknown key simply yields no condition rather than an error. Metadata
// fields have no family support, so every logical field is single-member
// and flattens losslessly to its physical key.
resolved, warning := querybuilder.ResolveLogicalFields(key, querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
keys := querybuilder.SingleKeys(resolved)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)

View File

@@ -58,6 +58,19 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsE
return columns, nil
}
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
func (m *fieldMapper) ExistsFor(ctx context.Context, _ valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
if err != nil {
return "", err
}
pred := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
if exists {
return pred, nil
}
return "NOT " + pred, nil
}
func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
columns, err := m.getColumn(ctx, startNs, endNs, key)
if err != nil {

View File

@@ -139,7 +139,10 @@ func (c *conditionBuilder) ConditionFor(
return nil, nil, err
}
keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys))
// Audit fields have no family support, so every logical field is
// single-member and flattens losslessly to its physical key.
resolved, warning := querybuilder.ResolveLogicalFields(key, querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
keys := querybuilder.SingleKeys(resolved)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)

View File

@@ -97,6 +97,19 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
return m.getColumn(ctx, key)
}
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
func (m *fieldMapper) ExistsFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, key)
if err != nil {
return "", err
}
columns, err := m.getColumn(ctx, key)
if err != nil {
return "", err
}
return querybuilder.ExistsExpression(columns, key, tsStart, tsEnd, fieldExpression, exists)
}
func (m *fieldMapper) ColumnExpressionFor(
ctx context.Context,
orgID valuer.UUID,

View File

@@ -452,7 +452,7 @@ func (c *conditionBuilder) ConditionFor(
value any,
sb *sqlbuilder.SelectBuilder,
) ([]string, []string, error) {
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
matches := querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys)
skipResourceFilter := options.SkipResourceFilter
// search() resolves its own (optional) scope; handle it before key resolution.
@@ -460,7 +460,10 @@ func (c *conditionBuilder) ConditionFor(
return c.conditionForSearch(ctx, orgID, key, value, sb)
}
keys, warning := querybuilder.ResolveKeys(key, matches)
// Logs fields have no family support yet, so every logical field is
// single-member and flattens losslessly to its physical key.
resolved, warning := querybuilder.ResolveLogicalFields(key, matches)
keys := querybuilder.SingleKeys(resolved)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)

View File

@@ -279,7 +279,7 @@ func (m *fieldMapper) ColumnExpressionFor(
}
var stmts []string
for _, key := range candidates {
guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, key, true)
guard, err := m.ExistsFor(ctx, orgID, tsStart, tsEnd, key, true)
if err != nil {
return "", err
}
@@ -308,7 +308,7 @@ func (m *fieldMapper) ColumnExpressionFor(
if !m.membershipGuarded(ctx, orgID, tsStart, tsEnd, candidates[0]) {
return m.FieldFor(ctx, orgID, tsStart, tsEnd, candidates[0])
}
guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, candidates[0], true)
guard, err := m.ExistsFor(ctx, orgID, tsStart, tsEnd, candidates[0], true)
if err != nil {
return "", err
}
@@ -326,7 +326,7 @@ func (m *fieldMapper) ColumnExpressionFor(
var stmts []string
for _, key := range candidates {
guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, key, true)
guard, err := m.ExistsFor(ctx, orgID, tsStart, tsEnd, key, true)
if err != nil {
return "", err
}
@@ -569,7 +569,8 @@ func (m *fieldMapper) membershipGuarded(ctx context.Context, orgID valuer.UUID,
return columnType == schema.ColumnTypeEnumMap || columnType == schema.ColumnTypeEnumJSON
}
func (m *fieldMapper) existsExpressionFor(
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
func (m *fieldMapper) ExistsFor(
ctx context.Context,
orgID valuer.UUID,
tsStart, tsEnd uint64,

View File

@@ -162,7 +162,9 @@ func (c *conditionBuilder) ConditionFor(
return nil, nil, err
}
keys := querybuilder.MatchingFieldKeys(key, fieldKeys)
// Metric labels have no family support, so every logical field is
// single-member and flattens losslessly to its physical key.
keys := querybuilder.SingleKeys(querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
var warnings []string
if len(keys) == 0 {
if _, isColumn := timeSeriesV4Columns[key.Name]; isColumn {

View File

@@ -97,6 +97,18 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsE
return m.getColumn(ctx, tsStart, tsEnd, key)
}
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
// Intrinsic fields always exist; labels are checked for key membership.
func (m *fieldMapper) ExistsFor(_ context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
if slices.Contains(IntrinsicFields, key.Name) {
return "true", nil
}
if exists {
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
}
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
}
func (m *fieldMapper) ColumnExpressionFor(
ctx context.Context,
orgID valuer.UUID,

View File

@@ -9,6 +9,7 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -19,12 +20,15 @@ import (
type conditionBuilder struct {
fm qbtypes.FieldMapper
// fl evaluates the resolve_semconv_families flag during resolution.
// A nil flagger keeps resolution literal.
fl flagger.Flagger
}
var _ qbtypes.ConditionBuilder = (*conditionBuilder)(nil)
func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
return &conditionBuilder{fm: fm}
func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionBuilder {
return &conditionBuilder{fm: fm, fl: fl}
}
func (c *conditionBuilder) conditionFor(
@@ -32,7 +36,7 @@ func (c *conditionBuilder) conditionFor(
orgID valuer.UUID,
startNs uint64,
endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
logical *telemetrytypes.LogicalField,
operator qbtypes.FilterOperator,
value any,
sb *sqlbuilder.SelectBuilder,
@@ -42,13 +46,13 @@ func (c *conditionBuilder) conditionFor(
value = querybuilder.FormatValueForContains(value)
}
fieldExpression, err := c.fm.FieldFor(ctx, orgID, startNs, endNs, key)
fieldExpression, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, c.fm, logical)
if err != nil {
return "", err
}
// TODO(srikanthccv): maybe extend this to every possible attribute
if key.Name == "duration_nano" || key.Name == "durationNano" { // QoL improvement
if logical.Name == "duration_nano" || logical.Name == "durationNano" { // QoL improvement
switch v := value.(type) {
case string:
if duration, err := time.ParseDuration(v); err == nil {
@@ -65,7 +69,9 @@ func (c *conditionBuilder) conditionFor(
}
}
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
// Coercion switches only on the data type, which every member shares, so
// the first member stands in for the field.
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(logical.Single(), value, fieldExpression, operator)
// regular operators
switch operator {
@@ -154,11 +160,7 @@ func (c *conditionBuilder) conditionFor(
// in the query builder, `exists` and `not exists` are used for
// key membership checks, so depending on the column type, the condition changes
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key)
if err != nil {
return "", err
}
pred, err := querybuilder.ExistsExpression(columns, key, startNs, endNs, fieldExpression, operator == qbtypes.FilterOperatorExists)
pred, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, c.fm, logical, operator == qbtypes.FilterOperatorExists)
if err != nil {
return "", err
}
@@ -210,10 +212,10 @@ func (c *conditionBuilder) ConditionFor(
return nil, nil, err
}
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
matches := querybuilder.MatchingLogicalFields(ctx, orgID, c.fl, key, fieldKeys)
skipResourceFilter := options.SkipResourceFilter
keys, warning := querybuilder.ResolveKeys(key, matches)
logicalFields, warning := querybuilder.ResolveLogicalFields(key, matches)
var warnings []string
if warning != "" {
warnings = append(warnings, warning)
@@ -221,10 +223,10 @@ func (c *conditionBuilder) ConditionFor(
// A bare key that names a real column filters on the column too — first. When metadata
// only knows the name under other contexts, prepend the column and keep metadata matches
// only where their type is consistent with it (a corrupt entry can't degrade the column).
if key.FieldContext == telemetrytypes.FieldContextUnspecified && len(keys) > 0 {
if key.FieldContext == telemetrytypes.FieldContextUnspecified && len(logicalFields) > 0 {
hasColumn := false
for _, k := range keys {
if k.FieldContext == telemetrytypes.FieldContextSpan {
for _, logical := range logicalFields {
if logical.FieldContext == telemetrytypes.FieldContextSpan {
hasColumn = true
break
}
@@ -232,49 +234,49 @@ func (c *conditionBuilder) ConditionFor(
if !hasColumn {
probe := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextSpan, key.FieldDataType)
if cols, colErr := c.fm.ColumnFor(ctx, orgID, startNs, endNs, probe); colErr == nil && len(cols) > 0 {
combined := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)+1)
combined = append(combined, probe)
for _, k := range keys {
if columnMatchesDataType(cols[0], k.FieldDataType) {
combined = append(combined, k)
combined := make([]*telemetrytypes.LogicalField, 0, len(logicalFields)+1)
combined = append(combined, telemetrytypes.SingleLogicalField(key.Name, probe))
for _, logical := range logicalFields {
if columnMatchesDataType(cols[0], logical.FieldDataType) {
combined = append(combined, logical)
}
}
keys = combined
logicalFields = combined
}
}
}
synthesized := false
if len(keys) == 0 {
if len(logicalFields) == 0 {
// Not in metadata. CandidateKeys resolves it: fold contexts (span/trace) get the
// metadata map so it can honor a real column, correct to a stripped-name metadata
// match, or synthesize; strict contexts pass nil and keep their synthesize path.
keys = c.fm.CandidateKeys(ctx, orgID, key, value, candidateLookupKeys(key, fieldKeys))
if len(keys) == 0 {
logicalFields = querybuilder.WrapAsLogicalFields(key.Name, c.fm.CandidateKeys(ctx, orgID, key, value, candidateLookupKeys(key, fieldKeys)))
if len(logicalFields) == 0 {
return nil, warnings, querybuilder.NewKeyNotFoundError(key.Name)
}
synthesized = true
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
}
// When a resource sub-query already covers the term, drop resource keys from the main
// When a resource sub-query already covers the term, drop resource fields from the main
// query. Synthesized keys are exempt: the sub-query skips keys absent from metadata.
if skipResourceFilter && !synthesized {
filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
for _, k := range keys {
if k.FieldContext != telemetrytypes.FieldContextResource {
filtered = append(filtered, k)
filtered := make([]*telemetrytypes.LogicalField, 0, len(logicalFields))
for _, logical := range logicalFields {
if logical.FieldContext != telemetrytypes.FieldContextResource {
filtered = append(filtered, logical)
}
}
if len(filtered) == 0 {
return nil, warnings, nil
}
keys = filtered
logicalFields = filtered
}
conds := make([]string, 0, len(keys))
for _, k := range keys {
cond, err := c.conditionForKey(ctx, orgID, startNs, endNs, k, operator, value, sb)
conds := make([]string, 0, len(logicalFields))
for _, logical := range logicalFields {
cond, err := c.conditionForLogicalField(ctx, orgID, startNs, endNs, logical, operator, value, sb)
if err != nil {
return nil, nil, err
}
@@ -283,28 +285,28 @@ func (c *conditionBuilder) ConditionFor(
return conds, warnings, nil
}
func (c *conditionBuilder) conditionForKey(
func (c *conditionBuilder) conditionForLogicalField(
ctx context.Context,
orgID valuer.UUID,
startNs uint64,
endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
logical *telemetrytypes.LogicalField,
operator qbtypes.FilterOperator,
value any,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
if c.isSpanScopeField(key.Name) {
return c.buildSpanScopeCondition(key, operator, value, startNs)
if c.isSpanScopeField(logical.Name) {
return c.buildSpanScopeCondition(logical.Single(), operator, value, startNs)
}
condition, err := c.conditionFor(ctx, orgID, startNs, endNs, key, operator, value, sb)
condition, err := c.conditionFor(ctx, orgID, startNs, endNs, logical, operator, value, sb)
if err != nil {
return "", err
}
if operator.AddDefaultExistsFilter() {
// skip adding exists filter for intrinsic fields
field, _ := c.fm.FieldFor(ctx, orgID, startNs, endNs, key)
field, _ := c.fm.FieldFor(ctx, orgID, startNs, endNs, logical.Single())
if slices.Contains(maps.Keys(IntrinsicFields), field) ||
slices.Contains(maps.Keys(IntrinsicFieldsDeprecated), field) ||
slices.Contains(maps.Keys(CalculatedFields), field) ||
@@ -312,7 +314,7 @@ func (c *conditionBuilder) conditionForKey(
return condition, nil
}
existsCondition, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
existsCondition, err := c.conditionFor(ctx, orgID, startNs, endNs, logical, qbtypes.FilterOperatorExists, nil, sb)
if err != nil {
return "", err
}

View File

@@ -2,6 +2,7 @@ package tracestelemetryschema
import (
"context"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"testing"
"time"
@@ -288,8 +289,8 @@ func TestConditionFor(t *testing.T) {
},
}
fm := NewFieldMapper()
conditionBuilder := NewConditionBuilder(fm)
fm := NewFieldMapper(flaggertest.New(t))
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
for _, tc := range testCases {
sb := sqlbuilder.NewSelectBuilder()
@@ -375,8 +376,8 @@ func TestConditionForResourceWithEvolution(t *testing.T) {
},
}
fm := NewFieldMapper()
conditionBuilder := NewConditionBuilder(fm)
fm := NewFieldMapper(flaggertest.New(t))
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
for _, tc := range testCases {
sb := sqlbuilder.NewSelectBuilder()
@@ -395,8 +396,8 @@ func TestConditionForResourceWithEvolution(t *testing.T) {
// user input and queries anyway, emitting a warning instead of failing.
func TestConditionForSynthesizedKeys(t *testing.T) {
ctx := context.Background()
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
fm := NewFieldMapper(flaggertest.New(t))
cb := NewConditionBuilder(fm, flaggertest.New(t))
// no metadata matches -> the builder must synthesize from user input
var noMatches map[string][]*telemetrytypes.TelemetryFieldKey

View File

@@ -0,0 +1,243 @@
package tracestelemetryschema
import (
"context"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/huandu/go-sqlbuilder"
"github.com/stretchr/testify/require"
)
// familyFlagOn returns a flagger with resolve_semconv_families on.
func familyFlagOn(t *testing.T) flagger.Flagger {
return flaggertest.WithBooleanFlags(t, map[string]bool{
flagger.FeatureResolveSemconvFamilies.String(): true,
})
}
// The tests below run against the deployment.environment(.name) family as
// trace resource attributes with the canonical evolution timeline, inside the
// JSON-column window. The single-member value expression at this range is
//
// multiIf(resource.`<name>` IS NOT NULL, resource.`<name>`::String,
// mapContains(resources_string, '<name>'), resources_string['<name>'], NULL)
//
// and the family expressions below compose it per member.
func TestConditionForFamilyMergesMembersCurrentFirst(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
startNs, endNs := uint64(1747947419000000000), uint64(1747983448000000000)
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
}
fl := familyFlagOn(t)
cb := NewConditionBuilder(NewFieldMapper(fl), fl)
// The requested spelling is the old name; precedence must still be
// current-first.
sb := sqlbuilder.NewSelectBuilder()
conds, warnings, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
require.NoError(t, err)
require.Empty(t, warnings, "a family is one logical field, never ambiguous with itself")
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE (COALESCE(NULLIF(multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL), ''), NULLIF(multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL), ''), '') = ? AND (multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) IS NOT NULL OR multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL) IS NOT NULL))", sql)
require.Equal(t, []any{"production"}, args)
}
func TestConditionForFamilyNegativeKeepsKeylessRows(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
startNs, endNs := uint64(1747947419000000000), uint64(1747983448000000000)
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
}
fl := familyFlagOn(t)
cb := NewConditionBuilder(NewFieldMapper(fl), fl)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotEqual, "production", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
// The trailing '' makes rows without any member read '' (single-key map
// semantics), so `!=` keeps including them; no exists filter is added.
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE COALESCE(NULLIF(multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL), ''), NULLIF(multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL), ''), '') <> ?", sql)
require.Equal(t, []any{"production"}, args)
}
func TestConditionForFamilyExists(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
startNs, endNs := uint64(1747947419000000000), uint64(1747983448000000000)
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
}
fl := familyFlagOn(t)
cb := NewConditionBuilder(NewFieldMapper(fl), fl)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotExists, nil, sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE NOT (multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) IS NOT NULL OR multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL) IS NOT NULL)", sql)
}
// With the flag off, both spellings can be in the metadata map and the
// condition still uses only the requested key. Users see no change until the
// flag is on.
func TestConditionForFamilyOffByDefault(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
startNs, endNs := uint64(1747947419000000000), uint64(1747983448000000000)
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
}
fl := flaggertest.New(t)
cb := NewConditionBuilder(NewFieldMapper(fl), fl)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE (multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) = ? AND multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) IS NOT NULL)", sql)
require.Equal(t, []any{"production"}, args)
}
// With only one member in metadata the condition is the exact single-key
// shape: composition only appears when metadata proves a second member.
func TestConditionForSingleMemberIsUnchanged(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
startNs, endNs := uint64(1747947419000000000), uint64(1747983448000000000)
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
}
delete(fieldKeys, "deployment.environment")
fl := familyFlagOn(t)
cb := NewConditionBuilder(NewFieldMapper(fl), fl)
sb := sqlbuilder.NewSelectBuilder()
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
require.NoError(t, err)
require.Len(t, conds, 1)
sb.Where(conds...)
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
require.Equal(t, "WHERE (multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) = ? AND multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) IS NOT NULL)", sql)
require.Equal(t, []any{"production"}, args)
}
func TestColumnExpressionForFamilyGroupBy(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
startNs, endNs := uint64(1747947419000000000), uint64(1747983448000000000)
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
"deployment.environment.name": {{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
"deployment.environment": {{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: MockEvolutionData(releaseTime),
}},
}
fm := NewFieldMapper(familyFlagOn(t))
expr, err := fm.ColumnExpressionFor(context.Background(), valuer.UUID{}, startNs, endNs,
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, telemetrytypes.FieldDataTypeString, fieldKeys)
require.NoError(t, err)
require.Equal(t, "multiIf((multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL) IS NOT NULL OR multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL) IS NOT NULL), COALESCE(NULLIF(multiIf(resource.`deployment.environment.name` IS NOT NULL, resource.`deployment.environment.name`::String, mapContains(resources_string, 'deployment.environment.name'), resources_string['deployment.environment.name'], NULL), ''), NULLIF(multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, mapContains(resources_string, 'deployment.environment'), resources_string['deployment.environment'], NULL), ''), ''), NULL)", expr)
}

View File

@@ -7,6 +7,7 @@ import (
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -159,12 +160,16 @@ var (
}
)
type fieldMapper struct{}
type fieldMapper struct {
// fl evaluates the resolve_semconv_families flag during resolution.
// A nil flagger keeps resolution literal.
fl flagger.Flagger
}
var _ qbtypes.FieldMapper = (*fieldMapper)(nil)
func NewFieldMapper() *fieldMapper {
return &fieldMapper{}
func NewFieldMapper(fl flagger.Flagger) *fieldMapper {
return &fieldMapper{fl: fl}
}
func (m *fieldMapper) getColumn(
@@ -336,6 +341,68 @@ func (m *fieldMapper) resolveColumnExprs(
return exprs, existExprs, columns, nil
}
// logicalForResolvedColumn upgrades a directly-resolvable key (the FieldFor
// probe succeeded) to its family when the metadata map proves membership;
// otherwise the key stays a single-member logical field.
func (m *fieldMapper) logicalForResolvedColumn(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) *telemetrytypes.LogicalField {
for _, logical := range querybuilder.MatchingLogicalFields(ctx, orgID, m.fl, field, keys) {
if logical.IsFamily() &&
logical.FieldContext == field.FieldContext &&
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || logical.FieldDataType == field.FieldDataType) {
return logical
}
}
return telemetrytypes.SingleLogicalField(field.Name, field)
}
// upgradeToFamilies swaps single-member candidates for their family when the
// metadata map proves membership. Candidate order and every non-family
// candidate stay exactly as the legacy flow produced them; sibling candidates
// of an already-emitted family are dropped rather than duplicated.
func (m *fieldMapper) upgradeToFamilies(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, candidates []*telemetrytypes.LogicalField, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.LogicalField {
var families []*telemetrytypes.LogicalField
for _, logical := range querybuilder.MatchingLogicalFields(ctx, orgID, m.fl, field, keys) {
if logical.IsFamily() {
families = append(families, logical)
}
}
if len(families) == 0 {
return candidates
}
out := make([]*telemetrytypes.LogicalField, 0, len(candidates))
emitted := make(map[*telemetrytypes.LogicalField]bool)
for _, candidate := range candidates {
var family *telemetrytypes.LogicalField
for _, fam := range families {
if fam.FieldContext != candidate.FieldContext || fam.FieldDataType != candidate.FieldDataType {
continue
}
memberOfFamily := candidate.Single().Name == field.Name
for _, member := range fam.Members {
if member.Name == candidate.Single().Name {
memberOfFamily = true
break
}
}
if memberOfFamily {
family = fam
break
}
}
if family == nil {
out = append(out, candidate)
continue
}
if emitted[family] {
continue
}
emitted[family] = true
out = append(out, family)
}
return out
}
// ColumnExpressionFor returns the bare (unaliased) SQL expression for the field, resolving
// unknown keys via CandidateKeys and wrapping guardable columns with exists-guard multiIfs
// so an absent key yields NULL.
@@ -348,18 +415,23 @@ func (m *fieldMapper) ColumnExpressionFor(
keys map[string][]*telemetrytypes.TelemetryFieldKey,
) (string, error) {
// Resolve the candidate column(s).
var candidates []*telemetrytypes.TelemetryFieldKey
// Resolve the candidate logical field(s).
var candidates []*telemetrytypes.LogicalField
switch _, err := m.FieldFor(ctx, orgID, startNs, endNs, field); {
case err == nil:
candidates = []*telemetrytypes.TelemetryFieldKey{field}
// A directly-resolvable key upgrades to its family when the metadata
// map proves membership; otherwise it stays single-member.
candidates = []*telemetrytypes.LogicalField{m.logicalForResolvedColumn(ctx, orgID, field, keys)}
case errors.Is(err, qbtypes.ErrColumnNotFound):
// column (when the bare name is one) plus metadata matches, else synthesized
// type-variant keys.
candidates = m.CandidateKeys(ctx, orgID, field, nil, keys)
if len(candidates) == 0 {
// The legacy candidate flow, unchanged: column (when the bare name is
// one) plus metadata matches, else synthesized type-variant keys. The
// family step below only swaps candidates for their family; it never
// changes candidate order or non-family behavior.
raw := m.CandidateKeys(ctx, orgID, field, nil, keys)
if len(raw) == 0 {
return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...)
}
candidates = m.upgradeToFamilies(ctx, orgID, field, querybuilder.WrapAsLogicalFields(field.Name, raw), keys)
default:
return "", err
}
@@ -373,21 +445,21 @@ func (m *fieldMapper) ColumnExpressionFor(
dummyValue = 0.0
}
stmts := make([]string, 0, len(candidates)*2)
for _, key := range candidates {
value, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
for _, logical := range candidates {
value, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, m, logical)
if err != nil {
return "", err
}
guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, key, true)
guard, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, m, logical, true)
if err != nil {
return "", err
}
coerced := value
// a time column keeps its native type; coercing it would yield seconds
if temporal, err := m.columnIsTemporal(ctx, startNs, endNs, key); err != nil {
if temporal, err := m.logicalIsTemporal(ctx, startNs, endNs, logical); err != nil {
return "", err
} else if !temporal {
coerced, _ = querybuilder.DataTypeCollisionHandledFieldName(key, dummyValue, value, qbtypes.FilterOperatorUnknown)
coerced, _ = querybuilder.DataTypeCollisionHandledFieldName(logical.Single(), dummyValue, value, qbtypes.FilterOperatorUnknown)
}
stmts = append(stmts, guard, coerced)
}
@@ -395,13 +467,14 @@ func (m *fieldMapper) ColumnExpressionFor(
}
if len(candidates) == 1 {
value, err := m.FieldFor(ctx, orgID, startNs, endNs, candidates[0])
logical := candidates[0]
value, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, m, logical)
if err != nil {
return "", err
}
exprs, existExprs, _, _ := m.resolveColumnExprs(ctx, startNs, endNs, candidates[0])
if len(exprs) == 1 && len(existExprs) == 1 {
guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, candidates[0], true)
exprs, existExprs, _, _ := m.resolveColumnExprs(ctx, startNs, endNs, logical.Single())
if !logical.IsFamily() && len(exprs) == 1 && len(existExprs) == 1 {
guard, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, m, logical, true)
if err != nil {
return "", err
}
@@ -413,12 +486,12 @@ func (m *fieldMapper) ColumnExpressionFor(
// Multiple candidates (collision / synth): multiIf picks the first that exists,
// stringified so branches share a type.
args := make([]string, 0, len(candidates))
for _, key := range candidates {
value, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
for _, logical := range candidates {
value, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, m, logical)
if err != nil {
return "", err
}
guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, key, true)
guard, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, m, logical, true)
if err != nil {
return "", err
}
@@ -427,6 +500,15 @@ func (m *fieldMapper) ColumnExpressionFor(
return fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", ")), nil
}
// logicalIsTemporal reports whether the logical field resolves to a single time
// column. A family is attribute-backed and never temporal.
func (m *fieldMapper) logicalIsTemporal(ctx context.Context, startNs, endNs uint64, logical *telemetrytypes.LogicalField) (bool, error) {
if logical.IsFamily() {
return false, nil
}
return m.columnIsTemporal(ctx, startNs, endNs, logical.Single())
}
// columnIsTemporal reports whether key resolves to a single time column, after evolution
// selection. Multiple columns mean an attribute-map union, which is never temporal.
func (m *fieldMapper) columnIsTemporal(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (bool, error) {
@@ -522,7 +604,8 @@ func (m *fieldMapper) CandidateKeys(ctx context.Context, _ valuer.UUID, field *t
return nil
}
func (m *fieldMapper) existsExpressionFor(
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
func (m *fieldMapper) ExistsFor(
ctx context.Context,
orgID valuer.UUID,
tsStart, tsEnd uint64,

View File

@@ -2,6 +2,7 @@ package tracestelemetryschema
import (
"context"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"testing"
"time"
@@ -107,7 +108,7 @@ func TestGetFieldKeyName(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fm := NewFieldMapper()
fm := NewFieldMapper(flaggertest.New(t))
result, err := fm.FieldFor(ctx, valuer.UUID{}, uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano()), &tc.key)
if tc.expectedError != nil {
@@ -195,7 +196,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fm := NewFieldMapper()
fm := NewFieldMapper(flaggertest.New(t))
result, err := fm.FieldFor(ctx, valuer.UUID{}, tc.tsStart, tc.tsEnd, &tc.key)
require.NoError(t, err)
assert.Equal(t, tc.expectedResult, result)
@@ -255,7 +256,7 @@ func TestColumnExpressionForTemporalColumn(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fm := NewFieldMapper()
fm := NewFieldMapper(flaggertest.New(t))
result, err := fm.ColumnExpressionFor(ctx, valuer.UUID{}, tsStart, tsEnd, &tc.key, tc.requiredDataType, nil)
require.NoError(t, err)
assert.Equal(t, tc.expectedResult, result)
@@ -283,7 +284,7 @@ func TestColumnExpressionForTimestampAttributeCollision(t *testing.T) {
},
}
fm := NewFieldMapper()
fm := NewFieldMapper(flaggertest.New(t))
t.Run("bare timestamp resolves to the intrinsic column alone", func(t *testing.T) {
bare := telemetrytypes.TelemetryFieldKey{Name: "timestamp"}

View File

@@ -2,6 +2,7 @@ package tracestelemetryschema
import (
"context"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"testing"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
@@ -14,8 +15,8 @@ import (
func TestSpanScopeFilterExpression(t *testing.T) {
// Test that span scope fields work in filter expressions
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
fm := NewFieldMapper(flaggertest.New(t))
cb := NewConditionBuilder(fm, flaggertest.New(t))
tests := []struct {
name string
@@ -128,8 +129,8 @@ func TestSpanScopeWithResourceFilter(t *testing.T) {
// For now, just verify the expression parses correctly
// In a real implementation, we'd need to check that the resource filter
// is properly skipped when span scope fields are present
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
fm := NewFieldMapper(flaggertest.New(t))
cb := NewConditionBuilder(fm, flaggertest.New(t))
// Prepare field keys for the test
fieldKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey)

View File

@@ -113,6 +113,21 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
FieldDataType: telemetrytypes.FieldDataTypeBool,
},
},
// both spellings of an enabled semantic-convention family
"deployment.environment.name": {
{
Name: "deployment.environment.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"deployment.environment": {
{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
}
for _, keys := range keysMap {
for _, key := range keys {

View File

@@ -46,7 +46,6 @@ var (
GCPServiceComputeEngine = ServiceID{valuer.NewString("computeengine")}
GCPServiceGKE = ServiceID{valuer.NewString("gke")}
GCPServiceCloudStorage = ServiceID{valuer.NewString("cloudstorage")}
GCPServiceCloudSQLMySQL = ServiceID{valuer.NewString("cloudsql_mysql")}
)
func (ServiceID) Enum() []any {
@@ -83,7 +82,6 @@ func (ServiceID) Enum() []any {
GCPServiceComputeEngine,
GCPServiceGKE,
GCPServiceCloudStorage,
GCPServiceCloudSQLMySQL,
}
}
@@ -126,7 +124,6 @@ var SupportedServices = map[CloudProviderType][]ServiceID{
GCPServiceComputeEngine,
GCPServiceGKE,
GCPServiceCloudStorage,
GCPServiceCloudSQLMySQL,
},
}

View File

@@ -201,18 +201,6 @@ func (storableDashboardData *StorableDashboardData) GetWidgetIds() []string {
return widgetIds
}
// ErrIfNotDeletable gates deletion on the columns alone, never on Data, so a
// dashboard whose data is corrupt or stuck on the v1 schema stays deletable.
func (storable StorableDashboard) ErrIfNotDeletable() error {
if storable.Locked {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cannot delete a locked dashboard, please unlock the dashboard to delete")
}
if !storable.Source.isUserDeletable() {
return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be deleted", storable.Source)
}
return nil
}
func (dashboard *Dashboard) ErrIfNotMutable() error {
if dashboard.Source == SourceIntegration {
return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "integration dashboards cannot be modified")

View File

@@ -4,7 +4,6 @@ import (
"context"
"testing"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
)
@@ -82,64 +81,3 @@ func TestCanUpdate_MultipleDeletions_ByDiff(t *testing.T) {
})
}
}
func TestStorableDashboardErrIfNotDeletable(t *testing.T) {
testCases := []struct {
subtestName string
locked bool
source Source
data StorableDashboardData
expectDeletable bool
}{
{
subtestName: "user dashboard on the v2 schema",
source: SourceUser,
data: StorableDashboardData{"metadata": map[string]any{"schemaVersion": SchemaVersion}},
expectDeletable: true,
},
{
subtestName: "user dashboard still on the v1 schema",
source: SourceUser,
data: StorableDashboardData{"widgets": makeTestWidgets("a")},
expectDeletable: true,
},
{
subtestName: "user dashboard with unreadable data",
source: SourceUser,
data: StorableDashboardData{"metadata": "not-an-object"},
expectDeletable: true,
},
{
subtestName: "locked user dashboard",
locked: true,
source: SourceUser,
data: StorableDashboardData{"widgets": makeTestWidgets("a")},
expectDeletable: false,
},
{
subtestName: "system dashboard",
source: SourceSystem,
data: StorableDashboardData{"widgets": makeTestWidgets("a")},
expectDeletable: false,
},
{
subtestName: "integration dashboard",
source: SourceIntegration,
data: StorableDashboardData{"widgets": makeTestWidgets("a")},
expectDeletable: false,
},
}
for _, tc := range testCases {
t.Run(tc.subtestName, func(t *testing.T) {
storable := StorableDashboard{
Identifiable: types.Identifiable{ID: valuer.GenerateUUID()},
OrgID: valuer.GenerateUUID(),
Locked: tc.locked,
Source: tc.source,
Data: tc.data,
}
assert.Equal(t, tc.expectDeletable, storable.ErrIfNotDeletable() == nil)
})
}
}

View File

@@ -129,6 +129,16 @@ func (d *DashboardV2) LockUnlock(lock bool, isAdmin bool, updatedBy string) erro
return nil
}
func (d *DashboardV2) ErrIfNotDeletable() error {
if d.Locked {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cannot delete a locked dashboard, please unlock the dashboard to delete")
}
if !d.Source.isUserDeletable() {
return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be deleted", d.Source)
}
return nil
}
func (d *DashboardV2) ErrIfNotClonable() error {
if !d.Source.isClonable() {
return errors.Newf(errors.TypeInvalidInput, ErrCodeDashboardImmutable, "%s dashboards cannot be cloned", d.Source)

View File

@@ -18,7 +18,14 @@ var (
ErrUnsupportedOperator = errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator")
)
// FieldMapper maps the telemetry field key to the table field name.
// FieldMapper is a signal's storage dialect: given a resolved physical key, it
// renders how this signal's tables express that key — its value (FieldFor),
// its presence (ExistsFor), and its backing columns (ColumnFor) — honoring
// materialization and column-generation choices carried on the key.
//
// ColumnExpressionFor and CandidateKeys additionally carry per-signal
// resolution behavior (unknown-name fallback, candidate ordering); they
// predate the resolution layer and are slated to move behind it.
type FieldMapper interface {
// FieldFor returns the field name for the given key.
FieldFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error)
@@ -34,6 +41,11 @@ type FieldMapper interface {
// the name (or `{context}.{name}`) first, else synthesized type-variant keys for sources
// that support it, else nil (caller errors). value is the filter operand, nil otherwise.
CandidateKeys(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, value any, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey
// ExistsFor returns the existence predicate for a single physical key (negated when
// exists is false), self-contained and arg-free so it can guard column expressions.
// It is the per-member primitive querybuilder.LogicalExistsExpr and the numeric branch
// of querybuilder.LogicalValueExpr compose family expressions from.
ExistsFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error)
}
// ConditionBuilder builds the conditions for a filter term. The builder owns key resolution:

View File

@@ -26,14 +26,14 @@ class ProviderAccountSpec:
provider: str
# params for the account created by default.
initial_params: dict
# params for the config an update (PUT) test sends.
updated_params: dict
# params -> the provider-keyed `config` block for a POST/PUT body.
build_config: Callable[[dict], dict]
# params -> the full config block the API is expected to return under
# config[provider] on GET/list. This may differ from what build_config sends:
# e.g. AWS accepts deploymentRegion on POST but the API does not echo it back.
expected_config: Callable[[dict], dict]
# only the suites that exercise updates need to supply it.
updated_params: dict = field(default_factory=dict)
# id shown in parametrized test names; defaults to the provider slug.
id: str = field(default="")
@@ -42,29 +42,6 @@ class ProviderAccountSpec:
object.__setattr__(self, "id", self.provider)
# Per-provider service shape.
@dataclass(frozen=True)
class ProviderServiceSpec:
provider: str
service_id: str
# GCP ships every service with supportedSignals.logs false, so a logs block
# is neither required on write nor persisted.
supports_logs: bool
account_config: dict
# id shown in parametrized test names; defaults to the provider slug.
id: str = field(default="")
def __post_init__(self) -> None:
if not self.id:
object.__setattr__(self, "id", self.provider)
def build_service_config(self, metrics_enabled: bool, logs_enabled: bool | None = None) -> dict:
config: dict = {"metrics": {"enabled": metrics_enabled}}
if self.supports_logs:
config["logs"] = {"enabled": metrics_enabled if logs_enabled is None else logs_enabled}
return {self.provider: config}
@pytest.fixture(scope="function")
def deprecated_create_cloud_integration_account(
request: pytest.FixtureRequest,

File diff suppressed because it is too large Load Diff

View File

@@ -1,52 +1,14 @@
from collections.abc import Callable
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.cloudintegrations import ProviderAccountSpec
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
AWS_ACCOUNT_SPEC = ProviderAccountSpec(
provider="aws",
initial_params={"deployment_region": "us-east-1", "regions": ["us-east-1", "us-west-2"]},
build_config=lambda p: {"aws": {"deploymentRegion": p["deployment_region"], "regions": p["regions"]}},
expected_config=lambda p: {"regions": p["regions"]},
)
GCP_ACCOUNT_SPEC = ProviderAccountSpec(
provider="gcp",
initial_params={
"deployment_project_id": "signoz-test-project",
"deployment_region": "us-central1",
"project_ids": ["signoz-test-project"],
},
build_config=lambda p: {
"gcp": {
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
}
},
expected_config=lambda p: {
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
},
)
PROVIDER_ACCOUNT_SPECS = [AWS_ACCOUNT_SPEC, GCP_ACCOUNT_SPEC]
provider_spec = pytest.mark.parametrize(
"spec",
PROVIDER_ACCOUNT_SPECS,
ids=[s.id for s in PROVIDER_ACCOUNT_SPECS],
)
def test_apply_license(
signoz: types.SigNoz,
@@ -58,19 +20,19 @@ def test_apply_license(
add_license(signoz, make_http_mocks, get_token)
@provider_spec
def test_create_account(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderAccountSpec,
) -> None:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
cloud_provider = "aws"
data = create_cloud_integration_account(
admin_token,
spec.provider,
config=spec.build_config(spec.initial_params),
cloud_provider,
deployment_region="us-east-1",
regions=["us-east-1", "us-west-2"],
)
assert "id" in data, "Response data should contain 'id' field"
@@ -78,17 +40,12 @@ def test_create_account(
assert "connectionArtifact" in data, "Response data should contain 'connectionArtifact' field"
artifact = data["connectionArtifact"]
assert "aws" in artifact, "connectionArtifact should contain 'aws' field"
assert "connectionUrl" in artifact["aws"], "connectionArtifact.aws should contain 'connectionUrl'"
if spec.provider == "aws":
assert "aws" in artifact, "connectionArtifact should contain 'aws' field"
assert "connectionUrl" in artifact["aws"], "connectionArtifact.aws should contain 'connectionUrl'"
connection_url = artifact["aws"]["connectionUrl"]
assert "console.aws.amazon.com/cloudformation" in connection_url, "connectionUrl should be an AWS CloudFormation URL"
assert f"region={spec.initial_params['deployment_region']}" in connection_url, "connectionUrl should contain the deployment region"
else:
# GCP is a manual flow: no one-click install artifact.
assert artifact.get("gcp") is None, f"GCP should not return a connection artifact, got: {artifact}"
connection_url = artifact["aws"]["connectionUrl"]
assert "console.aws.amazon.com/cloudformation" in connection_url, "connectionUrl should be an AWS CloudFormation URL"
assert "region=us-east-1" in connection_url, "connectionUrl should contain the deployment region"
def test_create_account_unsupported_provider(
@@ -119,36 +76,3 @@ def test_create_account_unsupported_provider(
response_data = response.json()
assert "error" in response_data, "Response should contain 'error' field"
def test_create_gcp_account_without_project_ids(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""GCP account config requires at least one project ID to monitor."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/cloud_integrations/gcp/accounts"),
headers={"Authorization": f"Bearer {admin_token}"},
json={
"config": {
"gcp": {
"deploymentProjectId": "signoz-test-project",
"deploymentRegion": "us-central1",
"projectIds": [],
}
},
"credentials": {
"sigNozApiURL": "https://test.signoz.cloud",
"sigNozApiKey": "test-key",
"ingestionUrl": "https://ingest.test.signoz.cloud",
"ingestionKey": "test-ingestion-key",
},
},
timeout=10,
)
assert response.status_code == HTTPStatus.BAD_REQUEST, f"Expected 400 for empty projectIds, got {response.status_code}: {response.text}"
assert "error" in response.json(), "Response should contain 'error' field"

View File

@@ -2,53 +2,14 @@ import uuid
from collections.abc import Callable
from http import HTTPStatus
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.cloudintegrations import (
ProviderAccountSpec,
simulate_agent_checkin,
)
from fixtures.cloudintegrations import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
AWS_ACCOUNT_SPEC = ProviderAccountSpec(
provider="aws",
initial_params={"deployment_region": "us-east-1", "regions": ["us-east-1"]},
build_config=lambda p: {"aws": {"deploymentRegion": p["deployment_region"], "regions": p["regions"]}},
expected_config=lambda p: {"regions": p["regions"]},
)
GCP_ACCOUNT_SPEC = ProviderAccountSpec(
provider="gcp",
initial_params={
"deployment_project_id": "signoz-test-project",
"deployment_region": "us-central1",
"project_ids": ["signoz-test-project"],
},
build_config=lambda p: {
"gcp": {
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
}
},
expected_config=lambda p: {
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
},
)
PROVIDER_ACCOUNT_SPECS = [AWS_ACCOUNT_SPEC, GCP_ACCOUNT_SPEC]
provider_spec = pytest.mark.parametrize(
"spec",
PROVIDER_ACCOUNT_SPECS,
ids=[s.id for s in PROVIDER_ACCOUNT_SPECS],
)
CLOUD_PROVIDER = "aws"
def test_apply_license(
@@ -61,28 +22,22 @@ def test_apply_license(
add_license(signoz, make_http_mocks, get_token)
@provider_spec
def test_agent_check_in(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderAccountSpec,
) -> None:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(
admin_token,
spec.provider,
config=spec.build_config(spec.initial_params),
)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER, regions=["us-east-1"])
account_id = account["id"]
provider_account_id = str(uuid.uuid4())
response = simulate_agent_checkin(
signoz,
admin_token,
spec.provider,
CLOUD_PROVIDER,
account_id,
provider_account_id,
data={"version": "v0.0.8"},
@@ -92,63 +47,57 @@ def test_agent_check_in(
data = response.json()["data"]
# New camelCase fields
assert data["cloudIntegrationId"] == account_id, "cloudIntegrationId should match"
assert data["providerAccountId"] == provider_account_id, "providerAccountId should match"
assert "integrationConfig" in data, "Response should contain 'integrationConfig'"
assert data["removedAt"] is None, "removedAt should be null for a live account"
if spec.provider == "aws":
# Backward compat for agents deployed before the camelCase response; AWS only.
assert data["account_id"] == account_id, "account_id (compat) should match"
assert data["cloud_account_id"] == provider_account_id, "cloud_account_id (compat) should match"
assert "integration_config" in data, "Response should contain 'integration_config' (compat)"
assert "removed_at" in data, "Response should contain 'removed_at' (compat)"
# Backward-compat snake_case fields
assert data["account_id"] == account_id, "account_id (compat) should match"
assert data["cloud_account_id"] == provider_account_id, "cloud_account_id (compat) should match"
assert "integration_config" in data, "Response should contain 'integration_config' (compat)"
assert "removed_at" in data, "Response should contain 'removed_at' (compat)"
integration_config = data["integrationConfig"]
assert "aws" in integration_config, "integrationConfig should contain 'aws' block"
assert integration_config["aws"]["enabledRegions"] == spec.initial_params["regions"], "enabledRegions should match account config"
else:
# GCP is a manual flow: the agent carries its own configuration.
assert data["integrationConfig"].get("gcp") is None, f"GCP should not return an integration config, got: {data['integrationConfig']}"
# integrationConfig should reflect the configured regions
integration_config = data["integrationConfig"]
assert "aws" in integration_config, "integrationConfig should contain 'aws' block"
assert integration_config["aws"]["enabledRegions"] == ["us-east-1"], "enabledRegions should match account config"
@provider_spec
def test_agent_check_in_account_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
spec: ProviderAccountSpec,
) -> None:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
fake_id = str(uuid.uuid4())
response = simulate_agent_checkin(signoz, admin_token, spec.provider, fake_id, str(uuid.uuid4()))
response = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, fake_id, str(uuid.uuid4()))
assert response.status_code == HTTPStatus.NOT_FOUND, f"Expected 404, got {response.status_code}: {response.text}"
@provider_spec
def test_duplicate_cloud_account_checkins(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderAccountSpec,
) -> None:
"""Test that two different accounts cannot check in with the same providerAccountId."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account1 = create_cloud_integration_account(admin_token, spec.provider, config=spec.build_config(spec.initial_params))
account2 = create_cloud_integration_account(admin_token, spec.provider, config=spec.build_config(spec.initial_params))
account1 = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account2 = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
assert account1["id"] != account2["id"], "Two accounts should have different IDs"
same_provider_account_id = str(uuid.uuid4())
# First check-in: account1 claims the provider account ID
response = simulate_agent_checkin(signoz, admin_token, spec.provider, account1["id"], same_provider_account_id)
response = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account1["id"], same_provider_account_id)
assert response.status_code == HTTPStatus.OK, f"Expected 200 for first check-in, got {response.status_code}: {response.text}"
# Second check-in: account2 tries to claim the same provider account ID → 409
response = simulate_agent_checkin(signoz, admin_token, spec.provider, account2["id"], same_provider_account_id)
response = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account2["id"], same_provider_account_id)
assert response.status_code == HTTPStatus.CONFLICT, f"Expected 409 for duplicate providerAccountId, got {response.status_code}: {response.text}"

View File

@@ -2,47 +2,18 @@ import uuid
from collections.abc import Callable
from http import HTTPStatus
import pytest
import requests
from sqlalchemy import bindparam, sql
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.cloudintegrations import (
ProviderServiceSpec,
simulate_agent_checkin,
)
from fixtures.cloudintegrations import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
AWS_SERVICE_SPEC = ProviderServiceSpec(
provider="aws",
service_id="rds",
supports_logs=True,
account_config={"aws": {"deploymentRegion": "us-east-1", "regions": ["us-east-1"]}},
)
GCP_SERVICE_SPEC = ProviderServiceSpec(
provider="gcp",
service_id="cloudsql_postgres",
supports_logs=False,
account_config={
"gcp": {
"deploymentProjectId": "signoz-test-project",
"deploymentRegion": "us-central1",
"projectIds": ["signoz-test-project"],
}
},
)
PROVIDER_SERVICE_SPECS = [AWS_SERVICE_SPEC, GCP_SERVICE_SPEC]
provider_spec = pytest.mark.parametrize(
"spec",
PROVIDER_SERVICE_SPECS,
ids=[s.id for s in PROVIDER_SERVICE_SPECS],
)
CLOUD_PROVIDER = "aws"
SERVICE_ID = "rds"
def test_apply_license(
@@ -55,18 +26,16 @@ def test_apply_license(
add_license(signoz, make_http_mocks, get_token)
@provider_spec
def test_list_services_without_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
spec: ProviderServiceSpec,
) -> None:
"""List the cloud provider's supported services"""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/services"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -84,37 +53,35 @@ def test_list_services_without_account(
assert "icon" in service, "Service should have 'icon' field"
assert "enabled" in service, "Service should have 'enabled' field"
listed_ids = {s["id"] for s in data["services"]}
assert spec.service_id in listed_ids, f"'{spec.service_id}' should be listed for {spec.provider}"
EC2_SERVICE_ID = "ec2"
@provider_spec
def test_list_account_services(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""ListAccountServicesMetadata reflects enabled state per service."""
"""ListAccountServicesMetadata reflects enabled state after enabling a service."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
put_response = requests.put(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{EC2_SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True)},
json={"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": True}}}},
timeout=10,
)
assert put_response.status_code == HTTPStatus.NO_CONTENT, f"Enable {spec.service_id} failed: {put_response.status_code}: {put_response.text}"
assert put_response.status_code == HTTPStatus.NO_CONTENT, f"Enable ec2 failed: {put_response.status_code}: {put_response.text}"
list_response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -125,28 +92,21 @@ def test_list_account_services(
assert isinstance(data["services"], list), "services should be a list"
assert len(data["services"]) > 0, "services list should be non-empty"
enabled_service = next((s for s in data["services"] if s["id"] == spec.service_id), None)
assert enabled_service is not None, f"Service '{spec.service_id}' not found in services list"
assert enabled_service["enabled"] is True, f"Service should be enabled, got: {enabled_service['enabled']}"
# The listing must report state per service, not blanket-enable or echo the write.
untouched_service = next((s for s in data["services"] if s["id"] != spec.service_id), None)
assert untouched_service is not None, "Expected more than one service in the listing"
assert untouched_service["enabled"] is False, f"Service '{untouched_service['id']}' was never enabled, got: {untouched_service['enabled']}"
ec2_service = next((s for s in data["services"] if s["id"] == EC2_SERVICE_ID), None)
assert ec2_service is not None, f"EC2 service '{EC2_SERVICE_ID}' not found in services list"
assert ec2_service["enabled"] is True, f"EC2 service should be enabled, got: {ec2_service['enabled']}"
@provider_spec
def test_get_service_details_without_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
spec: ProviderServiceSpec,
) -> None:
"""Get full service definition without specifying an account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -154,36 +114,31 @@ def test_get_service_details_without_account(
assert response.status_code == HTTPStatus.OK, f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert data["id"] == spec.service_id, f"id should be '{spec.service_id}'"
assert data["id"] == SERVICE_ID, f"id should be '{SERVICE_ID}'"
assert "title" in data, "Service should have 'title'"
assert "overview" in data, "Service should have 'overview' (markdown)"
assert "assets" in data, "Service should have 'assets'"
assert isinstance(data["assets"]["dashboards"], list), "assets.dashboards should be a list"
assert data["cloudIntegrationService"] is None, "cloudIntegrationService should be null without account context"
assert data["supportedSignals"]["metrics"] is True, "metrics should be a supported signal"
assert data["supportedSignals"]["logs"] is spec.supports_logs, f"logs support should be {spec.supports_logs} for {spec.provider}"
@provider_spec
def test_get_account_service(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""Get service for a specific account — all disabled by default."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -191,22 +146,20 @@ def test_get_account_service(
assert response.status_code == HTTPStatus.OK, f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert data["id"] == spec.service_id, f"id should be '{spec.service_id}'"
assert data["id"] == SERVICE_ID, f"id should be '{SERVICE_ID}'"
assert data["cloudIntegrationService"] is None, "cloudIntegrationService should be null before any config is set"
@provider_spec
def test_get_service_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
spec: ProviderServiceSpec,
) -> None:
"""Get a non-existent service ID returns 400 (invalid service ID is a bad request)."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/services/non-existent-service"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/non-existent-service"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -214,34 +167,32 @@ def test_get_service_not_found(
assert response.status_code == HTTPStatus.BAD_REQUEST, f"Expected 400, got {response.status_code}"
@provider_spec
def test_update_service_config(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""Enable a service and verify the config is persisted via GET."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
put_response = requests.put(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True)},
json={"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": True}}}},
timeout=10,
)
assert put_response.status_code == HTTPStatus.NO_CONTENT, f"Expected 204, got {put_response.status_code}: {put_response.text}"
get_response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -250,39 +201,33 @@ def test_update_service_config(
data = get_response.json()["data"]
svc = data["cloudIntegrationService"]
assert svc is not None, "cloudIntegrationService should be non-null after UpdateService"
assert svc["config"][spec.provider]["metrics"]["enabled"] is True, "metrics should be enabled"
assert svc["config"]["aws"]["metrics"]["enabled"] is True, "metrics should be enabled"
assert svc["config"]["aws"]["logs"]["enabled"] is True, "logs should be enabled"
assert svc["cloudIntegrationId"] == account_id, "cloudIntegrationId should match the account"
if spec.supports_logs:
assert svc["config"][spec.provider]["logs"]["enabled"] is True, "logs should be enabled"
else:
assert svc["config"][spec.provider].get("logs") is None, f"logs should not be stored for {spec.provider}, got: {svc['config'][spec.provider]}"
@provider_spec
def test_update_service_config_disable(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""Enable then disable a service — config change is persisted."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
endpoint = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}")
endpoint = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}")
# Enable
r = requests.put(
endpoint,
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True)},
json={"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": True}}}},
timeout=10,
)
assert r.status_code == HTTPStatus.NO_CONTENT, f"Enable failed: {r.status_code}: {r.text}"
@@ -291,13 +236,13 @@ def test_update_service_config_disable(
r = requests.put(
endpoint,
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(False)},
json={"config": {"aws": {"metrics": {"enabled": False}, "logs": {"enabled": False}}}},
timeout=10,
)
assert r.status_code == HTTPStatus.NO_CONTENT, f"Disable failed: {r.status_code}: {r.text}"
get_response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -305,57 +250,28 @@ def test_update_service_config_disable(
assert get_response.status_code == HTTPStatus.OK
svc = get_response.json()["data"]["cloudIntegrationService"]
assert svc is not None, "cloudIntegrationService should still be present after disable"
assert svc["config"][spec.provider]["metrics"]["enabled"] is False, "metrics should be disabled"
if spec.supports_logs:
assert svc["config"][spec.provider]["logs"]["enabled"] is False, "logs should be disabled"
assert svc["config"]["aws"]["metrics"]["enabled"] is False, "metrics should be disabled"
assert svc["config"]["aws"]["logs"]["enabled"] is False, "logs should be disabled"
@provider_spec
def test_update_service_account_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
spec: ProviderServiceSpec,
) -> None:
"""PUT with a non-existent account UUID returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.put(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{uuid.uuid4()}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{uuid.uuid4()}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True)},
json={"config": {"aws": {"metrics": {"enabled": True}}}},
timeout=10,
)
assert response.status_code == HTTPStatus.NOT_FOUND, f"Expected 404, got {response.status_code}"
def test_update_gcp_service_without_metrics_config(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""GCP services support metrics only, so a config omitting metrics is rejected."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, "gcp", config=GCP_SERVICE_SPEC.account_config)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
response = requests.put(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/gcp/accounts/{account_id}/services/{GCP_SERVICE_SPEC.service_id}"),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": {"gcp": {"logs": {"enabled": True}}}},
timeout=10,
)
assert response.status_code == HTTPStatus.BAD_REQUEST, f"Expected 400 when metrics config is missing, got {response.status_code}: {response.text}"
def test_list_services_unsupported_provider(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
@@ -373,32 +289,30 @@ def test_list_services_unsupported_provider(
assert response.status_code == HTTPStatus.BAD_REQUEST, f"Expected 400, got {response.status_code}"
@provider_spec
def test_list_services_account_removed(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""List services for a deleted account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
delete_response = requests.delete(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert delete_response.status_code == HTTPStatus.NO_CONTENT, f"Expected 204 on delete, got {delete_response.status_code}"
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -406,32 +320,30 @@ def test_list_services_account_removed(
assert response.status_code == HTTPStatus.NOT_FOUND, f"Expected 404, got {response.status_code}"
@provider_spec
def test_get_service_details_account_removed(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""Get service details for a deleted account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
delete_response = requests.delete(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert delete_response.status_code == HTTPStatus.NO_CONTENT, f"Expected 204 on delete, got {delete_response.status_code}"
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -439,68 +351,64 @@ def test_get_service_details_account_removed(
assert response.status_code == HTTPStatus.NOT_FOUND, f"Expected 404, got {response.status_code}"
@provider_spec
def test_update_service_account_removed(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""PUT service config for a deleted account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
delete_response = requests.delete(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert delete_response.status_code == HTTPStatus.NO_CONTENT, f"Expected 204 on delete, got {delete_response.status_code}"
response = requests.put(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True)},
json={"config": {"aws": {"metrics": {"enabled": True}}}},
timeout=10,
)
assert response.status_code == HTTPStatus.NOT_FOUND, f"Expected 404, got {response.status_code}"
@provider_spec
def test_enable_metrics_provisions_dashboards(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""Enabling metrics provisions dashboards visible in GetService and present in the DB."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
put_response = requests.put(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True, logs_enabled=False)},
json={"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": False}}}},
timeout=10,
)
assert put_response.status_code == HTTPStatus.NO_CONTENT, f"Expected 204, got {put_response.status_code}: {put_response.text}"
# Assertion 1: GetService returns provisioned dashboard UUIDs
get_svc_response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -509,7 +417,7 @@ def test_enable_metrics_provisions_dashboards(
data = get_svc_response.json()["data"]
svc = data["cloudIntegrationService"]
assert svc is not None, "cloudIntegrationService should be non-null after enabling metrics"
assert svc["config"][spec.provider]["metrics"]["enabled"] is True
assert svc["config"]["aws"]["metrics"]["enabled"] is True
dashboards_in_service = data["assets"]["dashboards"]
assert isinstance(dashboards_in_service, list) and len(dashboards_in_service) > 0, "assets.dashboards should be non-empty after enabling metrics"
@@ -537,37 +445,35 @@ def test_enable_metrics_provisions_dashboards(
assert provisioned_ids == db_ids, f"Dashboards {provisioned_ids - db_ids} are missing from the DB"
@provider_spec
def test_disable_metrics_deprovisions_dashboards(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
spec: ProviderServiceSpec,
) -> None:
"""Disabling metrics removes provisioned dashboards from both GetService and the dashboards list."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, spec.provider, config=spec.account_config)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(signoz, admin_token, spec.provider, account_id, str(uuid.uuid4()))
checkin = simulate_agent_checkin(signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4()))
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
endpoint = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}")
endpoint = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}")
# Enable metrics to provision dashboards first
enable_response = requests.put(
endpoint,
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(True, logs_enabled=False)},
json={"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": False}}}},
timeout=10,
)
assert enable_response.status_code == HTTPStatus.NO_CONTENT, f"Enable failed: {enable_response.status_code}: {enable_response.text}"
# Capture the provisioned dashboard IDs before disabling
get_svc_response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
@@ -579,14 +485,14 @@ def test_disable_metrics_deprovisions_dashboards(
disable_response = requests.put(
endpoint,
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": spec.build_service_config(False)},
json={"config": {"aws": {"metrics": {"enabled": False}, "logs": {"enabled": False}}}},
timeout=10,
)
assert disable_response.status_code == HTTPStatus.NO_CONTENT, f"Disable failed: {disable_response.status_code}: {disable_response.text}"
# Assertion 1: GetService no longer returns UUID dashboard IDs
get_svc_after = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{spec.provider}/accounts/{account_id}/services/{spec.service_id}"),
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)

View File

@@ -1,14 +1,20 @@
import os
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import get_column_data_from_response, make_query_request
TESTDATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "testdata")
FILTER_EXPRESSIONS_FILE = os.path.join(TESTDATA_DIR, "filter_expressions_10000.txt")
@pytest.mark.parametrize(
"expression,expected_logs",
@@ -174,3 +180,101 @@ def test_not_filter_expression(
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
assert set(get_column_data_from_response(response.json(), "body")) == expected_logs
def test_filter_expressions_no_server_error(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
insert_logs,
get_token: Callable[[str, str], str],
) -> None:
"""
Reads every line from filter_expressions_10000.txt and fires it as a filter
expression against the logs query endpoint.
Expressions may be valid (200) or invalid (400) — both are acceptable.
A 500 means the server crashed on the input and is a test failure.
All failing expressions are collected before asserting so the full list is
visible in one run.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=UTC)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=5),
body="alpha-log",
resources={
"f1": "v10",
"f2": "v20",
"f3": "v30",
},
attributes={
"f4": 40,
"f5": 50,
"f6": 60,
},
),
Logs(
timestamp=now - timedelta(seconds=3),
body="beta-log",
resources={
"f4": "v41",
"f5": "v51",
"f6": "v61",
},
attributes={
"f1": 11,
"f2": 21,
"f3": 31,
},
),
]
)
def _make_raw_logs_query(
signoz: types.SigNoz,
token: str,
filter_expression: str,
) -> requests.Response:
"""Helper to query raw logs with a filter expression over the last 30 seconds."""
now = datetime.now(tz=UTC)
return make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
request_type="raw",
queries=[
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "logs",
"disabled": False,
"limit": 100,
"offset": 0,
"filter": {"expression": filter_expression},
"order": [
{"key": {"name": "timestamp"}, "direction": "desc"},
{"key": {"name": "id"}, "direction": "desc"},
],
"having": {"expression": ""},
"aggregations": [{"expression": "count()"}],
},
}
],
)
failures: list[str] = []
with ThreadPoolExecutor(max_workers=40) as executor:
with open(FILTER_EXPRESSIONS_FILE, encoding="utf-8") as f:
futures = {executor.submit(_make_raw_logs_query, signoz, token, expr.rstrip("\n")): expr.rstrip("\n") for expr in f}
for future in as_completed(futures):
expr = futures[future]
if future.result().status_code == HTTPStatus.INTERNAL_SERVER_ERROR:
failures.append(expr)
assert len(failures) <= 0, f"{len(failures)} expression(s) caused HTTP 500:\n" + "\n".join(f" {expr!r}" for expr in failures)