Compare commits

..

1 Commits

Author SHA1 Message Date
Vinícius Lourenço
887c629501 refactor(sentry-config): disable tracing & ensure correct environment 2026-06-02 16:27:15 -03:00
76 changed files with 1677 additions and 3235 deletions

19
.github/CODEOWNERS vendored
View File

@@ -169,22 +169,3 @@ go.mod @therealpandey
## Dashboard V2
/frontend/src/pages/DashboardPageV2/ @SigNoz/pulse-frontend
/frontend/src/pages/DashboardsListPageV2/ @SigNoz/pulse-frontend
## Infrastructure Monitoring
/frontend/src/pages/InfrastructureMonitoring/ @SigNoz/pulse-frontend
/frontend/src/container/InfraMonitoringHosts/ @SigNoz/pulse-frontend
/frontend/src/container/InfraMonitoringK8s/ @SigNoz/pulse-frontend
## Alerts
/frontend/src/pages/AlertList/ @SigNoz/pulse-frontend
/frontend/src/pages/AlertDetails/ @SigNoz/pulse-frontend
/frontend/src/pages/CreateAlert/ @SigNoz/pulse-frontend
/frontend/src/pages/EditRules/ @SigNoz/pulse-frontend
/frontend/src/container/AlertHistory/ @SigNoz/pulse-frontend
/frontend/src/container/CreateAlertRule/ @SigNoz/pulse-frontend
/frontend/src/container/CreateAlertV2/ @SigNoz/pulse-frontend
/frontend/src/container/EditAlertV2/ @SigNoz/pulse-frontend
/frontend/src/container/FormAlertRules/ @SigNoz/pulse-frontend
/frontend/src/container/ListAlertRules/ @SigNoz/pulse-frontend
/frontend/src/container/TriggeredAlerts/ @SigNoz/pulse-frontend
/frontend/src/container/AnomalyAlertEvaluationView/ @SigNoz/pulse-frontend

View File

@@ -52,7 +52,6 @@ jobs:
- rootuser
- serviceaccount
- querier_json_body
- querier_skip_resource_fingerprint
- ttl
sqlstore-provider:
- postgres

View File

@@ -64,16 +64,16 @@ web:
settings:
posthog:
# Whether to enable PostHog in web.
enabled: false
enabled: true
appcues:
# Whether to enable Appcues in web.
enabled: false
enabled: true
sentry:
# Whether to enable Sentry in web.
enabled: false
enabled: true
pylon:
# Whether to enable Pylon in web.
enabled: false
enabled: true
##################### Cache #####################
cache:

View File

@@ -6572,15 +6572,6 @@ components:
nullable: true
type: array
type: object
SpantypesOtelSpanRef:
properties:
refType:
type: string
spanId:
type: string
traceId:
type: string
type: object
SpantypesPostableSpanMapper:
properties:
config:
@@ -6844,10 +6835,6 @@ components:
type: string
parent_span_id:
type: string
references:
items:
$ref: '#/components/schemas/SpantypesOtelSpanRef'
type: array
resource:
additionalProperties:
type: string
@@ -6873,8 +6860,6 @@ components:
type: string
trace_state:
type: string
required:
- references
type: object
TagtypesPostableTag:
properties:

View File

@@ -0,0 +1,16 @@
export function getCurrentEnvironment():
| 'production'
| 'staging'
| 'self-host' {
const host = document.location.host;
if (host.endsWith('staging.signoz.cloud')) {
return 'staging';
}
if (host.endsWith('signoz.cloud')) {
return 'production';
}
return 'self-host';
}

View File

@@ -43,6 +43,7 @@ import defaultRoutes, {
LIST_LICENSES,
SUPPORT_ROUTE,
} from './routes';
import { getCurrentEnvironment } from 'AppRoutes/getCurrentEnvironment';
function App(): JSX.Element {
const themeConfig = useThemeConfig();
@@ -348,22 +349,19 @@ function App(): JSX.Element {
!isSentryInitialized &&
(window.signozBootData?.settings?.sentry.enabled ?? true)
) {
const sentryEnvironment = getCurrentEnvironment();
Sentry.init({
dsn: process.env.SENTRY_DSN,
tunnel: process.env.TUNNEL_URL,
environment: 'production',
environment: sentryEnvironment,
integrations: [
Sentry.browserTracingIntegration(),
Sentry.replayIntegration({
maskAllText: false,
blockAllMedia: false,
}),
],
// Performance Monitoring
tracesSampleRate: 1.0, // Capture 100% of the transactions
// Set 'tracePropagationTargets' to control for which URLs distributed tracing should be enabled
tracePropagationTargets: [],
// Session Replay
tracesSampleRate: 0, // Ref: https://github.com/SigNoz/platform-pod/issues/2393#issuecomment-4603658055
replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production.
replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur.
beforeSend(event) {

View File

@@ -7768,21 +7768,6 @@ export interface SpantypesGettableTraceAggregationsDTO {
aggregations: SpantypesSpanAggregationResultDTO[];
}
export interface SpantypesOtelSpanRefDTO {
/**
* @type string
*/
refType?: string;
/**
* @type string
*/
spanId?: string;
/**
* @type string
*/
traceId?: string;
}
export type SpantypesWaterfallSpanDTOAttributesAnyOf = {
[key: string]: unknown;
};
@@ -7877,10 +7862,6 @@ export interface SpantypesWaterfallSpanDTO {
* @type string
*/
parent_span_id?: string;
/**
* @type array
*/
references: SpantypesOtelSpanRefDTO[];
/**
* @type object,null
*/

View File

@@ -120,8 +120,7 @@ export const interceptorRejected = async (
!(
response.config.url === '/sessions' && response.config.method === 'delete'
) &&
response.config.url !== '/authz/check' &&
response.config.url !== '/api/v2/reset_password_tokens/verify'
response.config.url !== '/authz/check'
) {
try {
const accessToken = getLocalStorageApi(LOCALSTORAGE.AUTH_TOKEN);

View File

@@ -0,0 +1 @@
<svg width="14" height="14" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#prefix__clip0_4062_7291)" stroke-width="1.167" stroke-linecap="round" stroke-linejoin="round"><path d="M7 12.833A5.833 5.833 0 107 1.167a5.833 5.833 0 000 11.666z" fill="#E5484D" stroke="#E5484D"/><path d="M8.75 5.25l-3.5 3.5M5.25 5.25l3.5 3.5" stroke="#121317"/></g><defs><clipPath id="prefix__clip0_4062_7291"><path fill="#fff" d="M0 0h14v14H0z"/></clipPath></defs></svg>

After

Width:  |  Height:  |  Size: 467 B

View File

@@ -4,7 +4,6 @@ import { Dot } from '@signozhq/icons';
import { Button } from '@signozhq/ui/button';
import { TooltipSimple } from '@signozhq/ui/tooltip';
import Noz from 'components/Noz/Noz';
import { NOZ_TOOLTIP_TITLE } from 'components/Noz/Noz.constants';
import { Popover } from 'antd';
import logEvent from 'api/common/logEvent';
import { AIAssistantEvents } from 'container/AIAssistant/events';
@@ -110,7 +109,7 @@ function HeaderRightSection({
</span>
) : null}
<TooltipSimple title={NOZ_TOOLTIP_TITLE}>
<TooltipSimple title="Noz">
<Button
variant="solid"
color="secondary"

View File

@@ -1,2 +0,0 @@
/** Shared hover copy for every Noz entry point (header, floating trigger, sidebar). */
export const NOZ_TOOLTIP_TITLE = 'Noz, your AI teammate';

View File

@@ -5,7 +5,6 @@ import { TooltipSimple } from '@signozhq/ui/tooltip';
import logEvent from 'api/common/logEvent';
import ROUTES from 'constants/routes';
import Noz from 'components/Noz/Noz';
import { NOZ_TOOLTIP_TITLE } from 'components/Noz/Noz.constants';
import { AIAssistantEvents, AIAssistantOpenSource } from '../events';
import { normalizePage } from '../hooks/useAIAssistantAnalyticsContext';
@@ -43,15 +42,16 @@ export default function AIAssistantTrigger(): JSX.Element | null {
}
return (
<TooltipSimple title={NOZ_TOOLTIP_TITLE}>
<TooltipSimple title="Noz">
<Button
variant="solid"
color="primary"
className={`${styles.trigger} noz-wave`}
onClick={handleOpen}
aria-label="Open Noz"
prefix={<Noz size={24} />}
/>
>
<Noz size={24} />
</Button>
</TooltipSimple>
);
}

View File

@@ -196,11 +196,7 @@ function Footer(): JSX.Element {
</Button>
);
if (alertValidationMessage) {
button = (
<Tooltip title={alertValidationMessage}>
<span>{button}</span>
</Tooltip>
);
button = <Tooltip title={alertValidationMessage}>{button}</Tooltip>;
}
return button;
}, [
@@ -228,11 +224,7 @@ function Footer(): JSX.Element {
</Button>
);
if (alertValidationMessage) {
button = (
<Tooltip title={alertValidationMessage}>
<span>{button}</span>
</Tooltip>
);
button = <Tooltip title={alertValidationMessage}>{button}</Tooltip>;
}
return button;
}, [

View File

@@ -9,6 +9,8 @@ import { useOptionsMenu } from 'container/OptionsMenu';
import { ArrowUp10, Minus } from '@signozhq/icons';
import { DataSource, StringOperators } from 'types/common/queryBuilder';
import QueryStatus from './QueryStatus';
function LogsActionsContainer({
listQuery,
selectedPanelType,
@@ -16,6 +18,10 @@ function LogsActionsContainer({
handleToggleFrequencyChart,
orderBy,
setOrderBy,
isFetching,
isLoading,
isError,
isSuccess,
}: {
listQuery: any;
selectedPanelType: PANEL_TYPES;
@@ -23,6 +29,10 @@ function LogsActionsContainer({
handleToggleFrequencyChart: () => void;
orderBy: string;
setOrderBy: (value: string) => void;
isFetching: boolean;
isLoading: boolean;
isError: boolean;
isSuccess: boolean;
}): JSX.Element {
const { options, config } = useOptionsMenu({
storageKey: LOCALSTORAGE.LOGS_LIST_OPTIONS,
@@ -96,6 +106,17 @@ function LogsActionsContainer({
</div>
</>
)}
{(selectedPanelType === PANEL_TYPES.TIME_SERIES ||
selectedPanelType === PANEL_TYPES.TABLE) && (
<div className="query-stats">
<QueryStatus
loading={isLoading || isFetching}
error={isError}
success={isSuccess}
/>
</div>
)}
</div>
</div>
</div>

View File

@@ -155,6 +155,40 @@
}
}
.query-stats {
display: flex;
align-items: center;
gap: 12px;
align-self: flex-end;
.rows {
color: var(--l2-foreground);
font-family: 'Geist Mono';
font-size: 12px;
font-style: normal;
font-weight: 400;
line-height: 18px; /* 150% */
letter-spacing: 0.36px;
}
.divider {
width: 1px;
height: 14px;
background: var(--l3-background);
}
.time {
color: var(--l2-foreground);
font-family: 'Geist Mono';
font-size: 12px;
font-style: normal;
font-weight: 400;
line-height: 18px; /* 150% */
letter-spacing: 0.36px;
}
}
.ant-btn {
border: none;
}

View File

@@ -0,0 +1,4 @@
.query-status {
display: flex;
align-items: center;
}

View File

@@ -0,0 +1,49 @@
import React, { useMemo } from 'react';
import { Color } from '@signozhq/design-tokens';
import { LoaderCircle, CircleCheck } from '@signozhq/icons';
import { Spin } from 'antd';
import solidXCircleUrl from '@/assets/Icons/solid-x-circle.svg';
import './QueryStatus.styles.scss';
interface IQueryStatusProps {
loading: boolean;
error: boolean;
success: boolean;
}
export default function QueryStatus(
props: IQueryStatusProps,
): React.ReactElement {
const { loading, error, success } = props;
const content = useMemo((): React.ReactElement => {
if (loading) {
return (
<Spin
spinning
size="small"
indicator={<LoaderCircle className="animate-spin" size="md" />}
/>
);
}
if (error) {
return (
<img
src={solidXCircleUrl}
alt="header"
className="error"
style={{ height: '14px', width: '14px' }}
/>
);
}
if (success) {
return (
<CircleCheck className="success" size={14} fill={Color.BG_ROBIN_500} />
);
}
return <div />;
}, [error, loading, success]);
return <div className="query-status">{content}</div>;
}

View File

@@ -160,7 +160,7 @@ function LogsExplorerViewsContainer({
'custom',
);
const { data, isLoading, isFetching, isError, error } =
const { data, isLoading, isFetching, isError, isSuccess, error } =
useGetExplorerQueryRange(
requestData,
selectedPanelType,
@@ -437,6 +437,10 @@ function LogsExplorerViewsContainer({
handleToggleFrequencyChart={handleToggleFrequencyChart}
orderBy={orderBy}
setOrderBy={setOrderBy}
isFetching={isFetching}
isLoading={isLoading}
isError={isError}
isSuccess={isSuccess}
/>
)}

View File

@@ -59,7 +59,7 @@ function AllAttributes({
);
const attributes = useMemo(
() => attributesData?.data?.attributes ?? [],
() => attributesData?.data.attributes ?? [],
[attributesData],
);

View File

@@ -56,7 +56,7 @@ function MetricDetails({
);
const metadata = useMemo(() => {
if (!metricMetadataResponse?.data) {
if (!metricMetadataResponse) {
return null;
}
const { type, description, unit, temporality, isMonotonic } =

View File

@@ -21,10 +21,6 @@
justify-content: center;
margin-bottom: 8px;
color: var(--semantic-primary-foreground);
&--error {
color: var(--destructive);
}
}
.reset-password-header-title {

View File

@@ -1,67 +0,0 @@
import { CircleAlert } from '@signozhq/icons';
import { Typography } from '@signozhq/ui/typography';
import AuthError from 'components/AuthError/AuthError';
import AuthPageContainer from 'components/AuthPageContainer';
import APIError from 'types/api/error';
import './ResetPassword.styles.scss';
interface TokenErrorContent {
title: string;
subtitle: string;
}
function getErrorContent(error?: APIError): TokenErrorContent {
const code = error?.getErrorCode();
if (code === 'reset_password_token_expired') {
return {
title: 'Reset Password token is expired',
subtitle:
'Password reset links are single-use and expire after a set period. Please request a new password reset link.',
};
}
if (code === 'reset_password_token_not_found') {
return {
title: 'Invalid Reset Link',
subtitle:
'This reset password link is invalid or has already been used. Please request a new password reset link.',
};
}
return {
title: 'Reset Link Unavailable',
subtitle:
'We could not validate your reset password link. Please request a new one.',
};
}
interface TokenErrorProps {
error?: APIError;
}
function TokenError({ error }: TokenErrorProps): JSX.Element {
const { title, subtitle } = getErrorContent(error);
return (
<AuthPageContainer>
<div className="reset-password-card reset-password-card--centered">
<div className="reset-password-header">
<div className="reset-password-header-icon reset-password-header-icon--error">
<CircleAlert size={32} />
</div>
<Typography.Title level={4} className="reset-password-header-title">
{title}
</Typography.Title>
<Typography.Text className="reset-password-header-subtitle">
{subtitle}
</Typography.Text>
</div>
{error && <AuthError error={error} />}
</div>
</AuthPageContainer>
);
}
export default TokenError;

View File

@@ -1,3 +1,4 @@
import { Logout } from 'api/utils';
import ROUTES from 'constants/routes';
import history from 'lib/history';
import { rest, server } from 'mocks-server/server';
@@ -16,6 +17,10 @@ jest.mock('lib/history', () => ({
},
}));
jest.mock('api/utils', () => ({
Logout: jest.fn(),
}));
const mockSuccessNotification = jest.fn();
const mockErrorNotification = jest.fn();
@@ -65,6 +70,17 @@ describe('ResetPassword Component', () => {
).toBeInTheDocument();
expect(screen.getByText(/signoz 1\.0\.0/i)).toBeInTheDocument();
});
it('redirects to login when token is missing', () => {
window.history.pushState({}, '', '/password-reset');
render(<ResetPassword version="1.0.0" />, undefined, {
initialRoute: '/password-reset',
});
expect(Logout).toHaveBeenCalled();
expect(mockHistoryPush).toHaveBeenCalledWith(ROUTES.LOGIN);
});
});
describe('Form Validation', () => {

View File

@@ -1,10 +1,11 @@
import { useState } from 'react';
import { useEffect, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useLocation } from 'react-use';
import { Button } from '@signozhq/ui/button';
import { Callout } from '@signozhq/ui/callout';
import { Form, Input as AntdInput } from 'antd';
import { Typography } from '@signozhq/ui/typography';
import { Logout } from 'api/utils';
import resetPasswordApi from 'api/v1/factor_password/resetPassword';
import AuthError from 'components/AuthError/AuthError';
import AuthPageContainer from 'components/AuthPageContainer';
@@ -37,6 +38,13 @@ function ResetPassword({ version }: ResetPasswordProps): JSX.Element {
const { notifications } = useNotifications();
const [form] = Form.useForm<FormValues>();
useEffect(() => {
if (!token) {
Logout();
history.push(ROUTES.LOGIN);
}
}, [token]);
const handleFormSubmit: () => Promise<void> = async () => {
try {
setLoading(true);

View File

@@ -5,6 +5,7 @@ import { Pin, PinOff } from '@signozhq/icons';
import { SidebarItem } from '../sideNav.types';
import './NavItem.styles.scss';
import './NavItem.styles.scss';
export default function NavItem({
@@ -26,7 +27,7 @@ export default function NavItem({
showIcon?: boolean;
dataTestId?: string;
}): JSX.Element {
const { label, icon, isBeta, isNew, isEarlyAccess, tooltip } = item;
const { label, icon, isBeta, isNew, isEarlyAccess } = item;
const handleTogglePinClick = (
event: React.MouseEvent<SVGSVGElement, MouseEvent>,
@@ -35,7 +36,7 @@ export default function NavItem({
onTogglePin?.(item);
};
const navItem = (
return (
<div
className={cx(
'nav-item',
@@ -106,15 +107,6 @@ export default function NavItem({
</div>
</div>
);
// Only non-pinnable items set `tooltip`; it would nest with the pin tooltip.
return tooltip ? (
<Tooltip title={tooltip} placement="right">
{navItem}
</Tooltip>
) : (
navItem
);
}
NavItem.defaultProps = {

View File

@@ -45,7 +45,6 @@ import {
} from './sideNav.types';
import { Style } from '@signozhq/design-tokens';
import Noz from 'components/Noz/Noz';
import { NOZ_TOOLTIP_TITLE } from 'components/Noz/Noz.constants';
export const getStartedMenuItem = {
key: ROUTES.GET_STARTED,
@@ -98,7 +97,6 @@ export const aiAssistantMenuItem = {
icon: <Noz size={16} />,
itemKey: 'ai-assistant',
isEarlyAccess: true,
tooltip: NOZ_TOOLTIP_TITLE,
};
export const shortcutMenuItem = {

View File

@@ -15,8 +15,6 @@ export interface SidebarItem {
isBeta?: boolean;
isNew?: boolean;
isEarlyAccess?: boolean;
/** Hover copy for the whole item row (e.g. Noz's early-access tagline). */
tooltip?: ReactNode;
isPinned?: boolean;
children?: SidebarItem[];
isExternal?: boolean;

View File

@@ -1,155 +0,0 @@
import { Logout } from 'api/utils';
import ROUTES from 'constants/routes';
import history from 'lib/history';
import { createErrorResponse, rest, server } from 'mocks-server/server';
import { render, screen, waitFor } from 'tests/test-utils';
import ResetPassword from '../index';
jest.mock('lib/history', () => ({
__esModule: true,
default: {
push: jest.fn(),
location: { search: '' },
},
}));
jest.mock('api/utils', () => ({
Logout: jest.fn().mockResolvedValue(undefined),
}));
const VERIFY_TOKEN_ENDPOINT = '*/api/v2/reset_password_tokens/verify';
const VERSION_ENDPOINT = '*/version';
const mockHistoryPush = history.push as jest.MockedFunction<
typeof history.push
>;
const successVerifyResponse = {
data: { id: 'token-id', token: 'valid-token' },
};
const successVersionResponse = {
version: '0.0.1',
ee: 'Y',
setupCompleted: true,
};
describe('ResetPassword Page', () => {
beforeEach(() => {
jest.clearAllMocks();
server.use(
rest.get(VERSION_ENDPOINT, (_, res, ctx) =>
res(ctx.status(200), ctx.json(successVersionResponse)),
),
);
});
afterEach(() => {
server.resetHandlers();
});
describe('Token validation on page load', () => {
it('shows spinner then form when token is valid', async () => {
server.use(
rest.post(VERIFY_TOKEN_ENDPOINT, (_, res, ctx) =>
res(ctx.delay(50), ctx.status(200), ctx.json(successVerifyResponse)),
),
);
window.history.pushState({}, '', '/password-reset?token=valid-token');
render(<ResetPassword />, undefined, {
initialRoute: '/password-reset?token=valid-token',
});
// Loading state: spinner visible, form and error absent
expect(screen.getByRole('img', { name: /loading/i })).toBeInTheDocument();
expect(screen.queryByTestId('password')).not.toBeInTheDocument();
expect(
screen.queryByText(/reset password token is expired/i),
).not.toBeInTheDocument();
// After verification resolves: form is shown
await waitFor(() => {
expect(screen.getByTestId('password')).toBeInTheDocument();
});
expect(screen.getByTestId('confirmPassword')).toBeInTheDocument();
});
it('shows "Invalid Reset Link" when token is not found (404)', async () => {
server.use(
rest.post(
VERIFY_TOKEN_ENDPOINT,
createErrorResponse(
404,
'reset_password_token_not_found',
'reset password token does not exist',
),
),
);
window.history.pushState({}, '', '/password-reset?token=invalid-token');
render(<ResetPassword />, undefined, {
initialRoute: '/password-reset?token=invalid-token',
});
await waitFor(() => {
expect(screen.getByText(/invalid reset link/i)).toBeInTheDocument();
});
expect(
screen.getByText(/invalid or has already been used/i),
).toBeInTheDocument();
expect(
screen.getByText(/reset password token does not exist/i),
).toBeInTheDocument();
});
it('shows "token is expired" when token is expired (401) without redirecting to login', async () => {
server.use(
rest.post(
VERIFY_TOKEN_ENDPOINT,
createErrorResponse(
401,
'reset_password_token_expired',
'reset password token has expired',
),
),
);
window.history.pushState({}, '', '/password-reset?token=expired-token');
render(<ResetPassword />, undefined, {
initialRoute: '/password-reset?token=expired-token',
});
await waitFor(() => {
expect(
screen.getByText(/reset password token is expired/i),
).toBeInTheDocument();
});
expect(
screen.getByText(/single-use and expire after a set period/i),
).toBeInTheDocument();
expect(
screen.getByText(/reset password token has expired/i),
).toBeInTheDocument();
// 401 from this endpoint must NOT trigger logout/redirect
expect(mockHistoryPush).not.toHaveBeenCalledWith(ROUTES.LOGIN);
expect(Logout).not.toHaveBeenCalled();
});
it('redirects to login when no token is in the URL', async () => {
window.history.pushState({}, '', '/password-reset');
render(<ResetPassword />, undefined, {
initialRoute: '/password-reset',
});
await waitFor(() => {
expect(mockHistoryPush).toHaveBeenCalledWith(ROUTES.LOGIN);
});
expect(Logout).toHaveBeenCalled();
});
});
});

View File

@@ -1,17 +1,8 @@
import { useEffect, useMemo } from 'react';
import { useEffect } from 'react';
import { useQuery } from 'react-query';
import { useLocation } from 'react-use';
import { AxiosError } from 'axios';
import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
import getUserVersion from 'api/v1/version/get';
import { verifyResetPasswordToken } from 'api/generated/services/users';
import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
import { Logout } from 'api/utils';
import Spinner from 'components/Spinner';
import ResetPasswordContainer from 'container/ResetPassword';
import TokenError from 'container/ResetPassword/TokenError';
import ROUTES from 'constants/routes';
import history from 'lib/history';
import { useAppContext } from 'providers/App/App';
import { useErrorModal } from 'providers/ErrorModalProvider';
import APIError from 'types/api/error';
@@ -19,65 +10,24 @@ import APIError from 'types/api/error';
function ResetPassword(): JSX.Element {
const { user, isLoggedIn } = useAppContext();
const { showErrorModal } = useErrorModal();
const { search } = useLocation();
const params = new URLSearchParams(search || '');
const token = params.get('token') || '';
useEffect(() => {
if (!token) {
void Logout();
history.push(ROUTES.LOGIN);
}
}, [token]);
const {
data: versionData,
isLoading: isVersionLoading,
error: versionError,
} = useQuery({
const { data, isLoading, error } = useQuery({
queryFn: getUserVersion,
queryKey: ['getUserVersion', user?.accessJwt],
enabled: !isLoggedIn,
});
const {
isLoading: isVerifying,
isError: isTokenError,
error: tokenError,
} = useQuery<
Awaited<ReturnType<typeof verifyResetPasswordToken>>,
AxiosError<RenderErrorResponseDTO>
>({
queryFn: () => verifyResetPasswordToken({ token }),
queryKey: ['verifyResetPasswordToken', token],
enabled: !!token,
retry: false,
});
const tokenApiError = useMemo(
() => convertToApiError(tokenError),
[tokenError],
);
useEffect(() => {
if (versionError) {
showErrorModal(versionError as APIError);
if (error) {
showErrorModal(error as APIError);
}
}, [versionError, showErrorModal]);
}, [error, showErrorModal]);
if (!token) {
if (isLoading) {
return <Spinner tip="Loading..." />;
}
if (isVersionLoading || isVerifying) {
return <Spinner tip="Validating your reset password token..." />;
}
if (isTokenError) {
return <TokenError error={tokenApiError} />;
}
return <ResetPasswordContainer version={versionData?.data.version || ''} />;
return <ResetPasswordContainer version={data?.data.version || ''} />;
}
export default ResetPassword;

View File

@@ -22,7 +22,6 @@ import styles from './AnalyticsPanel.module.scss';
interface AnalyticsPanelProps {
isOpen: boolean;
onClose: () => void;
onTabChange: (tab: string) => void;
}
const PANEL_WIDTH = 350;
@@ -33,7 +32,6 @@ const PANEL_MARGIN_BOTTOM = 50;
function AnalyticsPanel({
isOpen,
onClose,
onTabChange,
}: AnalyticsPanelProps): JSX.Element | null {
const aggregations = useTraceStore((s) => s.aggregations);
const colorByFieldName = useTraceStore((s) => s.colorByField.name);
@@ -120,7 +118,7 @@ function AnalyticsPanel({
/>
<div className={styles.body}>
<TabsRoot defaultValue="exec-time" onValueChange={onTabChange}>
<TabsRoot defaultValue="exec-time">
<TabsList variant="secondary">
<TabsTrigger value="exec-time" variant="secondary">
% exec time

View File

@@ -31,12 +31,7 @@ import Events from 'container/SpanDetailsDrawer/Events/Events';
import SpanLogs from 'container/SpanDetailsDrawer/SpanLogs/SpanLogs';
import { useSpanContextLogs } from 'container/SpanDetailsDrawer/SpanLogs/useSpanContextLogs';
import dayjs from 'dayjs';
import {
TraceDetailEventKeys,
TraceDetailEvents,
} from 'pages/TraceDetailsV3/events';
import { useMigratePinnedAttributes } from 'pages/TraceDetailsV3/hooks/useMigratePinnedAttributes';
import { useTraceDetailLogEvent } from 'pages/TraceDetailsV3/hooks/useTraceDetailLogEvent';
import {
getSpanAttribute,
getSpanDisplayData,
@@ -91,16 +86,6 @@ function SpanDetailsContent({
}): JSX.Element {
const FIVE_MINUTES_IN_MS = 5 * 60 * 1000;
const spanAttributeActions = useSpanAttributeActions();
const logTraceEvent = useTraceDetailLogEvent('v3', selectedSpan.trace_id);
const handleTabChange = useCallback(
(tab: string): void => {
logTraceEvent(TraceDetailEvents.SpanPanelTabChanged, {
[TraceDetailEventKeys.Tab]: tab,
[TraceDetailEventKeys.SpanId]: selectedSpan.span_id,
});
},
[logTraceEvent, selectedSpan.span_id],
);
const percentile = useSpanPercentile(selectedSpan);
const linkedSpans = useLinkedSpans((selectedSpan as any).references);
@@ -391,7 +376,7 @@ function SpanDetailsContent({
<div className={styles.tabsSection}>
{/* Step 9: ContentTabs */}
<TabsRoot defaultValue="overview" onValueChange={handleTabChange}>
<TabsRoot defaultValue="overview">
<TabsList variant="secondary">
<TabsTrigger value="overview" variant="secondary">
<Bookmark size={14} /> Overview

View File

@@ -1,4 +1,4 @@
import { useCallback, useRef, useState } from 'react';
import { useCallback, useState } from 'react';
import { useParams } from 'react-router-dom';
import { Button } from '@signozhq/ui/button';
import {
@@ -29,8 +29,6 @@ import KeyValueLabel from 'periscope/components/KeyValueLabel';
import { TraceDetailV2URLProps } from 'types/api/trace/getTraceV2';
import { DataSource } from 'types/common/queryBuilder';
import { TraceDetailEventKeys, TraceDetailEvents } from '../events';
import { useTraceDetailLogEvent } from '../hooks/useTraceDetailLogEvent';
import { useTraceStore } from '../stores/traceStore';
import AnalyticsPanel from '../SpanDetailsPanel/AnalyticsPanel/AnalyticsPanel';
import Filters from '../TraceWaterfall/TraceWaterfallStates/Success/Filters/Filters';
@@ -92,35 +90,11 @@ function TraceDetailsHeader({
const previewFields = useTraceStore((s) => s.previewFields);
const setPreviewFields = useTraceStore((s) => s.setPreviewFields);
const logTraceEvent = useTraceDetailLogEvent('v3', traceID || '');
const pageLoadedAtRef = useRef(Date.now());
const handleSwitchToOldView = useCallback((): void => {
logTraceEvent(TraceDetailEvents.ViewSwitched, {
[TraceDetailEventKeys.From]: 'v3',
[TraceDetailEventKeys.To]: 'v2',
[TraceDetailEventKeys.DwellMs]: Date.now() - pageLoadedAtRef.current,
});
setLocalStorageKey(LOCALSTORAGE.TRACE_DETAILS_PREFER_OLD_VIEW, 'true');
const oldUrl = `/trace-old/${traceID}${window.location.search}`;
history.replace(oldUrl);
}, [traceID, logTraceEvent]);
const handleToggleAnalytics = useCallback((): void => {
logTraceEvent(TraceDetailEvents.AnalyticsPanelToggled, {
[TraceDetailEventKeys.Open]: !isAnalyticsOpen,
});
setIsAnalyticsOpen((prev) => !prev);
}, [logTraceEvent, isAnalyticsOpen]);
const handleAnalyticsTabChange = useCallback(
(tab: string): void => {
logTraceEvent(TraceDetailEvents.AnalyticsTabChanged, {
[TraceDetailEventKeys.Tab]: tab,
});
},
[logTraceEvent],
);
}, [traceID]);
const handlePreviousBtnClick = useCallback((): void => {
if (hasInAppHistory()) {
@@ -193,7 +167,7 @@ function TraceDetailsHeader({
size="icon"
color="secondary"
aria-label="Analytics"
onClick={handleToggleAnalytics}
onClick={(): void => setIsAnalyticsOpen((prev) => !prev)}
>
<ChartPie size={14} />
</Button>
@@ -271,7 +245,6 @@ function TraceDetailsHeader({
<AnalyticsPanel
isOpen={isAnalyticsOpen}
onClose={(): void => setIsAnalyticsOpen(false)}
onTabChange={handleAnalyticsTabChange}
/>
</div>
);

View File

@@ -1,38 +0,0 @@
export enum TraceDetailEvents {
DataLoaded = 'Trace Detail: Data loaded',
ViewSwitched = 'Trace Detail: View switched',
FlameGraphToggled = 'Trace Detail: Flame graph toggled',
WaterfallToggled = 'Trace Detail: Waterfall toggled',
AnalyticsPanelToggled = 'Trace Detail: Analytics panel toggled',
AnalyticsTabChanged = 'Trace Detail: Analytics tab changed',
SpanPanelTabChanged = 'Trace Detail: Span panel tab changed',
}
export enum TraceDetailEventKeys {
// Injected on every event by useTraceDetailLogEvent
View = 'view',
TraceId = 'traceId',
// Data loaded — trace shape
TotalSpansCount = 'totalSpansCount',
NumServices = 'numServices',
TraceDurationMs = 'traceDurationMs',
HadErrors = 'hadErrors',
FlamegraphSampled = 'flamegraphSampled',
// Data loaded — persisted settings
SpanPanelVariant = 'spanPanelVariant',
ColorByField = 'colorByField',
PreviewFieldsCount = 'previewFieldsCount',
EntryPreferOldView = 'entryPreferOldView',
// View switched
From = 'from',
To = 'to',
DwellMs = 'dwellMs',
// Toggles / tabs
Expanded = 'expanded',
Open = 'open',
Tab = 'tab',
// Span panel tab changed
SpanId = 'spanId',
}
export type TraceDetailView = 'v2' | 'v3';

View File

@@ -1,88 +0,0 @@
import { act, renderHook } from '@testing-library/react';
import { TraceDetailEvents } from '../../events';
import { useTraceDetailLogEvent } from '../useTraceDetailLogEvent';
const logEventMock = jest.fn();
jest.mock('api/common/logEvent', () => ({
__esModule: true,
default: (...args: unknown[]): void => logEventMock(...args),
}));
describe('useTraceDetailLogEvent', () => {
beforeEach(() => {
logEventMock.mockClear();
});
it('injects view and traceId on every event', () => {
const { result } = renderHook(() =>
useTraceDetailLogEvent('v3', 'trace-123'),
);
act(() => {
result.current(TraceDetailEvents.DataLoaded, { totalSpansCount: 42 });
});
expect(logEventMock).toHaveBeenCalledTimes(1);
expect(logEventMock).toHaveBeenCalledWith(TraceDetailEvents.DataLoaded, {
view: 'v3',
traceId: 'trace-123',
totalSpansCount: 42,
});
});
it('injects view and traceId even when no attributes are passed', () => {
const { result } = renderHook(() =>
useTraceDetailLogEvent('v2', 'trace-456'),
);
act(() => {
result.current(TraceDetailEvents.ViewSwitched);
});
expect(logEventMock).toHaveBeenCalledWith(TraceDetailEvents.ViewSwitched, {
view: 'v2',
traceId: 'trace-456',
});
});
it('keeps a stable callback identity and emits the latest traceId', () => {
const { result, rerender } = renderHook(
({ traceId }) => useTraceDetailLogEvent('v3', traceId),
{ initialProps: { traceId: 'trace-1' } },
);
const firstIdentity = result.current;
rerender({ traceId: 'trace-2' });
expect(result.current).toBe(firstIdentity);
act(() => {
result.current(TraceDetailEvents.SpanPanelTabChanged, { spanId: 's1' });
});
expect(logEventMock).toHaveBeenCalledWith(
TraceDetailEvents.SpanPanelTabChanged,
{
view: 'v3',
traceId: 'trace-2',
spanId: 's1',
},
);
});
it('never throws if logEvent throws (analytics must not break the UI)', () => {
logEventMock.mockImplementationOnce(() => {
throw new Error('network down');
});
const { result } = renderHook(() =>
useTraceDetailLogEvent('v3', 'trace-123'),
);
expect(() => {
act(() => {
result.current(TraceDetailEvents.DataLoaded);
});
}).not.toThrow();
});
});

View File

@@ -1,39 +0,0 @@
import { useCallback, useRef } from 'react';
import logEvent from 'api/common/logEvent';
import {
TraceDetailEventKeys,
TraceDetailEvents,
TraceDetailView,
} from '../events';
export type TraceDetailLogEvent = (
event: TraceDetailEvents,
attributes?: Record<string, unknown>,
) => void;
export function useTraceDetailLogEvent(
view: TraceDetailView,
traceId: string,
): TraceDetailLogEvent {
const contextRef = useRef({ view, traceId });
contextRef.current = { view, traceId };
return useCallback(
(
event: TraceDetailEvents,
attributes: Record<string, unknown> = {},
): void => {
try {
void logEvent(event, {
[TraceDetailEventKeys.View]: contextRef.current.view,
[TraceDetailEventKeys.TraceId]: contextRef.current.traceId,
...attributes,
});
} catch {
// No-op. Logging must never throw into the UI.
}
},
[],
);
}

View File

@@ -20,10 +20,7 @@ import {
} from 'types/api/trace/getTraceV3';
import { COLOR_BY_FIELDS } from './constants';
import { TraceDetailEventKeys, TraceDetailEvents } from './events';
import { useTraceDetailLogEvent } from './hooks/useTraceDetailLogEvent';
import TraceStoreSync from './stores/TraceStoreSync';
import { useTraceStore } from './stores/traceStore';
import { AGGREGATIONS } from './utils/aggregations';
import { SpanDetailVariant } from './SpanDetailsPanel/constants';
import SpanDetailsPanel from './SpanDetailsPanel/SpanDetailsPanel';
@@ -59,14 +56,6 @@ function TraceDetailsV3(): JSX.Element {
const selectedSpanId = urlQuery.get('spanId') || undefined;
const { safeNavigate } = useSafeNavigate();
const logTraceEvent = useTraceDetailLogEvent('v3', traceId || '');
// Tracks which traceId the load event already fired for, so navigating
// between traces (the route component stays mounted) re-fires it once each.
const dataLoadedFiredForRef = useRef('');
const colorByField = useTraceStore((s) => s.colorByField);
const previewFieldsCount = useTraceStore((s) => s.previewFields.length);
const userPrefsReady = useTraceStore((s) => s.userPreferences !== null);
const handleSpanDetailsClose = useCallback((): void => {
urlQuery.delete('spanId');
safeNavigate({ search: urlQuery.toString() });
@@ -165,46 +154,6 @@ function TraceDetailsV3(): JSX.Element {
allSpansRef.current = allSpans;
}, [allSpans]);
useEffect(() => {
if (
!traceId ||
dataLoadedFiredForRef.current === traceId ||
!userPrefsReady
) {
return;
}
const payload = traceData?.payload;
if (!payload?.spans?.length) {
return;
}
dataLoadedFiredForRef.current = traceId;
const numServices = new Set(payload.spans.map((s) => s['service.name'])).size;
logTraceEvent(TraceDetailEvents.DataLoaded, {
[TraceDetailEventKeys.TotalSpansCount]: totalSpansCount,
[TraceDetailEventKeys.NumServices]: numServices,
[TraceDetailEventKeys.TraceDurationMs]:
payload.endTimestampMillis - payload.startTimestampMillis,
[TraceDetailEventKeys.HadErrors]: (payload.totalErrorSpansCount || 0) > 0,
[TraceDetailEventKeys.FlamegraphSampled]:
totalSpansCount > FLAMEGRAPH_SPAN_LIMIT,
[TraceDetailEventKeys.SpanPanelVariant]:
getLocalStorageKey(LOCALSTORAGE.TRACE_DETAILS_SPAN_DETAILS_POSITION) ||
SpanDetailVariant.DOCKED_RIGHT,
[TraceDetailEventKeys.ColorByField]: colorByField.name,
[TraceDetailEventKeys.PreviewFieldsCount]: previewFieldsCount,
[TraceDetailEventKeys.EntryPreferOldView]:
getLocalStorageKey(LOCALSTORAGE.TRACE_DETAILS_PREFER_OLD_VIEW) === 'true',
});
}, [
traceId,
userPrefsReady,
traceData,
totalSpansCount,
colorByField,
previewFieldsCount,
logTraceEvent,
]);
// Frontend mode: expand all parents by default when full data arrives
useEffect(() => {
if (isFullDataLoaded && allSpans.length > 0) {
@@ -284,12 +233,6 @@ function TraceDetailsV3(): JSX.Element {
const [activeKeys, setActiveKeys] = useState<string[]>(['flame', 'waterfall']);
const handleCollapseChange = (key: string): void => {
logTraceEvent(
key === 'flame'
? TraceDetailEvents.FlameGraphToggled
: TraceDetailEvents.WaterfallToggled,
{ [TraceDetailEventKeys.Expanded]: !activeKeys.includes(key) },
);
setActiveKeys((prev) =>
prev.includes(key) ? prev.filter((k) => k !== key) : [...prev, key],
);

View File

@@ -4,7 +4,6 @@ import { ChevronDown, Copy } from '@signozhq/icons';
import { Button } from '@signozhq/ui/button';
import { DropdownMenuSimple as Dropdown } from '@signozhq/ui/dropdown-menu';
import { toast } from '@signozhq/ui/sonner';
import logEvent from 'api/common/logEvent';
import { JsonView } from 'periscope/components/JsonView';
import { PrettyView } from 'periscope/components/PrettyView';
import { PrettyViewProps } from 'periscope/components/PrettyView';
@@ -13,8 +12,6 @@ import './DataViewer.styles.scss';
type ViewMode = 'pretty' | 'json';
const VIEW_MODE_CHANGED_EVENT = 'Data Viewer: View mode changed';
const VIEW_MODE_OPTIONS: { label: string; value: ViewMode }[] = [
{ label: 'Pretty', value: 'pretty' },
{ label: 'JSON', value: 'json' },
@@ -37,20 +34,6 @@ function DataViewer({
const jsonString = useMemo(() => JSON.stringify(data, null, 2), [data]);
const handleViewModeChange = (value: string): void => {
const next = value as ViewMode;
setViewMode(next);
try {
logEvent(VIEW_MODE_CHANGED_EVENT, {
viewMode: next,
path: window.location.pathname,
drawerKey,
});
} catch {
// No op
}
};
const handleCopy = (): void => {
const text = JSON.stringify(data, null, 2);
setCopy(text);
@@ -73,7 +56,7 @@ function DataViewer({
{
type: 'radio-group',
value: viewMode,
onChange: handleViewModeChange,
onChange: (value): void => setViewMode(value as ViewMode),
children: VIEW_MODE_OPTIONS.map((opt) => ({
type: 'radio',
key: opt.value,

View File

@@ -74,7 +74,7 @@ func (s *traceStore) GetTraceSpans(ctx context.Context, traceID string, summary
events, status_message, status_code_string, kind_string, parent_span_id,
flags, is_remote, trace_state, status_code,
db_name, db_operation, http_method, http_url, http_host,
external_http_method, external_http_url, response_status_code, links as references
external_http_method, external_http_url, response_status_code
FROM %s.%s
WHERE trace_id=? AND ts_bucket_start>=? AND ts_bucket_start<=?
ORDER BY timestamp ASC, name ASC`,
@@ -130,7 +130,7 @@ func (s *traceStore) GetTraceSpansByIDs(ctx context.Context, traceID string, sta
"events", "status_message", "status_code_string", "kind_string", "parent_span_id",
"flags", "is_remote", "trace_state", "status_code",
"db_name", "db_operation", "http_method", "http_url", "http_host",
"external_http_method", "external_http_url", "response_status_code", "links as references",
"external_http_method", "external_http_url", "response_status_code",
)
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
ids := make([]any, len(spanIDs))

View File

@@ -7,12 +7,6 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
)
type SkipResourceFingerprint struct {
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
// If count of fingerprint is above threshold, skip the fingerprint subquery and filter on main table instead.
Threshold uint64 `yaml:"threshold" mapstructure:"threshold"`
}
// Config represents the configuration for the querier.
type Config struct {
// CacheTTL is the TTL for cached query results
@@ -21,8 +15,6 @@ type Config struct {
FluxInterval time.Duration `yaml:"flux_interval" mapstructure:"flux_interval"`
// MaxConcurrentQueries is the maximum number of concurrent queries for missing ranges
MaxConcurrentQueries int `yaml:"max_concurrent_queries" mapstructure:"max_concurrent_queries"`
// SkipResourceFingerprint configures when the resource fingerprint subquery is skipped in favor of main-table filtering.
SkipResourceFingerprint SkipResourceFingerprint `yaml:"skip_resource_fingerprint" mapstructure:"skip_resource_fingerprint"`
}
// NewConfigFactory creates a new config factory for querier.
@@ -36,10 +28,6 @@ func newConfig() factory.Config {
CacheTTL: 168 * time.Hour,
FluxInterval: 5 * time.Minute,
MaxConcurrentQueries: 4,
SkipResourceFingerprint: SkipResourceFingerprint{
Enabled: false,
Threshold: 100000,
},
}
}
@@ -54,9 +42,6 @@ func (c Config) Validate() error {
if c.MaxConcurrentQueries <= 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_concurrent_queries must be positive, got %v", c.MaxConcurrentQueries)
}
if c.SkipResourceFingerprint.Enabled && c.SkipResourceFingerprint.Threshold == 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "skip_resource_fingerprint.threshold must be > 0 when enabled")
}
return nil
}

View File

@@ -88,8 +88,6 @@ func newProvider(
traceAggExprRewriter,
telemetryStore,
flagger,
cfg.SkipResourceFingerprint.Enabled,
cfg.SkipResourceFingerprint.Threshold,
)
// Create trace operator statement builder
@@ -123,9 +121,6 @@ func newProvider(
telemetrylogs.DefaultFullTextColumn,
telemetrylogs.GetBodyJSONKey,
flagger,
telemetryStore,
cfg.SkipResourceFingerprint.Enabled,
cfg.SkipResourceFingerprint.Threshold,
)
// Create audit statement builder

View File

@@ -89,9 +89,6 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
telemetrylogs.DefaultFullTextColumn,
telemetrylogs.GetBodyJSONKey,
fl,
nil,
false,
100000,
)
return querier.New(
@@ -137,8 +134,6 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
traceAggExprRewriter,
telemetryStore,
fl,
false,
100000,
)
return querier.New(

View File

@@ -186,7 +186,7 @@ func (c *conditionBuilder) conditionFor(
column := columns[0]
if len(key.Evolutions) > 0 {
// we will use the corresponding column and its evolution entry for the query
newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
newColumns, _, err := selectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
if err != nil {
return "", err
}

View File

@@ -3,7 +3,11 @@ package telemetrylogs
import (
"context"
"fmt"
"slices"
"sort"
"strconv"
"strings"
"time"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz-otel-collector/utils"
@@ -133,6 +137,113 @@ func (m *fieldMapper) getColumn(ctx context.Context, key *telemetrytypes.Telemet
return nil, qbtypes.ErrColumnNotFound
}
// selectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
// Logic:
// - Finds the latest base evolution (<= tsStartTime) across ALL columns
// - Rejects all evolutions before this latest base evolution
// - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
// - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
// - Results are sorted by ReleaseTime descending (newest first)
func selectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
copy(sortedEvolutions, evolutions)
// sort the evolutions by ReleaseTime ascending
sort.Slice(sortedEvolutions, func(i, j int) bool {
return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
})
tsStartTime := time.Unix(0, int64(tsStart))
tsEndTime := time.Unix(0, int64(tsEnd))
// Build evolution map: column name -> evolution
evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
for _, evolution := range sortedEvolutions {
if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
// since if there is duplicate we would just use the oldest one.
continue
}
evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
}
// Find the latest base evolution (<= tsStartTime) across ALL columns
// Evolutions are sorted, so we can break early
var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
for _, evolution := range sortedEvolutions {
if evolution.ReleaseTime.After(tsStartTime) {
break
}
latestBaseEvolutionAcrossAll = evolution
}
// We shouldn't reach this, it basically means there is something wrong with the evolutions data
if latestBaseEvolutionAcrossAll == nil {
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
}
columnLookUpMap := make(map[string]*schema.Column)
for _, column := range columns {
columnLookUpMap[column.Name] = column
}
// Collect column-evolution pairs
type colEvoPair struct {
column *schema.Column
evolution *telemetrytypes.EvolutionEntry
}
pairs := []colEvoPair{}
for _, evolution := range evolutionMap {
// Reject evolutions before the latest base evolution
if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
continue
}
// skip evolutions after tsEndTime
if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
continue
}
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
}
pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
}
// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
if len(pairs) == 0 {
for _, column := range columns {
// Use latestBaseEvolutionAcrossAll if this column name matches its column name
if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
}
}
}
// Sort by ReleaseTime descending (newest first)
slices.SortFunc(pairs, func(a, b colEvoPair) int {
// Sort by ReleaseTime descending (newest first)
if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
return -1
}
if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
return 1
}
return 0
})
// Extract results
newColumns := make([]*schema.Column, len(pairs))
evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
for i, pair := range pairs {
newColumns[i] = pair.column
evolutionsEntries[i] = pair.evolution
}
return newColumns, evolutionsEntries, nil
}
func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
columns, err := m.getColumn(ctx, key)
if err != nil {
@@ -143,7 +254,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *
var evolutionsEntries []*telemetrytypes.EvolutionEntry
if len(key.Evolutions) > 0 {
// we will use the corresponding column and its evolution entry for the query
newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
newColumns, evolutionsEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
if err != nil {
return "", err
}

View File

@@ -536,6 +536,390 @@ func TestFieldForWithEvolutions(t *testing.T) {
}
}
func TestSelectEvolutionsForColumns(t *testing.T) {
testCases := []struct {
name string
columns []*schema.Column
evolutions []*telemetrytypes.EvolutionEntry
tsStart uint64
tsEnd uint64
expectedColumns []string // column names
expectedEvols []string // evolution column names
expectedError bool
errorStr string
}{
{
name: "New evolutions at tsStartTime - should include latest evolution",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resource"},
expectedEvols: []string{"resource"},
},
{
name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
expectedEvols: []string{"resource", "resources_string"},
},
{
name: "Columns without matching evolutions - should exclude them",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"], // no evolution for this
logsV2Columns["attributes_string"], // no evolution for this
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"},
expectedEvols: []string{"resources_string"},
},
{
name: "New evolutions at tsEndTime - should not include new evolution",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"},
expectedEvols: []string{"resources_string"},
},
{
name: "New evolutions after tsEndTime - should exclude new",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"},
expectedEvols: []string{"resources_string"},
},
{
name: "Empty columns array",
columns: []*schema.Column{},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{},
expectedEvols: []string{},
expectedError: true,
errorStr: "column resources_string not found",
},
{
name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
columns: []*schema.Column{
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resource"},
expectedEvols: []string{"resource"}, // should use first one (older)
},
{
name: "Genuine Duplicate evolutions with new version- should consider both",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 2,
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string", "resource"},
expectedEvols: []string{"resources_string", "resource"}, // should use first one (older)
},
{
name: "Evolution exactly at tsEndTime",
columns: []*schema.Column{
logsV2Columns["resources_string"],
logsV2Columns["resource"],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
ReleaseTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
expectedEvols: []string{"resources_string"},
},
{
name: "Single evolution after tsStartTime - JSON body",
columns: []*schema.Column{
logsV2Columns[LogsV2BodyV2Column],
logsV2Columns[LogsV2BodyPromotedColumn],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyV2Column,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "__all__",
ReleaseTime: time.Unix(0, 0),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "user.name",
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
expectedEvols: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
},
{
name: "No evolution after tsStartTime - JSON body",
columns: []*schema.Column{
logsV2Columns[LogsV2BodyV2Column],
logsV2Columns[LogsV2BodyPromotedColumn],
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyV2Column,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "__all__",
ReleaseTime: time.Unix(0, 0),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "user.name",
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{LogsV2BodyPromotedColumn},
expectedEvols: []string{LogsV2BodyPromotedColumn},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
resultColumns, resultEvols, err := selectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
if tc.expectedError {
assert.Contains(t, err.Error(), tc.errorStr)
} else {
require.NoError(t, err)
assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
resultColumnNames := make([]string, len(resultColumns))
for i, col := range resultColumns {
resultColumnNames[i] = col.Name
}
resultEvolNames := make([]string, len(resultEvols))
for i, evol := range resultEvols {
resultEvolNames[i] = evol.ColumnName
}
for i := range tc.expectedColumns {
assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
}
for i := range tc.expectedEvols {
assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
}
// Verify sorting: should be descending by ReleaseTime
for i := 0; i < len(resultEvols)-1; i++ {
assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
"evolutions should be sorted descending by ReleaseTime")
}
}
})
}
}
func TestFieldForWithMaterialized(t *testing.T) {
ctx := context.Background()

View File

@@ -1205,9 +1205,6 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
return statementBuilder, mockMetadataStore

View File

@@ -11,7 +11,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -20,14 +19,13 @@ import (
)
type logQueryStatementBuilder struct {
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
resourceFilterResolver *telemetryresourcefilter.ResourceFingerprintResolver[qbtypes.LogAggregation]
aggExprRewriter qbtypes.AggExprRewriter
fl flagger.Flagger
skipResourceFingerprintEnabled bool
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
aggExprRewriter qbtypes.AggExprRewriter
fl flagger.Flagger
fullTextColumn *telemetrytypes.TelemetryFieldKey
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
@@ -44,13 +42,10 @@ func NewLogQueryStatementBuilder(
fullTextColumn *telemetrytypes.TelemetryFieldKey,
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
fl flagger.Flagger,
telemetryStore telemetrystore.TelemetryStore,
skipResourceFingerprintEnable bool,
skipResourceFingerprintThreshold uint64,
) *logQueryStatementBuilder {
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
resourceFilterResolver := telemetryresourcefilter.NewResolver[qbtypes.LogAggregation](
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.LogAggregation](
settings,
DBName,
LogsResourceV2TableName,
@@ -60,21 +55,18 @@ func NewLogQueryStatementBuilder(
fullTextColumn,
jsonKeyToKey,
fl,
telemetryStore,
skipResourceFingerprintThreshold,
)
return &logQueryStatementBuilder{
logger: logsSettings.Logger(),
metadataStore: metadataStore,
fm: fieldMapper,
cb: conditionBuilder,
resourceFilterResolver: resourceFilterResolver,
aggExprRewriter: aggExprRewriter,
fl: fl,
skipResourceFingerprintEnabled: skipResourceFingerprintEnable,
fullTextColumn: fullTextColumn,
jsonKeyToKey: jsonKeyToKey,
logger: logsSettings.Logger(),
metadataStore: metadataStore,
fm: fieldMapper,
cb: conditionBuilder,
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
aggExprRewriter: aggExprRewriter,
fl: fl,
fullTextColumn: fullTextColumn,
jsonKeyToKey: jsonKeyToKey,
}
}
@@ -279,11 +271,9 @@ func (b *logQueryStatementBuilder) buildListQuery(
bodyJSONEnabled = b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" {
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
@@ -325,7 +315,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
@@ -383,11 +373,9 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
bodyJSONEnabled = b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" {
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
@@ -431,7 +419,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
// Add FROM clause
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
@@ -543,11 +531,9 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
bodyJSONEnabled = b.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" && !skipResourceCTE {
} else if frag != "" && !skipResourceCTE {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
@@ -590,7 +576,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
@@ -654,7 +640,6 @@ func (b *logQueryStatementBuilder) addFilterCondition(
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
skipResourceFilter bool,
) (querybuilder.PreparedWhereClause, error) {
var preparedWhereClause querybuilder.PreparedWhereClause
@@ -671,7 +656,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
ConditionBuilder: b.cb,
FieldKeys: keys,
BodyJSONEnabled: bodyJSONEnabled,
SkipResourceFilter: skipResourceFilter,
SkipResourceFilter: true,
FullTextColumn: b.fullTextColumn,
JsonKeyToKey: b.jsonKeyToKey,
Variables: variables,
@@ -722,30 +707,33 @@ func (b *logQueryStatementBuilder) maybeAttachResourceFilter(
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
variables map[string]qbtypes.VariableItem,
) (cteSQL string, cteArgs []any, skipResourceFilter bool, err error) {
) (cteSQL string, cteArgs []any, err error) {
if b.skipResourceFingerprintEnabled {
decision, err := b.resourceFilterResolver.Resolve(ctx, query, start, end, variables)
if err != nil {
return "", nil, true, err
}
switch decision {
case qbtypes.ResourceFilterResolveKindNoOp:
return "", nil, true, nil
case qbtypes.ResourceFilterResolveKindFallback:
return "", nil, false, nil
}
}
stmt, err := b.resourceFilterResolver.StatementBuilder().Build(
ctx, start, end, qbtypes.RequestTypeRaw, query, variables,
)
stmt, err := b.buildResourceFilterCTE(ctx, query, start, end, variables)
if err != nil {
return "", nil, true, err
return "", nil, err
}
if stmt == nil {
return "", nil, true, nil
return "", nil, nil
}
sb.Where("resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args, true, nil
return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args, nil
}
func (b *logQueryStatementBuilder) buildResourceFilterCTE(
ctx context.Context,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
return b.resourceFilterStmtBuilder.Build(
ctx,
start,
end,
qbtypes.RequestTypeRaw,
query,
variables,
)
}

View File

@@ -2,36 +2,19 @@ package telemetrylogs
import (
"context"
"regexp"
"testing"
"time"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/require"
)
type regexQueryMatcher struct{}
func (m *regexQueryMatcher) Match(expectedSQL, actualSQL string) error {
re, err := regexp.Compile(expectedSQL)
if err != nil {
return err
}
if !re.MatchString(actualSQL) {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "expected query to match %s, got %s", expectedSQL, actualSQL)
}
return nil
}
func TestStatementBuilderTimeSeries(t *testing.T) {
// Create a test release time
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
@@ -229,9 +212,6 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
for _, c := range cases {
@@ -373,9 +353,6 @@ func TestStatementBuilderListQuery(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
for _, c := range cases {
@@ -522,9 +499,6 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
for _, c := range cases {
@@ -601,9 +575,6 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
for _, c := range cases {
@@ -699,9 +670,6 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
for _, c := range cases {
@@ -926,9 +894,6 @@ func TestAdjustKey(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
for _, c := range cases {
@@ -1074,9 +1039,6 @@ func TestStmtBuilderBodyField(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
@@ -1176,9 +1138,6 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
nil,
false,
100000,
)
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
@@ -1198,110 +1157,3 @@ func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
})
}
}
// TestSkipResourceFingerprintLogs exercises the three resolver outcomes for
// logs: use-CTE (count < threshold), fallback (count >= threshold), and the
// legacy path (feature disabled).
func TestSkipResourceFingerprintLogs(t *testing.T) {
const (
startMs = uint64(1747947419000)
endMs = uint64(1747983448000)
threshold = uint64(10)
)
query := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{
Expression: "service.name = 'redis-manual'",
},
Limit: 5,
}
t.Run("disabled uses the legacy CTE", func(t *testing.T) {
sb := newSkipResourceFingerprintLogsBuilder(t, nil, false, threshold)
stmt, err := sb.Build(context.Background(), startMs, endMs, qbtypes.RequestTypeRaw, query, nil)
require.NoError(t, err)
require.Contains(t, stmt.Query, "__resource_filter AS (SELECT fingerprint")
require.Contains(t, stmt.Query, "resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
})
t.Run("CTE attached when count below threshold", func(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{uint64(2)}))
sb := newSkipResourceFingerprintLogsBuilder(t, mockStore, true, threshold)
stmt, err := sb.Build(context.Background(), startMs, endMs, qbtypes.RequestTypeRaw, query, nil)
require.NoError(t, err)
require.Contains(t, stmt.Query, "__resource_filter AS (SELECT fingerprint")
require.Contains(t, stmt.Query, "resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
require.NoError(t, mock.ExpectationsWereMet())
})
t.Run("fallback when count at or above threshold", func(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_logs\.distributed_logs_v2_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{threshold}))
sb := newSkipResourceFingerprintLogsBuilder(t, mockStore, true, threshold)
stmt, err := sb.Build(context.Background(), startMs, endMs, qbtypes.RequestTypeRaw, query, nil)
require.NoError(t, err)
require.NotContains(t, stmt.Query, "__resource_filter AS")
require.NotContains(t, stmt.Query, "resource_fingerprint")
require.Contains(t, stmt.Query, "service.name")
require.NoError(t, mock.ExpectationsWereMet())
})
}
func newSkipResourceFingerprintLogsBuilder(
t *testing.T,
telemetryStore telemetrystore.TelemetryStore,
skipEnable bool,
threshold uint64,
) *logQueryStatementBuilder {
t.Helper()
fl := flaggertest.New(t)
fm := NewFieldMapper(fl)
cb := NewConditionBuilder(fm, fl)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC))
aggExprRewriter := querybuilder.NewAggExprRewriter(
instrumentationtest.New().ToProviderSettings(),
DefaultFullTextColumn,
fm,
cb,
GetBodyJSONKey,
fl,
)
return NewLogQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
fl,
telemetryStore,
skipEnable,
threshold,
)
}

View File

@@ -344,11 +344,6 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
})
}
}
if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
return nil, false, err
}
return keys, complete, nil
}
@@ -694,7 +689,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
}
}
if err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
if _, err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
return nil, false, err
}
@@ -2375,8 +2370,8 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
return entries, nil
}
// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
// Get retrieves all evolutions for the given selectors from DB.
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) (map[string][]*telemetrytypes.EvolutionEntry, error) {
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
for _, keySelector := range keysToUpdate {
@@ -2390,7 +2385,7 @@ func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Co
evolutions, err := k.fetchEvolutionEntryFromClickHouse(ctx, metadataKeySelectors)
if err != nil {
return errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
}
evolutionsByUniqueKey := make(map[string][]*telemetrytypes.EvolutionEntry)
@@ -2421,7 +2416,7 @@ func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Co
}
}
}
return nil
return evolutionsByUniqueKey, nil
}
// chunkSizeFirstSeenMetricMetadata limits the number of tuples per SQL query to avoid hitting the max_query_size limit.

View File

@@ -2,12 +2,8 @@ package telemetrymetadata
import (
"context"
"regexp"
"testing"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
@@ -17,24 +13,12 @@ import (
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/stretchr/testify/assert"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/stretchr/testify/require"
)
type regexMatcher struct {
}
func (m *regexMatcher) Match(expectedSQL, actualSQL string) error {
re, err := regexp.Compile(expectedSQL)
if err != nil {
return err
}
if !re.MatchString(actualSQL) {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "expected query to contain %s, got %s", expectedSQL, actualSQL)
}
return nil
}
func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()

View File

@@ -0,0 +1,582 @@
package telemetrymetadata
import (
"context"
"regexp"
"testing"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/telemetrytraces"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/stretchr/testify/assert"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/stretchr/testify/require"
)
func newTestTelemetryMetaStoreTestHelper(t *testing.T, store telemetrystore.TelemetryStore) telemetrytypes.MetadataStore {
t.Helper()
return NewTelemetryMetaStore(
instrumentationtest.New().ToProviderSettings(),
store,
telemetrytraces.DBName,
telemetrytraces.TagAttributesV2TableName,
telemetrytraces.SpanAttributesKeysTblName,
telemetrytraces.SpanIndexV3TableName,
telemetrymetrics.DBName,
telemetrymetrics.AttributesMetadataTableName,
telemetrymeter.DBName,
telemetrymeter.SamplesAgg1dTableName,
telemetrylogs.DBName,
telemetrylogs.LogsV2TableName,
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
DBName,
AttributesMetadataLocalTableName,
ColumnEvolutionMetadataTableName,
flaggertest.New(t),
)
}
type regexMatcher struct {
}
func (m *regexMatcher) Match(expectedSQL, actualSQL string) error {
re, err := regexp.Compile(expectedSQL)
if err != nil {
return err
}
if !re.MatchString(actualSQL) {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "expected query to contain %s, got %s", expectedSQL, actualSQL)
}
return nil
}
func TestGetKeys(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
rows := cmock.NewRows([]cmock.ColumnType{
{Name: "statement", Type: "String"},
}, [][]any{{"CREATE TABLE signoz_traces.signoz_index_v3"}})
mock.
ExpectSelect("SHOW CREATE TABLE signoz_traces.distributed_signoz_index_v3").
WillReturnRows(rows)
query := `SELECT.*`
mock.ExpectQuery(query).
WithArgs("%http.method%", telemetrytypes.FieldDataTypeString.TagDataType(), 11).
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
{Name: "tag_key", Type: "String"},
{Name: "tag_type", Type: "String"},
{Name: "tag_data_type", Type: "String"},
{Name: "priority", Type: "UInt8"},
}, [][]any{{"http.method", "tag", "String", 1}, {"http.method", "tag", "String", 1}}))
keys, _, err := metadata.GetKeys(context.Background(), &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeString,
Name: "http.method",
Limit: 10,
})
if err != nil {
t.Fatalf("Failed to get keys: %v", err)
}
t.Logf("Keys: %v", keys)
}
func TestApplyBackwardCompatibleKeys(t *testing.T) {
tests := []struct {
name string
inputKeys []*telemetrytypes.TelemetryFieldKey
expectedKeys []string
notExpectedKey string
}{
{
name: "bidirectional mapping: net.peer.name -> server.address",
inputKeys: []*telemetrytypes.TelemetryFieldKey{
{
Name: "net.peer.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
expectedKeys: []string{"net.peer.name", "server.address"},
},
{
name: "bidirectional mapping: server.address -> net.peer.name",
inputKeys: []*telemetrytypes.TelemetryFieldKey{
{
Name: "server.address",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
expectedKeys: []string{"server.address", "net.peer.name"},
},
{
name: "bidirectional mapping: http.url -> url.full",
inputKeys: []*telemetrytypes.TelemetryFieldKey{
{
Name: "http.url",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
expectedKeys: []string{"http.url", "url.full"},
},
{
name: "bidirectional mapping: url.full -> http.url",
inputKeys: []*telemetrytypes.TelemetryFieldKey{
{
Name: "url.full",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
expectedKeys: []string{"url.full", "http.url"},
},
{
name: "key without alias",
inputKeys: []*telemetrytypes.TelemetryFieldKey{
{
Name: "custom.attribute",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
expectedKeys: []string{"custom.attribute"},
notExpectedKey: "server.address",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
hasTraces := false
hasLogs := false
for _, key := range tt.inputKeys {
switch key.Signal {
case telemetrytypes.SignalTraces:
hasTraces = true
case telemetrytypes.SignalLogs:
hasLogs = true
}
}
if hasTraces {
mock.ExpectSelect("SHOW CREATE TABLE signoz_traces.distributed_signoz_index_v3").
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
{Name: "statement", Type: "String"},
}, [][]any{{"CREATE TABLE signoz_traces.signoz_index_v3"}}))
var args []interface{}
var rows [][]any
for _, key := range tt.inputKeys {
if key.Signal == telemetrytypes.SignalTraces {
tagType := "tag"
if key.FieldContext == telemetrytypes.FieldContextResource {
tagType = "resource"
}
args = append(args, "%"+key.Name+"%", tagType, key.FieldDataType.TagDataType())
rows = append(rows, []any{key.Name, tagType, key.FieldDataType.TagDataType(), 1})
}
}
args = append(args, 11)
mock.ExpectQuery(`SELECT.*`).
WithArgs(args...).
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
{Name: "tag_key", Type: "String"},
{Name: "tag_type", Type: "String"},
{Name: "tag_data_type", Type: "String"},
{Name: "priority", Type: "UInt8"},
}, rows))
}
if hasLogs {
var args []interface{}
var rows [][]any
for _, key := range tt.inputKeys {
if key.Signal == telemetrytypes.SignalLogs {
tagType := "tag"
if key.FieldContext == telemetrytypes.FieldContextResource {
tagType = "resource"
}
args = append(args, "%"+key.Name+"%", tagType, key.FieldDataType.TagDataType())
rows = append(rows, []any{key.Name, tagType, key.FieldDataType.TagDataType(), 1})
}
}
args = append(args, 11)
mock.ExpectQuery(`SELECT.*`).
WithArgs(args...).
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
{Name: "tag_key", Type: "String"},
{Name: "tag_type", Type: "String"},
{Name: "tag_data_type", Type: "String"},
{Name: "priority", Type: "UInt8"},
}, rows))
}
selectors := []*telemetrytypes.FieldKeySelector{}
for _, key := range tt.inputKeys {
selectors = append(selectors, &telemetrytypes.FieldKeySelector{
Signal: key.Signal,
FieldContext: key.FieldContext,
FieldDataType: key.FieldDataType,
Name: key.Name,
Limit: 10,
})
}
resultMap, _, err := metadata.GetKeysMulti(context.Background(), selectors)
require.NoError(t, err, "GetKeysMulti should not return an error")
for _, expectedKey := range tt.expectedKeys {
assert.Contains(t, resultMap, expectedKey, "Expected key %q to exist in result map", expectedKey)
}
if tt.notExpectedKey != "" {
assert.NotContains(t, resultMap, tt.notExpectedKey, "Did not expect key %q to exist in result map", tt.notExpectedKey)
}
for _, srcKey := range tt.inputKeys {
backwardCompatKeys := GetBackwardCompatKeysForSignal(srcKey.Signal)
if aliasKey, ok := backwardCompatKeys[srcKey.Name]; ok {
aliasExistedInInput := false
for _, inputKey := range tt.inputKeys {
if inputKey.Name == aliasKey {
aliasExistedInInput = true
break
}
}
if !aliasExistedInInput {
if aliasEntries, exists := resultMap[aliasKey]; exists && len(aliasEntries) > 0 {
aliasEntry := aliasEntries[0]
assert.Equal(t, srcKey.Signal, aliasEntry.Signal, "Alias %q should have same signal", aliasKey)
assert.Equal(t, srcKey.FieldContext, aliasEntry.FieldContext, "Alias %q should have same field context", aliasKey)
assert.Equal(t, srcKey.FieldDataType, aliasEntry.FieldDataType, "Alias %q should have same field data type", aliasKey)
}
}
}
}
require.NoError(t, mock.ExpectationsWereMet(), "All SQL expectations should be met")
})
}
}
func TestEnrichWithIntrinsicMetricKeys(t *testing.T) {
result := enrichWithIntrinsicMetricKeys(
map[string][]*telemetrytypes.TelemetryFieldKey{},
[]*telemetrytypes.FieldKeySelector{
{
Signal: telemetrytypes.SignalMetrics,
Name: "metric",
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
},
},
)
require.Contains(t, result, "metric_name")
assert.Equal(t, telemetrytypes.FieldContextMetric, result["metric_name"][0].FieldContext)
result = enrichWithIntrinsicMetricKeys(
map[string][]*telemetrytypes.TelemetryFieldKey{},
[]*telemetrytypes.FieldKeySelector{
{
Signal: telemetrytypes.SignalMetrics,
Name: "metric",
FieldContext: telemetrytypes.FieldContextAttribute,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
},
},
)
assert.NotContains(t, result, "metric_name")
result = enrichWithIntrinsicMetricKeys(
map[string][]*telemetrytypes.TelemetryFieldKey{},
[]*telemetrytypes.FieldKeySelector{
{
Signal: telemetrytypes.SignalMetrics,
MetricContext: &telemetrytypes.MetricContext{
MetricNamespace: "system.cpu",
},
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
},
},
)
assert.NotContains(t, result, "metric_name")
}
func TestGetMetricFieldValuesIntrinsicMetricName(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
valueRows := cmock.NewRows([]cmock.ColumnType{
{Name: "metric_name", Type: "String"},
}, [][]any{{"metric.a"}, {"metric.b"}})
query := `SELECT .*metric_name.*` + telemetrymetrics.TimeseriesV41weekTableName + `.*GROUP BY.*metric_name`
mock.ExpectQuery(query).
WithArgs(51).
WillReturnRows(valueRows)
metadataRows := cmock.NewRows([]cmock.ColumnType{
{Name: "attr_string_value", Type: "String"},
}, [][]any{})
mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? LIMIT ?")).
WithArgs("metric_name", 49).
WillReturnRows(metadataRows)
values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
FieldKeySelector: &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalMetrics,
Name: "metric_name",
Limit: 50,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
},
Limit: 50,
})
require.NoError(t, err)
assert.True(t, complete)
assert.ElementsMatch(t, []string{"metric.a", "metric.b"}, values.StringValues)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestGetMetricFieldValuesIntrinsicBoolReturnsEmpty(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
metadataRows := cmock.NewRows([]cmock.ColumnType{
{Name: "attr_string_value", Type: "String"},
}, [][]any{})
mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? AND attr_datatype = ? AND attr_string_value = ? LIMIT ?")).
WithArgs("is_monotonic", telemetrytypes.FieldDataTypeBool.TagDataType(), "true", 11).
WillReturnRows(metadataRows)
values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
FieldKeySelector: &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalMetrics,
Name: "is_monotonic",
FieldDataType: telemetrytypes.FieldDataTypeBool,
Limit: 10,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
},
Value: "true",
Limit: 10,
})
require.NoError(t, err)
assert.True(t, complete)
assert.Empty(t, values.StringValues)
assert.Empty(t, values.BoolValues)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestGetMetricFieldValuesAppliesMetricNamespace(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
valueRows := cmock.NewRows([]cmock.ColumnType{
{Name: "attr_string_value", Type: "String"},
}, [][]any{{"value.a"}})
mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? AND metric_name LIKE ? LIMIT ?")).
WithArgs("custom_key", "system.cpu%", 11).
WillReturnRows(valueRows)
values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
FieldKeySelector: &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalMetrics,
Name: "custom_key",
Limit: 10,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
MetricContext: &telemetrytypes.MetricContext{
MetricNamespace: "system.cpu",
},
},
Limit: 10,
})
require.NoError(t, err)
assert.True(t, complete)
assert.ElementsMatch(t, []string{"value.a"}, values.StringValues)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestGetMetricFieldValuesIntrinsicMetricNameAppliesMetricNamespace(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
valueRows := cmock.NewRows([]cmock.ColumnType{
{Name: "metric_name", Type: "String"},
}, [][]any{{"system.cpu.utilization"}})
mock.ExpectQuery(regexp.QuoteMeta("SELECT metric_name FROM signoz_metrics.distributed_time_series_v4_1week WHERE metric_name LIKE ? GROUP BY metric_name LIMIT ?")).
WithArgs("system.cpu%", 51).
WillReturnRows(valueRows)
metadataRows := cmock.NewRows([]cmock.ColumnType{
{Name: "attr_string_value", Type: "String"},
}, [][]any{})
mock.ExpectQuery(regexp.QuoteMeta("SELECT DISTINCT attr_string_value FROM signoz_metrics.distributed_metadata WHERE attr_name = ? AND metric_name LIKE ? LIMIT ?")).
WithArgs("metric_name", "system.cpu%", 50).
WillReturnRows(metadataRows)
values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
FieldKeySelector: &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalMetrics,
Name: "metric_name",
Limit: 50,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
MetricContext: &telemetrytypes.MetricContext{
MetricNamespace: "system.cpu",
},
},
Limit: 50,
})
require.NoError(t, err)
assert.True(t, complete)
assert.ElementsMatch(t, []string{"system.cpu.utilization"}, values.StringValues)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestGetMeterSourceMetricFieldValuesAppliesMetricNamespace(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
rows := cmock.NewRows([]cmock.ColumnType{
{Name: "attr", Type: "Array(String)"},
}, [][]any{{[]string{"service.name", "frontend"}}})
mock.ExpectQuery(`SELECT .*distributed_samples_agg_1d.*metric_name LIKE .*`).
WithArgs("service.name", "\\_\\_%", "system.cpu%", "", 11).
WillReturnRows(rows)
values, complete, err := metadata.(*telemetryMetaStore).getMeterSourceMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
FieldKeySelector: &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalMetrics,
Source: telemetrytypes.SourceMeter,
Name: "service.name",
MetricContext: &telemetrytypes.MetricContext{
MetricNamespace: "system.cpu",
},
},
Limit: 10,
})
require.NoError(t, err)
assert.True(t, complete)
assert.ElementsMatch(t, []string{"frontend"}, values.StringValues)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestGetMetricsKeysAppliesMetricNamespace(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
rows := cmock.NewRows([]cmock.ColumnType{
{Name: "name", Type: "String"},
{Name: "field_context", Type: "String"},
{Name: "field_data_type", Type: "String"},
{Name: "priority", Type: "UInt8"},
}, [][]any{{"service.name", "resource", "String", 1}})
mock.ExpectQuery(`(?s)SELECT.*distributed_metadata.*metric_name LIKE.*`).
WithArgs("%service%", "\\_\\_%", "system.cpu%", 11).
WillReturnRows(rows)
keys, complete, err := metadata.(*telemetryMetaStore).getMetricsKeys(context.Background(), []*telemetrytypes.FieldKeySelector{
{
Signal: telemetrytypes.SignalMetrics,
Name: "service",
Limit: 10,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
MetricContext: &telemetrytypes.MetricContext{
MetricNamespace: "system.cpu",
},
},
})
require.NoError(t, err)
assert.True(t, complete)
assert.Len(t, keys, 1)
assert.Equal(t, "service.name", keys[0].Name)
require.NoError(t, mock.ExpectationsWereMet())
}
func TestGetMeterSourceMetricKeysAppliesMetricNamespace(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := newTestTelemetryMetaStoreTestHelper(t, mockTelemetryStore)
rows := cmock.NewRows([]cmock.ColumnType{
{Name: "attr_name", Type: "String"},
}, [][]any{{"service.name"}})
mock.ExpectQuery(`SELECT.*distributed_samples_agg_1d.*metric_name LIKE.*`).
WithArgs("%service%", "\\_\\_%", "system.cpu%", 10).
WillReturnRows(rows)
keys, complete, err := metadata.(*telemetryMetaStore).getMeterSourceMetricKeys(context.Background(), []*telemetrytypes.FieldKeySelector{
{
Signal: telemetrytypes.SignalMetrics,
Source: telemetrytypes.SourceMeter,
Name: "service",
Limit: 10,
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
MetricContext: &telemetrytypes.MetricContext{
MetricNamespace: "system.cpu",
},
},
})
require.NoError(t, err)
assert.True(t, complete)
assert.Len(t, keys, 1)
assert.Equal(t, "service.name", keys[0].Name)
require.NoError(t, mock.ExpectationsWereMet())
}

View File

@@ -1,77 +0,0 @@
package telemetryresourcefilter
import (
"context"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/telemetrystore"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
type ResourceFingerprintResolver[T any] struct {
stmtBuilder *resourceFilterStatementBuilder[T]
telemetryStore telemetrystore.TelemetryStore
threshold uint64
}
func NewResolver[T any](
settings factory.ProviderSettings,
dbName string,
tableName string,
signal telemetrytypes.Signal,
source telemetrytypes.Source,
metadataStore telemetrytypes.MetadataStore,
fullTextColumn *telemetrytypes.TelemetryFieldKey,
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
fl flagger.Flagger,
telemetryStore telemetrystore.TelemetryStore,
threshold uint64,
) *ResourceFingerprintResolver[T] {
return &ResourceFingerprintResolver[T]{
stmtBuilder: New[T](
settings,
dbName,
tableName,
signal,
source,
metadataStore,
fullTextColumn,
jsonKeyToKey,
fl,
),
telemetryStore: telemetryStore,
threshold: threshold,
}
}
func (r *ResourceFingerprintResolver[T]) StatementBuilder() qbtypes.StatementBuilder[T] {
return r.stmtBuilder
}
func (r *ResourceFingerprintResolver[T]) Resolve(
ctx context.Context,
query qbtypes.QueryBuilderQuery[T],
start, end uint64,
variables map[string]qbtypes.VariableItem,
) (qbtypes.ResourceFilterResolveKind, error) {
countStmt, err := r.stmtBuilder.BuildCount(ctx, start, end, query, variables)
if err != nil {
return qbtypes.ResourceFilterResolveKindNoOp, err
}
if countStmt == nil {
return qbtypes.ResourceFilterResolveKindNoOp, nil
}
var count uint64
row := r.telemetryStore.ClickhouseDB().QueryRow(ctx, countStmt.Query, countStmt.Args...)
if err := row.Scan(&count); err != nil {
return qbtypes.ResourceFilterResolveKindNoOp, err
}
if count >= r.threshold {
return qbtypes.ResourceFilterResolveKindFallback, nil
}
return qbtypes.ResourceFilterResolveKindUseCTE, nil
}

View File

@@ -127,25 +127,6 @@ func (b *resourceFilterStatementBuilder[T]) Build(
}, nil
}
// BuildCount returns a statement that counts the distinct fingerprints matching
// the resource filter. Returns (nil, nil) when the filter is a no-op.
func (b *resourceFilterStatementBuilder[T]) BuildCount(
ctx context.Context,
start uint64,
end uint64,
query qbtypes.QueryBuilderQuery[T],
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
inner, err := b.Build(ctx, start, end, qbtypes.RequestTypeRaw, query, variables)
if err != nil || inner == nil {
return nil, err
}
return &qbtypes.Statement{
Query: fmt.Sprintf("SELECT count() FROM (%s)", inner.Query),
Args: inner.Args,
}, nil
}
// addConditions adds both filter and time conditions to the query.
// Returns true (isNoOp) when the filter expression evaluated to no resource conditions,
// meaning the CTE would select all fingerprints and should be skipped entirely.

View File

@@ -161,33 +161,7 @@ func (c *conditionBuilder) conditionFor(
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
var value any
column := columns[0]
if len(key.Evolutions) > 0 {
// we will use the corresponding column and its evolution entry for the query
newColumns, _, err := qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
if err != nil {
return "", err
}
if len(newColumns) == 0 {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name)
}
// Multiple columns means fieldExpression is a multiIf returning NULL when none match,
// so a simple null check is sufficient.
if len(newColumns) > 1 {
if operator == qbtypes.FilterOperatorExists {
return sb.IsNotNull(fieldExpression), nil
} else {
return sb.IsNull(fieldExpression), nil
}
}
// otherwise we have to find the correct exist operator based on the column type
column = newColumns[0]
}
switch column.Type.GetType() {
switch columns[0].Type.GetType() {
case schema.ColumnTypeEnumJSON:
if operator == qbtypes.FilterOperatorExists {
return sb.IsNotNull(fieldExpression), nil
@@ -204,7 +178,7 @@ func (c *conditionBuilder) conditionFor(
return sb.E(fieldExpression, value), nil
}
case schema.ColumnTypeEnumLowCardinality:
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
switch elementType := columns[0].Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
case schema.ColumnTypeEnumString:
value = ""
if operator == qbtypes.FilterOperatorExists {
@@ -228,14 +202,14 @@ func (c *conditionBuilder) conditionFor(
return sb.E(fieldExpression, value), nil
}
case schema.ColumnTypeEnumMap:
keyType := column.Type.(schema.MapColumnType).KeyType
keyType := columns[0].Type.(schema.MapColumnType).KeyType
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, columns[0].Type)
}
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
switch valueType := columns[0].Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
if key.Materialized {
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
}
@@ -248,7 +222,7 @@ func (c *conditionBuilder) conditionFor(
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
}
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", columns[0].Type)
}
}
return "", nil

View File

@@ -3,7 +3,6 @@ package telemetrytraces
import (
"context"
"testing"
"time"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -15,7 +14,6 @@ import (
func TestConditionFor(t *testing.T) {
ctx := context.Background()
mockEvolution := mockEvolutionData(time.Date(2025, 10, 26, 0, 10, 0, 0, time.UTC))
testCases := []struct {
name string
key telemetrytypes.TelemetryFieldKey
@@ -215,7 +213,6 @@ func TestConditionFor(t *testing.T) {
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: mockEvolution,
},
operator: qbtypes.FilterOperatorExists,
value: nil,
@@ -228,7 +225,6 @@ func TestConditionFor(t *testing.T) {
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: mockEvolution,
},
operator: qbtypes.FilterOperatorNotExists,
value: nil,
@@ -306,85 +302,3 @@ func TestConditionFor(t *testing.T) {
})
}
}
func TestConditionForResourceWithEvolution(t *testing.T) {
ctx := context.Background()
releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
evolutions := mockEvolutionData(releaseTime)
testCases := []struct {
name string
key telemetrytypes.TelemetryFieldKey
operator qbtypes.FilterOperator
tsStart uint64
tsEnd uint64
expectedSQL string
}{
{
name: "Exists - window after release - JSON only",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: evolutions,
},
operator: qbtypes.FilterOperatorExists,
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL",
},
{
name: "NotExists - window after release - JSON only",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: evolutions,
},
operator: qbtypes.FilterOperatorNotExists,
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedSQL: "WHERE resource.`service.name`::String IS NULL",
},
{
name: "Exists - window before release - map only",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: evolutions,
},
operator: qbtypes.FilterOperatorExists,
tsStart: uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?",
},
{
name: "Exists - window straddles release - multiIf null check",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: evolutions,
},
operator: qbtypes.FilterOperatorExists,
tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
},
}
fm := NewFieldMapper()
conditionBuilder := NewConditionBuilder(fm)
for _, tc := range testCases {
sb := sqlbuilder.NewSelectBuilder()
t.Run(tc.name, func(t *testing.T) {
cond, err := conditionBuilder.ConditionFor(ctx, tc.tsStart, tc.tsEnd, &tc.key, tc.operator, nil, sb)
require.NoError(t, err)
sb.Where(cond)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, tc.expectedSQL)
})
}
}

View File

@@ -1,8 +1,6 @@
package telemetrytraces
import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
var (
IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{

View File

@@ -174,7 +174,7 @@ func (m *defaultFieldMapper) getColumn(
) ([]*schema.Column, error) {
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
return []*schema.Column{indexV3Columns["resources_string"], indexV3Columns["resource"]}, nil
return []*schema.Column{indexV3Columns["resource"]}, nil
case telemetrytypes.FieldContextScope:
return []*schema.Column{}, qbtypes.ErrColumnNotFound
case telemetrytypes.FieldContextAttribute:
@@ -254,92 +254,63 @@ func (m *defaultFieldMapper) FieldFor(
if err != nil {
return "", err
}
var newColumns []*schema.Column
var evolutionsEntries []*telemetrytypes.EvolutionEntry
if len(key.Evolutions) > 0 {
// we will use the corresponding column and its evolution entry for the query
newColumns, evolutionsEntries, err = qbtypes.SelectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
if err != nil {
return "", err
}
} else {
newColumns = columns
if len(columns) != 1 {
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
}
column := columns[0]
exprs := []string{}
existExpr := []string{}
for i, column := range newColumns {
// Use evolution column name if available, otherwise use the column name
columnName := column.Name
if evolutionsEntries != nil && evolutionsEntries[i] != nil {
columnName = evolutionsEntries[i].ColumnName
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
// json is only supported for resource context as of now
if key.FieldContext != telemetrytypes.FieldContextResource {
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
}
oldColumn := indexV3Columns["resources_string"]
oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
if key.Materialized {
oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
} else {
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
}
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,
schema.ColumnTypeEnumInt8,
schema.ColumnTypeEnumInt16,
schema.ColumnTypeEnumBool,
schema.ColumnTypeEnumDateTime64,
schema.ColumnTypeEnumFixedString:
return column.Name, nil
case schema.ColumnTypeEnumLowCardinality:
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
case schema.ColumnTypeEnumString:
return column.Name, nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for low cardinality column type %s", elementType, column.Type)
}
case schema.ColumnTypeEnumMap:
keyType := column.Type.(schema.MapColumnType).KeyType
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
}
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
// json is only supported for resource context as of now
if key.FieldContext != telemetrytypes.FieldContextResource {
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
}
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,
schema.ColumnTypeEnumInt8,
schema.ColumnTypeEnumInt16,
schema.ColumnTypeEnumBool,
schema.ColumnTypeEnumDateTime64,
schema.ColumnTypeEnumFixedString:
exprs = append(exprs, column.Name)
case schema.ColumnTypeEnumLowCardinality:
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
case schema.ColumnTypeEnumString:
exprs = append(exprs, column.Name)
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for low cardinality column type %s", elementType, column.Type)
}
case schema.ColumnTypeEnumMap:
keyType := column.Type.(schema.MapColumnType).KeyType
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
}
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
// a key could have been materialized, if so return the materialized column name
if key.Materialized {
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
existExpr = append(existExpr, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)))
} else {
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
}
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
// a key could have been materialized, if so return the materialized column name
if key.Materialized {
return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
}
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
}
}
if len(exprs) == 1 {
return exprs[0], nil
} else if len(exprs) > 1 {
// Ensure existExpr has the same length as exprs
if len(existExpr) != len(exprs) {
return "", errors.New(errors.TypeInternal, errors.CodeInternal, "length of exist exprs doesn't match to that of exprs")
}
finalExprs := []string{}
for i, expr := range exprs {
finalExprs = append(finalExprs, fmt.Sprintf("%s, %s", existExpr[i], expr))
}
return "multiIf(" + strings.Join(finalExprs, ", ") + ", NULL)", nil
}
// should not reach here
return columns[0].Name, nil
return column.Name, nil
}
// ColumnExpressionFor returns the column expression for the given field

View File

@@ -3,7 +3,6 @@ package telemetrytraces
import (
"context"
"testing"
"time"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -14,7 +13,6 @@ import (
func TestGetFieldKeyName(t *testing.T) {
ctx := context.Background()
mockEvolution := mockEvolutionData(time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC))
testCases := []struct {
name string
key telemetrytypes.TelemetryFieldKey
@@ -65,7 +63,6 @@ func TestGetFieldKeyName(t *testing.T) {
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
Evolutions: mockEvolution,
},
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
expectedError: nil,
@@ -77,7 +74,6 @@ func TestGetFieldKeyName(t *testing.T) {
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Materialized: true,
Evolutions: mockEvolution,
},
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)",
expectedError: nil,
@@ -96,7 +92,7 @@ func TestGetFieldKeyName(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fm := NewFieldMapper()
result, err := fm.FieldFor(ctx, uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano()), &tc.key)
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
if tc.expectedError != nil {
assert.Equal(t, tc.expectedError, err)
@@ -107,86 +103,3 @@ func TestGetFieldKeyName(t *testing.T) {
})
}
}
func TestFieldForResourceWithEvolution(t *testing.T) {
ctx := context.Background()
releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
evolutions := mockEvolutionData(releaseTime)
testCases := []struct {
name string
key telemetrytypes.TelemetryFieldKey
tsStart uint64
tsEnd uint64
expectedResult string
}{
{
name: "Window straddles release - both columns",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
Evolutions: evolutions,
},
tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
},
{
name: "Window fully after release - JSON column only",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
Evolutions: evolutions,
},
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "resource.`service.name`::String",
},
{
name: "Window fully before release - map column only",
key: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
Evolutions: evolutions,
},
tsStart: uint64(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "resources_string['service.name']",
},
{
name: "Window fully after release - materialized resource",
key: telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Materialized: true,
Evolutions: evolutions,
},
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "resource.`deployment.environment`::String",
},
{
name: "Window straddles release - materialized resource",
key: telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Materialized: true,
Evolutions: evolutions,
},
tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`==true, `resource_string_deployment$$environment`, NULL)",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fm := NewFieldMapper()
result, err := fm.FieldFor(ctx, tc.tsStart, tc.tsEnd, &tc.key)
require.NoError(t, err)
assert.Equal(t, tc.expectedResult, result)
})
}
}

View File

@@ -24,13 +24,13 @@ var (
)
type traceQueryStatementBuilder struct {
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
resourceFilterResolver *telemetryresourcefilter.ResourceFingerprintResolver[qbtypes.TraceAggregation]
aggExprRewriter qbtypes.AggExprRewriter
skipResourceFingerprintEnabled bool
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
aggExprRewriter qbtypes.AggExprRewriter
telemetryStore telemetrystore.TelemetryStore
}
var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*traceQueryStatementBuilder)(nil)
@@ -43,12 +43,10 @@ func NewTraceQueryStatementBuilder(
aggExprRewriter qbtypes.AggExprRewriter,
telemetryStore telemetrystore.TelemetryStore,
flagger flagger.Flagger,
skipResourceFingerprintEnable bool,
skipResourceFingerprintThreshold uint64,
) *traceQueryStatementBuilder {
tracesSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrytraces")
resourceFilterResolver := telemetryresourcefilter.NewResolver[qbtypes.TraceAggregation](
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.TraceAggregation](
settings,
DBName,
TracesResourceV3TableName,
@@ -58,18 +56,16 @@ func NewTraceQueryStatementBuilder(
nil,
nil,
flagger,
telemetryStore,
skipResourceFingerprintThreshold,
)
return &traceQueryStatementBuilder{
logger: tracesSettings.Logger(),
metadataStore: metadataStore,
fm: fieldMapper,
cb: conditionBuilder,
resourceFilterResolver: resourceFilterResolver,
aggExprRewriter: aggExprRewriter,
skipResourceFingerprintEnabled: skipResourceFingerprintEnable,
logger: tracesSettings.Logger(),
metadataStore: metadataStore,
fm: fieldMapper,
cb: conditionBuilder,
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
aggExprRewriter: aggExprRewriter,
telemetryStore: telemetryStore,
}
}
@@ -86,6 +82,13 @@ func (b *traceQueryStatementBuilder) Build(
start = querybuilder.ToNanoSecs(start)
end = querybuilder.ToNanoSecs(end)
keySelectors := getKeySelectors(query)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
/*
Adding a tech debt note here:
This piece of code is a hot fix and should be removed once we close issue: engineering-pod/issues/3622
@@ -121,19 +124,8 @@ func (b *traceQueryStatementBuilder) Build(
-------------------------------- End of tech debt ----------------------------
*/
// We modify SelectFields above (injecting default fields), and those default
// fields can carry keys that need evolutions, so fetch keys after that.
keySelectors := getKeySelectors(query)
query = b.adjustKeys(ctx, keys, query, requestType)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
for _, action := range adjustTraceKeys(keys, &query, requestType) {
// TODO: change to debug level once we are confident about the behavior
b.logger.InfoContext(ctx, "key adjustment action", slog.String("action", action))
}
// Create SQL builder
q := sqlbuilder.NewSelectBuilder()
@@ -201,30 +193,24 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation])
return keySelectors
}
// mergeDeprecatedTraceKeys prepends deprecated intrinsic/calculated trace field
// definitions to the keys map. We do this during statement building, not at
// metadata fetch time, because:
// 1. Filter expressions that reference deprecated columns must continue to
// resolve — otherwise they fail with "key not found".
// 2. Doing it at metadata fetch time would also surface deprecated keys in
// autocomplete suggestions, which we don't want.
// 3. We prepend (not append) so the intrinsic/calculated entry wins ordering
// in the multi_if SQL expression.
func mergeDeprecatedTraceKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey) {
func (b *traceQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], requestType qbtypes.RequestType) qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation] {
// add deprecated fields only during statement building
// why?
// 1. to not fail filter expression that use deprecated cols
// 2. this could have been moved to metadata fetching itself, however, that
// would mean, they also show up in suggestions we we don't want to do
// 3. reason for not doing a simple append is to keep intrinsic/calculated field first so that it gets
// priority in multi_if sql expression
for fieldKeyName, fieldKey := range IntrinsicFieldsDeprecated {
keys[fieldKeyName] = append([]*telemetrytypes.TelemetryFieldKey{&fieldKey}, keys[fieldKeyName]...)
}
for fieldKeyName, fieldKey := range CalculatedFieldsDeprecated {
keys[fieldKeyName] = append([]*telemetrytypes.TelemetryFieldKey{&fieldKey}, keys[fieldKeyName]...)
}
}
func adjustTraceKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, query *qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], requestType qbtypes.RequestType) []string {
mergeDeprecatedTraceKeys(keys)
// Adjust keys for alias expressions in aggregations
actions := querybuilder.AdjustKeysForAliasExpressions(query, requestType)
actions := querybuilder.AdjustKeysForAliasExpressions(&query, requestType)
/*
Check if user is using multiple contexts or data types for same field name
@@ -242,7 +228,7 @@ func adjustTraceKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, query
and make it just http.status_code and remove the duplicate entry.
*/
actions = append(actions, querybuilder.AdjustDuplicateKeys(query)...)
actions = append(actions, querybuilder.AdjustDuplicateKeys(&query)...)
/*
Now adjust each key to have correct context and data type
@@ -250,20 +236,24 @@ func adjustTraceKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, query
Reason for doing this is to not create an unexpected behavior for users
*/
for idx := range query.SelectFields {
actions = append(actions, adjustTraceKey(&query.SelectFields[idx], keys)...)
actions = append(actions, b.adjustKey(&query.SelectFields[idx], keys)...)
}
for idx := range query.GroupBy {
actions = append(actions, adjustTraceKey(&query.GroupBy[idx].TelemetryFieldKey, keys)...)
actions = append(actions, b.adjustKey(&query.GroupBy[idx].TelemetryFieldKey, keys)...)
}
for idx := range query.Order {
actions = append(actions, adjustTraceKey(&query.Order[idx].Key.TelemetryFieldKey, keys)...)
actions = append(actions, b.adjustKey(&query.Order[idx].Key.TelemetryFieldKey, keys)...)
}
return actions
for _, action := range actions {
// TODO: change to debug level once we are confident about the behavior
b.logger.InfoContext(ctx, "key adjustment action", slog.String("action", action))
}
return query
}
// adjustTraceKey resolves a single TelemetryFieldKey against the keys map.
func adjustTraceKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
func (b *traceQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
// for recording actions taken
actions := []string{}
@@ -312,11 +302,9 @@ func (b *traceQueryStatementBuilder) buildListQuery(
cteArgs [][]any
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" {
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
@@ -334,7 +322,7 @@ func (b *traceQueryStatementBuilder) buildListQuery(
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
}
@@ -395,17 +383,15 @@ func (b *traceQueryStatementBuilder) buildTraceQuery(
cteArgs [][]any
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, distSB, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, distSB, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" {
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, distSB, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, distSB, start, end, query, keys, variables)
if err != nil {
return nil, err
}
@@ -506,11 +492,9 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
cteArgs [][]any
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" {
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
@@ -551,7 +535,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
}
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
}
@@ -660,11 +644,9 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
cteArgs [][]any
)
frag, args, skipResourceFilter, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables)
if err != nil {
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
}
if frag != "" && !skipResourceCTE {
} else if frag != "" && !skipResourceCTE {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
@@ -706,7 +688,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
sb.From(fmt.Sprintf("%s.%s", DBName, SpanIndexV3TableName))
// Add filter conditions
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables, skipResourceFilter)
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
}
@@ -769,7 +751,6 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
skipResourceFilter bool,
) (querybuilder.PreparedWhereClause, error) {
var preparedWhereClause querybuilder.PreparedWhereClause
@@ -783,7 +764,7 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
FieldMapper: b.fm,
ConditionBuilder: b.cb,
FieldKeys: keys,
SkipResourceFilter: skipResourceFilter,
SkipResourceFilter: true,
Variables: variables,
StartNs: start,
EndNs: end,
@@ -824,30 +805,34 @@ func (b *traceQueryStatementBuilder) maybeAttachResourceFilter(
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
start, end uint64,
variables map[string]qbtypes.VariableItem,
) (cteSQL string, cteArgs []any, skipResourceFilter bool, err error) {
) (cteSQL string, cteArgs []any, err error) {
if b.skipResourceFingerprintEnabled {
decision, err := b.resourceFilterResolver.Resolve(ctx, query, start, end, variables)
if err != nil {
return "", nil, true, err
}
switch decision {
case qbtypes.ResourceFilterResolveKindNoOp:
return "", nil, true, nil
case qbtypes.ResourceFilterResolveKindFallback:
return "", nil, false, nil
}
}
stmt, err := b.resourceFilterResolver.StatementBuilder().Build(
ctx, start, end, qbtypes.RequestTypeRaw, query, variables,
)
stmt, err := b.buildResourceFilterCTE(ctx, query, start, end, variables)
if err != nil {
return "", nil, true, err
return "", nil, err
}
if stmt == nil {
return "", nil, true, nil
return "", nil, nil
}
sb.Where("resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args, true, nil
return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args, nil
}
func (b *traceQueryStatementBuilder) buildResourceFilterCTE(
ctx context.Context,
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
start, end uint64,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
return b.resourceFilterStmtBuilder.Build(
ctx,
start,
end,
qbtypes.RequestTypeRaw,
query,
variables,
)
}

View File

@@ -2,40 +2,20 @@ package telemetrytraces
import (
"context"
"regexp"
"testing"
"time"
cmock "github.com/SigNoz/clickhouse-go-mock"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/require"
)
type regexQueryMatcher struct{}
func (m *regexQueryMatcher) Match(expectedSQL, actualSQL string) error {
re, err := regexp.Compile(expectedSQL)
if err != nil {
return err
}
if !re.MatchString(actualSQL) {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "expected query to match %s, got %s", expectedSQL, actualSQL)
}
return nil
}
func TestStatementBuilder(t *testing.T) {
// releaseTime is chosen so it lands inside the standard [1747947419000, 1747983448000]ms
// test window, keeping the multiIf SQL form for resource fields.
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
requestType qbtypes.RequestType
@@ -375,7 +355,7 @@ func TestStatementBuilder(t *testing.T) {
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
@@ -387,8 +367,6 @@ func TestStatementBuilder(t *testing.T) {
aggExprRewriter,
nil,
fl,
false,
100000,
)
vars := map[string]qbtypes.VariableItem{
@@ -416,7 +394,6 @@ func TestStatementBuilder(t *testing.T) {
}
func TestStatementBuilderListQuery(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
requestType qbtypes.RequestType
@@ -673,7 +650,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
@@ -685,8 +662,6 @@ func TestStatementBuilderListQuery(t *testing.T) {
aggExprRewriter,
nil,
fl,
false,
100000,
)
for _, c := range cases {
@@ -708,7 +683,6 @@ func TestStatementBuilderListQuery(t *testing.T) {
}
func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
requestType qbtypes.RequestType
@@ -729,15 +703,6 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"service.name": {
{
Name: "service.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
},
},
},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
@@ -763,15 +728,6 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"service.name": {
{
Name: "service.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Evolutions: mockEvolutionData(time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)),
},
},
},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
@@ -802,7 +758,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = c.keysMap
if mockMetadataStore.KeysMap == nil {
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
}
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
@@ -815,8 +771,6 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
aggExprRewriter,
nil,
fl,
false,
100000,
)
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
@@ -834,92 +788,7 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
}
}
func TestStatementBuilderGroupByResourceEvolution(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
startMs uint64
endMs uint64
expected qbtypes.Statement
}{
{
name: "window straddles release - both JSON and map branches",
startMs: 1747947419000, // 2025-05-22 21:56:59 UTC, ~3m before release
endMs: 1747983448000, // 2025-05-23 07:57:28 UTC, ~10h after release
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
{
name: "window after release - JSON column only",
startMs: 1747960000000, // 2025-05-23 00:26:40 UTC, ~2.5h after release
endMs: 1747983448000, // 2025-05-23 07:57:28 UTC
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
Args: []any{"1747960000000000000", "1747983448000000000", uint64(1747958200), uint64(1747983448), 10, "1747960000000000000", "1747983448000000000", uint64(1747958200), uint64(1747983448)},
},
},
{
name: "window before release - map column only",
startMs: 1747900000000, // 2025-05-22 08:26:40 UTC, ~13.5h before release
endMs: 1747947000000, // 2025-05-22 21:50:00 UTC, ~10m before release
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(mapContains(resources_string, 'service.name') = ?, resources_string['service.name'], NULL)) AS `service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
Args: []any{true, "1747900000000000000", "1747947000000000000", uint64(1747898200), uint64(1747947000), 10, true, "1747900000000000000", "1747947000000000000", uint64(1747898200), uint64(1747947000)},
},
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
statementBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
nil,
fl,
false,
100000,
)
query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{Expression: "count()"},
},
Filter: &qbtypes.Filter{},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
},
},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
q, err := statementBuilder.Build(context.Background(), c.startMs, c.endMs, qbtypes.RequestTypeTimeSeries, query, nil)
require.NoError(t, err)
require.Equal(t, c.expected.Query, q.Query)
require.Equal(t, c.expected.Args, q.Args)
})
}
}
func TestStatementBuilderTraceQuery(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
requestType qbtypes.RequestType
@@ -1042,7 +911,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
@@ -1054,8 +923,6 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
aggExprRewriter,
nil,
fl,
false,
100000,
)
for _, c := range cases {
@@ -1077,7 +944,6 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
}
func TestAdjustKey(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
inputKey telemetrytypes.TelemetryFieldKey
@@ -1091,7 +957,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: IntrinsicFields["trace_id"],
},
{
@@ -1101,7 +967,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextBody, // incorrect context
FieldDataType: telemetrytypes.FieldDataTypeInt64,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "duration_nano",
FieldContext: telemetrytypes.FieldContextSpan, // should be corrected
@@ -1115,7 +981,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextSpan, // correct context
FieldDataType: telemetrytypes.FieldDataTypeInt64,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "duration_nano",
FieldContext: telemetrytypes.FieldContextSpan, // should be corrected
@@ -1129,8 +995,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
keysMap: buildCompleteFieldKeyMap(),
expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
},
{
name: "single matching key with context specified - override",
@@ -1139,8 +1005,8 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["cart.items_count"][0],
keysMap: buildCompleteFieldKeyMap(),
expectedKey: *buildCompleteFieldKeyMap()["cart.items_count"][0],
},
{
name: "multiple matching keys - all materialized",
@@ -1177,7 +1043,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "mixed.materialization.key",
FieldDataType: telemetrytypes.FieldDataTypeString,
@@ -1191,7 +1057,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "mixed.materialization.key",
FieldContext: telemetrytypes.FieldContextAttribute,
@@ -1206,7 +1072,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "unknown.field",
Materialized: false,
@@ -1219,7 +1085,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextAttribute,
@@ -1234,7 +1100,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "cart.items_count",
FieldContext: telemetrytypes.FieldContextAttribute,
@@ -1249,7 +1115,7 @@ func TestAdjustKey(t *testing.T) {
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedKey: telemetrytypes.TelemetryFieldKey{
Name: "user.id",
FieldContext: telemetrytypes.FieldContextAttribute,
@@ -1259,13 +1125,28 @@ func TestAdjustKey(t *testing.T) {
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
statementBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
nil,
fl,
)
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
// Create a copy of the input key to avoid modifying the original
key := c.inputKey
// Call adjustKey
adjustTraceKey(&key, c.keysMap)
statementBuilder.adjustKey(&key, c.keysMap)
// Verify the key was adjusted as expected
require.Equal(t, c.expectedKey.Name, key.Name, "key name should match")
@@ -1277,7 +1158,6 @@ func TestAdjustKey(t *testing.T) {
}
func TestAdjustKeys(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
cases := []struct {
name string
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]
@@ -1303,7 +1183,7 @@ func TestAdjustKeys(t *testing.T) {
},
},
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
{
Name: "service.name",
@@ -1340,7 +1220,7 @@ func TestAdjustKeys(t *testing.T) {
},
},
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedGroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
@@ -1387,7 +1267,7 @@ func TestAdjustKeys(t *testing.T) {
},
},
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedOrder: []qbtypes.OrderBy{
{
Key: qbtypes.OrderByKey{
@@ -1446,7 +1326,7 @@ func TestAdjustKeys(t *testing.T) {
},
},
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
expectedSelectFields: []telemetrytypes.TelemetryFieldKey{
{
Name: "trace_id",
@@ -1501,7 +1381,7 @@ func TestAdjustKeys(t *testing.T) {
},
},
},
keysMap: buildCompleteFieldKeyMap(releaseTime),
keysMap: buildCompleteFieldKeyMap(),
// After alias adjustment, name becomes "span.duration" with FieldContextUnspecified
// "span.duration" is not in keysMap, so context stays unspecified
expectedOrder: []qbtypes.OrderBy{
@@ -1519,6 +1399,21 @@ func TestAdjustKeys(t *testing.T) {
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
statementBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
nil,
fl,
)
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
// Create a deep copy of the keys map to avoid modifying the original
@@ -1529,7 +1424,7 @@ func TestAdjustKeys(t *testing.T) {
}
// Call adjustKeys
adjustTraceKeys(keysMapCopy, &c.query, qbtypes.RequestTypeScalar)
c.query = statementBuilder.adjustKeys(context.Background(), keysMapCopy, c.query, qbtypes.RequestTypeScalar)
// Verify select fields were adjusted
if c.expectedSelectFields != nil {
@@ -1588,115 +1483,3 @@ func TestAdjustKeys(t *testing.T) {
})
}
}
// TestSkipResourceFingerprint exercises the three resolver outcomes when
// skip_resource_fingerprint is enabled: use-CTE (count < threshold),
// fallback (count >= threshold), and the legacy path (feature disabled).
func TestSkipResourceFingerprint(t *testing.T) {
const (
startMs = uint64(1747947419000)
endMs = uint64(1747983448000)
threshold = uint64(10)
)
query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
Filter: &qbtypes.Filter{
Expression: "service.name = 'redis-manual'",
},
SelectFields: []telemetrytypes.TelemetryFieldKey{
{
Name: "name",
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
Limit: 5,
}
t.Run("disabled uses the legacy CTE", func(t *testing.T) {
sb := newSkipResourceFingerprintBuilder(t, nil, false, threshold)
stmt, err := sb.Build(context.Background(), startMs, endMs, qbtypes.RequestTypeRaw, query, nil)
require.NoError(t, err)
require.Contains(t, stmt.Query, "__resource_filter AS (SELECT fingerprint")
require.Contains(t, stmt.Query, "resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
})
t.Run("CTE attached when count below threshold", func(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
// Only the count query runs against the telemetry store; the CTE
// itself is embedded as SQL in the main query (no extra round trip).
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{uint64(2)}))
sb := newSkipResourceFingerprintBuilder(t, mockStore, true, threshold)
stmt, err := sb.Build(context.Background(), startMs, endMs, qbtypes.RequestTypeRaw, query, nil)
require.NoError(t, err)
require.Contains(t, stmt.Query, "__resource_filter AS (SELECT fingerprint")
require.Contains(t, stmt.Query, "resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
require.NoError(t, mock.ExpectationsWereMet())
})
t.Run("fallback when count at or above threshold", func(t *testing.T) {
mockStore := telemetrystoretest.New(telemetrystore.Config{}, &regexQueryMatcher{})
mock := mockStore.Mock()
mock.ExpectQueryRow(`SELECT count\(\) FROM \(SELECT fingerprint FROM signoz_traces\.distributed_traces_v3_resource`).
WillReturnRow(cmock.NewRow([]cmock.ColumnType{
{Name: "count", Type: "UInt64"},
}, []any{threshold}))
sb := newSkipResourceFingerprintBuilder(t, mockStore, true, threshold)
stmt, err := sb.Build(context.Background(), startMs, endMs, qbtypes.RequestTypeRaw, query, nil)
require.NoError(t, err)
require.NotContains(t, stmt.Query, "__resource_filter AS")
require.NotContains(t, stmt.Query, "resource_fingerprint")
// resource conditions are pushed onto the main table via the
// resource.`service.name` / resources_string lookup
require.Contains(t, stmt.Query, "service.name")
require.NoError(t, mock.ExpectationsWereMet())
})
}
func newSkipResourceFingerprintBuilder(
t *testing.T,
telemetryStore telemetrystore.TelemetryStore,
skipEnable bool,
threshold uint64,
) *traceQueryStatementBuilder {
t.Helper()
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(
instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl,
)
return NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
telemetryStore,
fl,
skipEnable,
threshold,
)
}

View File

@@ -1,12 +1,10 @@
package telemetrytraces
import (
"time"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
"service.name": {
{
@@ -117,33 +115,7 @@ func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
for _, keys := range keysMap {
for _, key := range keys {
key.Signal = telemetrytypes.SignalTraces
if key.FieldContext == telemetrytypes.FieldContextResource {
key.Evolutions = mockEvolutionData(releaseTime)
}
}
}
return keysMap
}
// mockEvolutionData returns the canonical resource-column evolution timeline used in tests:
// the legacy resources_string map at epoch 0 and the JSON resource column released at releaseTime.
func mockEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {
return []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalTraces,
ColumnName: "resources_string",
FieldContext: telemetrytypes.FieldContextResource,
ColumnType: "Map(LowCardinality(String), String)",
FieldName: "__all__",
ReleaseTime: time.Unix(0, 0),
},
{
Signal: telemetrytypes.SignalTraces,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
ReleaseTime: releaseTime,
},
}
}

View File

@@ -216,13 +216,6 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
}
b.stmtBuilder.logger.DebugContext(ctx, "Retrieved keys for query", slog.String("query_name", queryName), slog.Int("keys_count", len(keys)))
// The CTE only selects spans matching the filter. Aggregations, group by
// and order by run later in buildFinalQuery, so RequestTypeRaw is fine here.
for _, action := range adjustTraceKeys(keys, query, qbtypes.RequestTypeRaw) {
// TODO: change to debug level once we are confident about the behavior
b.stmtBuilder.logger.InfoContext(ctx, "key adjustment action", slog.String("action", action))
}
// Build resource filter CTE for this specific query
resourceFilterCTEName := fmt.Sprintf("__resource_filter_%s", cteName)
resourceStmt, err := b.buildResourceFilterCTE(ctx, *query)
@@ -424,28 +417,21 @@ func (b *traceOperatorCTEBuilder) buildNotCTE(leftCTE, rightCTE string) (string,
}
func (b *traceOperatorCTEBuilder) buildFinalQuery(ctx context.Context, selectFromCTE string, requestType qbtypes.RequestType) (*qbtypes.Statement, error) {
keySelectors := b.getKeySelectors()
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
b.adjustOperatorKeys(ctx, keys, requestType)
switch requestType {
case qbtypes.RequestTypeRaw:
return b.buildListQuery(ctx, selectFromCTE, keys)
return b.buildListQuery(ctx, selectFromCTE)
case qbtypes.RequestTypeTimeSeries:
return b.buildTimeSeriesQuery(ctx, selectFromCTE, keys)
return b.buildTimeSeriesQuery(ctx, selectFromCTE)
case qbtypes.RequestTypeTrace:
return b.buildTraceQuery(ctx, selectFromCTE, keys)
return b.buildTraceQuery(ctx, selectFromCTE)
case qbtypes.RequestTypeScalar:
return b.buildScalarQuery(ctx, selectFromCTE, keys)
return b.buildScalarQuery(ctx, selectFromCTE)
default:
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType)
}
}
func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
sb := sqlbuilder.NewSelectBuilder()
// Select core fields
@@ -467,6 +453,22 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
"parent_span_id": true,
}
// Get keys for selectFields
keySelectors := b.getKeySelectors()
for _, field := range b.operator.SelectFields {
keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
Name: field.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: field.FieldContext,
FieldDataType: field.FieldDataType,
})
}
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
// Add selectFields using ColumnExpressionFor since we now have all base table columns
for _, field := range b.operator.SelectFields {
if selectedFields[field.Name] {
@@ -516,44 +518,6 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
}, nil
}
// adjustOperatorKeys runs the same key adjustments as adjustTraceKeys, but on
// the operator's own fields. The operator has a different struct shape than
// QueryBuilderQuery, so we copy the relevant fields into a temp query, run
// the shared helpers, and copy the results back.
func (b *traceOperatorCTEBuilder) adjustOperatorKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, requestType qbtypes.RequestType) {
mergeDeprecatedTraceKeys(keys)
tmp := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Aggregations: b.operator.Aggregations,
SelectFields: b.operator.SelectFields,
GroupBy: b.operator.GroupBy,
Order: b.operator.Order,
}
actions := querybuilder.AdjustKeysForAliasExpressions(&tmp, requestType)
actions = append(actions, querybuilder.AdjustDuplicateKeys(&tmp)...)
for idx := range tmp.SelectFields {
actions = append(actions, adjustTraceKey(&tmp.SelectFields[idx], keys)...)
}
for idx := range tmp.GroupBy {
actions = append(actions, adjustTraceKey(&tmp.GroupBy[idx].TelemetryFieldKey, keys)...)
}
for idx := range tmp.Order {
actions = append(actions, adjustTraceKey(&tmp.Order[idx].Key.TelemetryFieldKey, keys)...)
}
// Copy back the slices the helpers can rewrite.
b.operator.Aggregations = tmp.Aggregations
b.operator.SelectFields = tmp.SelectFields
b.operator.GroupBy = tmp.GroupBy
b.operator.Order = tmp.Order
for _, action := range actions {
b.stmtBuilder.logger.InfoContext(ctx, "key adjustment action", slog.String("action", action))
}
}
func (b *traceOperatorCTEBuilder) getKeySelectors() []*telemetrytypes.FieldKeySelector {
var keySelectors []*telemetrytypes.FieldKeySelector
@@ -581,15 +545,6 @@ func (b *traceOperatorCTEBuilder) getKeySelectors() []*telemetrytypes.FieldKeySe
})
}
for _, sf := range b.operator.SelectFields {
keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
Name: sf.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: sf.FieldContext,
FieldDataType: sf.FieldDataType,
})
}
for i := range keySelectors {
keySelectors[i].Signal = telemetrytypes.SignalTraces
}
@@ -597,7 +552,7 @@ func (b *traceOperatorCTEBuilder) getKeySelectors() []*telemetrytypes.FieldKeySe
return keySelectors
}
func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select(fmt.Sprintf(
@@ -605,6 +560,12 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
int64(b.operator.StepInterval.Seconds()),
))
keySelectors := b.getKeySelectors()
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
var allGroupByArgs []any
for _, gb := range b.operator.GroupBy {
@@ -683,7 +644,8 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
combinedArgs := append(allGroupByArgs, allAggChArgs...)
// Add HAVING clause if specified
if err := b.addHavingClause(sb); err != nil {
err = b.addHavingClause(sb)
if err != nil {
return nil, err
}
@@ -710,11 +672,17 @@ func (b *traceOperatorCTEBuilder) buildTraceSummaryCTE(selectFromCTE string) {
b.addCTE("trace_summary", sql, args, []string{"all_spans", selectFromCTE})
}
func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
b.buildTraceSummaryCTE(selectFromCTE)
sb := sqlbuilder.NewSelectBuilder()
keySelectors := b.getKeySelectors()
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
var allGroupByArgs []any
for _, gb := range b.operator.GroupBy {
@@ -796,7 +764,8 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
sb.GroupBy(groupByKeys...)
}
if err := b.addHavingClause(sb); err != nil {
err = b.addHavingClause(sb)
if err != nil {
return nil, err
}
@@ -852,9 +821,15 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
}, nil
}
func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFromCTE string, keys map[string][]*telemetrytypes.TelemetryFieldKey) (*qbtypes.Statement, error) {
func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFromCTE string) (*qbtypes.Statement, error) {
sb := sqlbuilder.NewSelectBuilder()
keySelectors := b.getKeySelectors()
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
var allGroupByArgs []any
for _, gb := range b.operator.GroupBy {
@@ -936,7 +911,8 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
combinedArgs := append(allGroupByArgs, allAggChArgs...)
// Add HAVING clause if specified
if err := b.addHavingClause(sb); err != nil {
err = b.addHavingClause(sb)
if err != nil {
return nil, err
}

View File

@@ -14,25 +14,6 @@ import (
"github.com/stretchr/testify/require"
)
func newTestTraceOperatorStatementBuilder(t *testing.T) *traceOperatorStatementBuilder {
t.Helper()
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
traceStmtBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore, fm, cb, aggExprRewriter, nil, fl, false, 100000,
)
return NewTraceOperatorStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore, fm, cb, traceStmtBuilder, aggExprRewriter, fl,
)
}
func TestTraceOperatorStatementBuilder(t *testing.T) {
cases := []struct {
name string
@@ -482,7 +463,32 @@ func TestTraceOperatorStatementBuilder(t *testing.T) {
},
}
statementBuilder := newTestTraceOperatorStatementBuilder(t)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
traceStmtBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
nil,
fl,
)
statementBuilder := NewTraceOperatorStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
traceStmtBuilder,
aggExprRewriter,
fl,
)
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -573,7 +579,32 @@ func TestTraceOperatorStatementBuilderErrors(t *testing.T) {
},
}
statementBuilder := newTestTraceOperatorStatementBuilder(t)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
fl := flaggertest.New(t)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil, fl)
traceStmtBuilder := NewTraceQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
nil,
fl,
)
statementBuilder := NewTraceOperatorStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
traceStmtBuilder,
aggExprRewriter,
fl,
)
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -595,142 +626,3 @@ func TestTraceOperatorStatementBuilderErrors(t *testing.T) {
})
}
}
func TestTraceOperatorStatementBuilderAdjustsKeys(t *testing.T) {
cases := []struct {
name string
requestType qbtypes.RequestType
operator qbtypes.QueryBuilderTraceOperator
builderFilter string
wantSQL string
wantArgs []any
}{
{
name: "deprecated duration filter in referenced builder query",
requestType: qbtypes.RequestTypeRaw,
operator: qbtypes.QueryBuilderTraceOperator{
Expression: "A",
Limit: 10,
},
builderFilter: "durationNano = '3s'",
wantSQL: "duration_nano = ?",
wantArgs: []any{int64(3000000000)},
},
{
name: "context-prefixed aggregation alias in order by",
requestType: qbtypes.RequestTypeScalar,
operator: qbtypes.QueryBuilderTraceOperator{
Expression: "A",
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
Alias: "span.count_",
},
},
Order: []qbtypes.OrderBy{
{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "count_",
FieldContext: telemetrytypes.FieldContextSpan,
},
},
Direction: qbtypes.OrderDirectionDesc,
},
},
},
wantSQL: "ORDER BY __result_0 desc",
},
}
statementBuilder := newTestTraceOperatorStatementBuilder(t)
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
err := c.operator.ParseExpression()
require.NoError(t, err)
filter := c.builderFilter
if filter == "" {
filter = "service.name = 'frontend'"
}
q, err := statementBuilder.Build(
context.Background(),
1747947419000,
1747983448000,
c.requestType,
c.operator,
&qbtypes.CompositeQuery{
Queries: []qbtypes.QueryEnvelope{
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Name: "A",
Signal: telemetrytypes.SignalTraces,
Filter: &qbtypes.Filter{Expression: filter},
},
},
},
},
)
require.NoError(t, err)
require.Contains(t, q.Query, c.wantSQL)
for _, arg := range c.wantArgs {
require.Contains(t, q.Args, arg)
}
})
}
}
// TestTraceOperatorStatementBuilderDeduplicatesKeys checks that a trace
// operator with the same field name listed twice in GroupBy (once with a
// context, once without) ends up with a single column in the outer SELECT
// and a single entry in GROUP BY.
func TestTraceOperatorStatementBuilderDeduplicatesKeys(t *testing.T) {
statementBuilder := newTestTraceOperatorStatementBuilder(t)
operator := qbtypes.QueryBuilderTraceOperator{
Expression: "A",
Aggregations: []qbtypes.TraceAggregation{
{Expression: "count()"},
},
GroupBy: []qbtypes.GroupByKey{
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "http.method",
FieldContext: telemetrytypes.FieldContextAttribute,
}},
// Same name, no context — should be merged with the entry above.
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "http.method",
}},
},
}
require.NoError(t, operator.ParseExpression())
q, err := statementBuilder.Build(
context.Background(),
1747947419000,
1747983448000,
qbtypes.RequestTypeScalar,
operator,
&qbtypes.CompositeQuery{
Queries: []qbtypes.QueryEnvelope{
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Name: "A",
Signal: telemetrytypes.SignalTraces,
Filter: &qbtypes.Filter{Expression: "service.name = 'frontend'"},
},
},
},
},
)
require.NoError(t, err)
require.Contains(t, q.Query,
"SELECT toString(multiIf(mapContains(attributes_string, 'http.method') = ?, attributes_string['http.method'], NULL)) AS `http.method`, count() AS __result_0 FROM A GROUP BY `http.method` ORDER BY __result_0 DESC")
}

View File

@@ -4,26 +4,24 @@ import (
"context"
"strings"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/assert"
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
"github.com/stretchr/testify/require"
)
func TestTraceTimeRangeOptimization(t *testing.T) {
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
mockMetadataStore.KeysMap["trace_id"] = []*telemetrytypes.TelemetryFieldKey{{
Name: "trace_id",
FieldContext: telemetrytypes.FieldContextSpan,
@@ -46,10 +44,8 @@ func TestTraceTimeRangeOptimization(t *testing.T) {
fm,
cb,
aggExprRewriter,
nil, // telemetryStore is nil - adaptive path is disabled
nil, // telemetryStore is nil - optimization won't happen but code path is tested
fl,
false,
100000,
)
tests := []struct {

View File

@@ -1,119 +0,0 @@
package querybuildertypesv5
import (
"slices"
"sort"
"strconv"
"time"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// SelectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
// Logic:
// - Finds the latest base evolution (<= tsStartTime) across ALL columns
// - Rejects all evolutions before this latest base evolution
// - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
// - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
// - Results are sorted by ReleaseTime descending (newest first)
func SelectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
copy(sortedEvolutions, evolutions)
// sort the evolutions by ReleaseTime ascending
sort.Slice(sortedEvolutions, func(i, j int) bool {
return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
})
tsStartTime := time.Unix(0, int64(tsStart))
tsEndTime := time.Unix(0, int64(tsEnd))
// Build evolution map: column name -> evolution
evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
for _, evolution := range sortedEvolutions {
if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
// since if there is duplicate we would just use the oldest one.
continue
}
evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
}
// Find the latest base evolution (<= tsStartTime) across ALL columns
// Evolutions are sorted, so we can break early
var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
for _, evolution := range sortedEvolutions {
if evolution.ReleaseTime.After(tsStartTime) {
break
}
latestBaseEvolutionAcrossAll = evolution
}
// We shouldn't reach this, it basically means there is something wrong with the evolutions data
if latestBaseEvolutionAcrossAll == nil {
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
}
columnLookUpMap := make(map[string]*schema.Column)
for _, column := range columns {
columnLookUpMap[column.Name] = column
}
// Collect column-evolution pairs
type colEvoPair struct {
column *schema.Column
evolution *telemetrytypes.EvolutionEntry
}
pairs := []colEvoPair{}
for _, evolution := range evolutionMap {
// Reject evolutions before the latest base evolution
if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
continue
}
// skip evolutions after tsEndTime
if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
continue
}
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
}
pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
}
// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
if len(pairs) == 0 {
for _, column := range columns {
// Use latestBaseEvolutionAcrossAll if this column name matches its column name
if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
}
}
}
// Sort by ReleaseTime descending (newest first)
slices.SortFunc(pairs, func(a, b colEvoPair) int {
// Sort by ReleaseTime descending (newest first)
if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
return -1
}
if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
return 1
}
return 0
})
// Extract results
newColumns := make([]*schema.Column, len(pairs))
evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
for i, pair := range pairs {
newColumns[i] = pair.column
evolutionsEntries[i] = pair.evolution
}
return newColumns, evolutionsEntries, nil
}

View File

@@ -1,414 +0,0 @@
package querybuildertypesv5
import (
"testing"
"time"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
LogsV2BodyV2Column = "body_v2"
LogsV2BodyPromotedColumn = "body_promoted"
)
var (
resources_string = &schema.Column{Name: "resources_string", Type: schema.MapColumnType{
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
ValueType: schema.ColumnTypeString,
}}
resource = &schema.Column{Name: "resource", Type: schema.JSONColumnType{}}
attributes_string = &schema.Column{Name: "attributes_string", Type: schema.MapColumnType{
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
ValueType: schema.ColumnTypeString,
}}
body_v2 = &schema.Column{Name: LogsV2BodyV2Column, Type: schema.JSONColumnType{}}
body_promoted = &schema.Column{Name: LogsV2BodyPromotedColumn, Type: schema.JSONColumnType{}}
)
func TestSelectEvolutionsForColumns(t *testing.T) {
testCases := []struct {
name string
columns []*schema.Column
evolutions []*telemetrytypes.EvolutionEntry
tsStart uint64
tsEnd uint64
expectedColumns []string // column names
expectedEvols []string // evolution column names
expectedError bool
errorStr string
}{
{
name: "New evolutions at tsStartTime - should include latest evolution",
columns: []*schema.Column{
resources_string,
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resource"},
expectedEvols: []string{"resource"},
},
{
name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
columns: []*schema.Column{
resources_string,
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
expectedEvols: []string{"resource", "resources_string"},
},
{
name: "Columns without matching evolutions - should exclude them",
columns: []*schema.Column{
resources_string,
resource, // no evolution for this
attributes_string, // no evolution for this
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"},
expectedEvols: []string{"resources_string"},
},
{
name: "New evolutions at tsEndTime - should not include new evolution",
columns: []*schema.Column{
resources_string,
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"},
expectedEvols: []string{"resources_string"},
},
{
name: "New evolutions after tsEndTime - should exclude new",
columns: []*schema.Column{
resources_string,
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"},
expectedEvols: []string{"resources_string"},
},
{
name: "Empty columns array",
columns: []*schema.Column{},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{},
expectedEvols: []string{},
expectedError: true,
errorStr: "column resources_string not found",
},
{
name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
columns: []*schema.Column{
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resource"},
expectedEvols: []string{"resource"}, // should use first one (older)
},
{
name: "Genuine Duplicate evolutions with new version- should consider both",
columns: []*schema.Column{
resources_string,
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 0,
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 1,
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
Version: 2,
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string", "resource"},
expectedEvols: []string{"resources_string", "resource"}, // should use first one (older)
},
{
name: "Evolution exactly at tsEndTime",
columns: []*schema.Column{
resources_string,
resource,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resources_string",
ColumnType: "Map(LowCardinality(String), String)",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: "resource",
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextResource,
FieldName: "__all__",
ReleaseTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
expectedEvols: []string{"resources_string"},
},
{
name: "Single evolution after tsStartTime - JSON body",
columns: []*schema.Column{
body_v2,
body_promoted,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyV2Column,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "__all__",
ReleaseTime: time.Unix(0, 0),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "user.name",
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
expectedEvols: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
},
{
name: "No evolution after tsStartTime - JSON body",
columns: []*schema.Column{
body_v2,
body_promoted,
},
evolutions: []*telemetrytypes.EvolutionEntry{
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyV2Column,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "__all__",
ReleaseTime: time.Unix(0, 0),
},
{
Signal: telemetrytypes.SignalLogs,
ColumnName: LogsV2BodyPromotedColumn,
ColumnType: "JSON()",
FieldContext: telemetrytypes.FieldContextBody,
FieldName: "user.name",
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
},
},
tsStart: uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedColumns: []string{LogsV2BodyPromotedColumn},
expectedEvols: []string{LogsV2BodyPromotedColumn},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
resultColumns, resultEvols, err := SelectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
if tc.expectedError {
assert.Contains(t, err.Error(), tc.errorStr)
} else {
require.NoError(t, err)
assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
resultColumnNames := make([]string, len(resultColumns))
for i, col := range resultColumns {
resultColumnNames[i] = col.Name
}
resultEvolNames := make([]string, len(resultEvols))
for i, evol := range resultEvols {
resultEvolNames[i] = evol.ColumnName
}
for i := range tc.expectedColumns {
assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
}
for i := range tc.expectedEvols {
assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
}
// Verify sorting: should be descending by ReleaseTime
for i := 0; i < len(resultEvols)-1; i++ {
assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
"evolutions should be sorted descending by ReleaseTime")
}
}
})
}
}

View File

@@ -1,9 +0,0 @@
package querybuildertypesv5
type ResourceFilterResolveKind int
const (
ResourceFilterResolveKindNoOp ResourceFilterResolveKind = iota
ResourceFilterResolveKindUseCTE
ResourceFilterResolveKindFallback
)

View File

@@ -54,12 +54,6 @@ type Event struct {
IsError bool `json:"isError,omitempty"`
}
type OtelSpanRef struct {
TraceId string `json:"traceId,omitempty"`
SpanId string `json:"spanId,omitempty"`
RefType string `json:"refType,omitempty"`
}
// WaterfallSpan represents the span in waterfall response,
// this uses snake_case keys for response as a special case since these
// keys can be directly used to query spans and client need to know the actual fields.
@@ -80,7 +74,6 @@ type WaterfallSpan struct {
TimeUnix uint64 `json:"time_unix"`
TraceID string `json:"trace_id"`
TraceState string `json:"trace_state"`
References []OtelSpanRef `json:"references" required:"true" nullable:"false"`
// Calculated fields https://signoz.io/docs/traces-management/guides/derived-fields-spans
DBName string `json:"db_name,omitempty"`
@@ -135,7 +128,6 @@ type StorableSpan struct {
ExternalHTTPMethod string `ch:"external_http_method"`
ExternalHTTPURL string `ch:"external_http_url"`
ResponseStatusCode string `ch:"response_status_code"`
References string `ch:"references"`
}
// MinimalSpan with only the fields needed to build the parent-child tree.
@@ -293,14 +285,6 @@ func (item *StorableSpan) UnmarshalledEvents() []Event {
return events
}
func (item *StorableSpan) UnmarshalledRefs() []OtelSpanRef {
refs := []OtelSpanRef{}
if err := json.Unmarshal([]byte(item.References), &refs); err != nil {
return nil // skip malformed values
}
return refs
}
func (item *StorableSpan) ToWaterfallSpan(traceID string) *WaterfallSpan {
resources := make(map[string]string)
maps.Copy(resources, item.ResourcesString)
@@ -334,7 +318,6 @@ func (item *StorableSpan) ToWaterfallSpan(traceID string) *WaterfallSpan {
Children: make([]*WaterfallSpan, 0),
TimeUnix: uint64(item.StartTime.UnixNano()),
ServiceName: item.ServiceName,
References: item.UnmarshalledRefs(),
}
}

View File

@@ -24,7 +24,7 @@ type SettingsConfig struct {
Posthog PosthogConfig `mapstructure:"posthog"`
Appcues AppcuesConfig `mapstructure:"appcues"`
Sentry SentryConfig `mapstructure:"sentry"`
Pylon PylonConfig `mapstructure:"pylon"`
Pylon PylonConfig `mapstructure:"sentry"`
}
type PosthogConfig struct {
@@ -54,16 +54,16 @@ func newConfig() factory.Config {
Directory: "/etc/signoz/web",
Settings: SettingsConfig{
Posthog: PosthogConfig{
Enabled: false,
Enabled: true,
},
Appcues: AppcuesConfig{
Enabled: false,
Enabled: true,
},
Sentry: SentryConfig{
Enabled: false,
Enabled: true,
},
Pylon: PylonConfig{
Enabled: false,
Enabled: true,
},
},
}

View File

@@ -43,42 +43,3 @@ func TestNewWithEnvProvider(t *testing.T) {
assert.Equal(t, expected, actual)
}
func TestSettingsConfigWithEnvProvider(t *testing.T) {
testCases := []struct {
name string
env string
expected SettingsConfig
}{
{name: "posthog", env: "SIGNOZ_WEB_SETTINGS_POSTHOG_ENABLED", expected: SettingsConfig{Posthog: PosthogConfig{Enabled: true}}},
{name: "appcues", env: "SIGNOZ_WEB_SETTINGS_APPCUES_ENABLED", expected: SettingsConfig{Appcues: AppcuesConfig{Enabled: true}}},
{name: "sentry", env: "SIGNOZ_WEB_SETTINGS_SENTRY_ENABLED", expected: SettingsConfig{Sentry: SentryConfig{Enabled: true}}},
{name: "pylon", env: "SIGNOZ_WEB_SETTINGS_PYLON_ENABLED", expected: SettingsConfig{Pylon: PylonConfig{Enabled: true}}},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
t.Setenv(testCase.env, "true")
conf, err := config.New(
context.Background(),
config.ResolverConfig{
Uris: []string{"env:"},
ProviderFactories: []config.ProviderFactory{
envprovider.NewFactory(),
},
},
[]factory.ConfigFactory{
NewConfigFactory(),
},
)
require.NoError(t, err)
actual := &Config{}
err = conf.Unmarshal("web", actual)
require.NoError(t, err)
assert.Equal(t, testCase.expected, actual.Settings)
})
}
}

View File

@@ -449,21 +449,6 @@ def index_series_by_label(
return series_by_label
def assert_grouped_series(
series_by_group: dict[str, dict],
expected_values_by_group: dict[str, dict[int, int]],
) -> None:
assert set(series_by_group.keys()) == set(expected_values_by_group.keys())
for group_name, expected_by_ts in expected_values_by_group.items():
actual_values = sorted(
series_by_group[group_name]["values"],
key=lambda value: value["timestamp"],
)
expected_values = [{"timestamp": timestamp, "value": value} for timestamp, value in sorted(expected_by_ts.items())]
assert actual_values == expected_values
def find_named_result(
results: list[dict[str, Any]],
name: str,
@@ -474,57 +459,6 @@ def find_named_result(
)
def assert_scalar_value(
response: requests.Response,
name: str,
expected: Any,
*,
row: int = 0,
col: int = 0,
) -> None:
"""Assert that the named scalar result has `expected` at data[row][col]."""
result = find_named_result(response.json()["data"]["data"]["results"], name)
assert result is not None, f"no result for query {name}"
assert result["data"][row][col] == expected, f"expected {expected} at [{row}][{col}], got {result['data'][row][col]}"
def assert_grouped_scalar(
response: requests.Response,
name: str,
*,
expected_groups: int,
expected_columns: int,
last_col_value: Any | None = None,
) -> None:
"""Assert grouped scalar result has the expected column count and group count.
If `last_col_value` is set and there is exactly one group, also assert the
last column of that single row equals it (a common aggregation-value check)."""
result = find_named_result(response.json()["data"]["data"]["results"], name)
assert result is not None, f"no result for query {name}"
columns = result["columns"]
rows = result["data"]
assert len(columns) == expected_columns, f"expected {expected_columns} columns, got {len(columns)}: {columns}"
assert len(rows) == expected_groups, f"expected {expected_groups} groups, got {len(rows)}: {rows}"
if last_col_value is not None and expected_groups == 1:
assert rows[0][-1] == last_col_value, f"expected last col {last_col_value}, got row {rows[0]}"
def assert_raw_row_subset(
response: requests.Response,
name: str,
expected: dict[str, Any],
*,
row: int = 0,
) -> None:
"""Assert that the named raw result's rows[row]['data'] is a superset of `expected`."""
result = find_named_result(response.json()["data"]["data"]["results"], name)
assert result is not None, f"no result for query {name}"
rows = result["rows"]
assert rows is not None, f"no rows for query {name}"
data = rows[row]["data"]
assert expected.items() <= data.items(), f"expected subset {expected}, got data {data}"
def build_scalar_query(
name: str,
signal: str,
@@ -713,28 +647,6 @@ def assert_identical_query_response(response1: requests.Response, response2: req
assert response1.json()["data"]["data"]["results"] == response2.json()["data"]["data"]["results"], "Response data do not match"
# we already create the evolution for resource during schema migration
# since we have to create test data around it, we need to get the evolution time
def get_resource_evolution_time(signoz: types.SigNoz, signal: str) -> datetime:
result = signoz.telemetrystore.conn.query(
"""
SELECT release_time
FROM signoz_metadata.distributed_column_evolution_metadata
WHERE signal = %(signal)s
AND field_context = 'resource'
AND field_name = '__all__'
AND column_name = 'resource'
LIMIT 1
""",
parameters={"signal": signal},
).result_rows
assert result, f"Expected {signal} resource evolution metadata to exist"
release_time_ns = int(result[0][0])
return datetime.fromtimestamp(release_time_ns / 1e9, tz=UTC)
def generate_logs_with_corrupt_metadata() -> list[Logs]:
"""
Specifically, entries with 'id', 'timestamp', 'severity_text', 'severity_number' and 'body' fields in metadata

View File

@@ -6,7 +6,7 @@ import uuid
from abc import ABC
from collections.abc import Callable, Generator
from enum import Enum
from typing import Any, Literal
from typing import Any
from urllib.parse import urlparse
import numpy as np
@@ -236,7 +236,6 @@ class Traces(ABC):
attributes_number: dict[str, np.float64]
attributes_bool: dict[str, bool]
resources_string: dict[str, str]
resource_json: dict[str, str]
events: list[str]
links: str
response_status_code: str
@@ -274,7 +273,6 @@ class Traces(ABC):
links: list[TracesLink] = [],
trace_state: str = "",
flags: np.uint32 = 0,
resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
) -> None:
if timestamp is None:
timestamp = datetime.datetime.now()
@@ -324,11 +322,8 @@ class Traces(ABC):
self.db_name = ""
self.db_operation = ""
# Process resources and derive service_name. Spans written before the
# JSON-resource evolution time only populate resources_string (legacy_only);
# spans at or after the evolution time dual-write to both columns.
# Process resources and derive service_name
self.resources_string = {k: str(v) for k, v in resources.items()}
self.resource_json = {} if resource_write_mode == "legacy_only" else dict(self.resources_string)
self.service_name = self.resources_string.get("service.name", "default-service")
for k, v in self.resources_string.items():
@@ -580,7 +575,7 @@ class Traces(ABC):
self.db_operation,
self.has_error,
self.is_remote,
self.resource_json,
self.resources_string,
],
dtype=object,
)

View File

@@ -1,20 +1,39 @@
from collections.abc import Callable
from datetime import datetime, timedelta
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import (
assert_grouped_series,
build_group_by_field,
build_logs_aggregation,
get_resource_evolution_time,
index_series_by_label,
make_query_request,
)
# we already create the evolution for resource during schema migration
# since we have to create test data around it, we need to get the evolution time
def _get_logs_resource_evolution_time_json(signoz: types.SigNoz) -> datetime:
result = signoz.telemetrystore.conn.query(
"""
SELECT release_time
FROM signoz_metadata.distributed_column_evolution_metadata
WHERE signal = 'logs'
AND field_context = 'resource'
AND field_name = '__all__'
AND column_name = 'resource'
LIMIT 1
"""
).result_rows
assert result, "Expected logs resource evolution metadata to exist"
release_time_ns = int(result[0][0])
return datetime.fromtimestamp(release_time_ns / 1e9, tz=UTC)
# Logs with timestamps before the evolution time will have resources written only to resources_string.
# Logs with timestamps at or after the evolution time will have resources written to both resources_string and resource_json.
def _build_evolved_log(
@@ -78,6 +97,21 @@ def _query_grouped_log_series(
return index_series_by_label(aggregations[0]["series"], group_by)
def _assert_grouped_series(
series_by_group: dict[str, dict],
expected_values_by_group: dict[str, dict[int, int]],
) -> None:
assert set(series_by_group.keys()) == set(expected_values_by_group.keys())
for group_name, expected_by_ts in expected_values_by_group.items():
actual_values = sorted(
series_by_group[group_name]["values"],
key=lambda value: value["timestamp"],
)
expected_values = [{"timestamp": timestamp, "value": value} for timestamp, value in sorted(expected_by_ts.items())]
assert actual_values == expected_values
def _test_logs_resource_evolution(
signoz: types.SigNoz,
token: str,
@@ -91,7 +125,7 @@ def _test_logs_resource_evolution(
# 5. Query the logs after the evolution time.
# Both aggregation and group by should be checked.
"""
evolution_time = get_resource_evolution_time(signoz, "logs")
evolution_time = _get_logs_resource_evolution_time_json(signoz)
evolution_time = evolution_time.replace(second=0, microsecond=0)
before_2 = evolution_time - timedelta(minutes=10)
@@ -129,7 +163,7 @@ def _test_logs_resource_evolution(
)
before_series = _query_grouped_log_series(signoz, token, before_2 - timedelta(minutes=1), before_1 + timedelta(minutes=1))
assert_grouped_series(
_assert_grouped_series(
before_series,
expected_values_by_group={
"svc-before-2": {
@@ -142,7 +176,7 @@ def _test_logs_resource_evolution(
)
after_series = _query_grouped_log_series(signoz, token, after_1 - timedelta(minutes=1), after_2 + timedelta(minutes=1))
assert_grouped_series(
_assert_grouped_series(
after_series,
expected_values_by_group={
"svc-after-1": {
@@ -155,7 +189,7 @@ def _test_logs_resource_evolution(
)
spanning_series = _query_grouped_log_series(signoz, token, before_2, after_2 + timedelta(minutes=1))
assert_grouped_series(
_assert_grouped_series(
spanning_series,
expected_values_by_group={
"svc-before-2": {
@@ -182,7 +216,7 @@ def _test_logs_resource_evolution(
group_by="deployment.environment",
aggregation="count_distinct(service.name)",
)
assert_grouped_series(
_assert_grouped_series(
aggregation_series,
expected_values_by_group={
"integration": {

View File

@@ -1,206 +0,0 @@
from collections.abc import Callable
from datetime import datetime, timedelta
from http import HTTPStatus
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
assert_grouped_series,
build_group_by_field,
build_logs_aggregation,
get_resource_evolution_time,
index_series_by_label,
make_query_request,
)
from fixtures.traces import TraceIdGenerator, Traces
# Spans with timestamps before the evolution time will have resources written only to resources_string.
# Spans with timestamps at or after the evolution time will have resources written to both resources_string and resource (JSON).
def _build_evolved_span(
timestamp: datetime,
evolution_time: datetime,
service_name: str,
name: str,
) -> Traces:
resource_write_mode = "legacy_only" if timestamp < evolution_time else "dual_write"
return Traces(
timestamp=timestamp,
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name=name,
resources={
"service.name": service_name,
"deployment.environment": "integration",
},
resource_write_mode=resource_write_mode,
)
def _query_grouped_trace_series(
signoz: types.SigNoz,
token: str,
start: datetime,
end: datetime,
group_by: str = "service.name",
aggregation: str = "count()",
) -> dict[str, list[dict]]:
response = make_query_request(
signoz,
token,
start_ms=int(start.timestamp() * 1000),
end_ms=int(end.timestamp() * 1000),
request_type="time_series",
queries=[
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "traces",
"stepInterval": 60,
"disabled": False,
"groupBy": [build_group_by_field(group_by)],
"having": {"expression": ""},
"aggregations": [build_logs_aggregation(aggregation)],
},
}
],
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
results = response.json()["data"]["data"]["results"]
assert len(results) == 1
aggregations = results[0]["aggregations"]
assert len(aggregations) == 1
return index_series_by_label(aggregations[0]["series"], group_by)
def _test_traces_resource_evolution(
signoz: types.SigNoz,
token: str,
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""
# 1. Get the evolution time.
# 2. Ingest spans before the evolution time.
# 3. Ingest spans after the evolution time.
# 4. Query the spans before the evolution time.
# 5. Query the spans after the evolution time.
# Both aggregation and group by should be checked.
"""
evolution_time = get_resource_evolution_time(signoz, "traces")
evolution_time = evolution_time.replace(second=0, microsecond=0)
before_2 = evolution_time - timedelta(minutes=10)
before_1 = evolution_time - timedelta(minutes=5)
after_1 = evolution_time + timedelta(minutes=5)
after_2 = evolution_time + timedelta(minutes=10)
insert_traces(
[
_build_evolved_span(
timestamp=before_2,
evolution_time=evolution_time,
service_name="svc-before-2",
name="span before evolution 2",
),
_build_evolved_span(
timestamp=before_1,
evolution_time=evolution_time,
service_name="svc-before-1",
name="span before evolution 1",
),
_build_evolved_span(
timestamp=after_1,
evolution_time=evolution_time,
service_name="svc-after-1",
name="span after evolution 1",
),
_build_evolved_span(
timestamp=after_2,
evolution_time=evolution_time,
service_name="svc-after-2",
name="span after evolution 2",
),
]
)
before_series = _query_grouped_trace_series(signoz, token, before_2 - timedelta(minutes=1), before_1 + timedelta(minutes=1))
assert_grouped_series(
before_series,
expected_values_by_group={
"svc-before-2": {
int(before_2.timestamp() * 1000): 1,
},
"svc-before-1": {
int(before_1.timestamp() * 1000): 1,
},
},
)
after_series = _query_grouped_trace_series(signoz, token, after_1 - timedelta(minutes=1), after_2 + timedelta(minutes=1))
assert_grouped_series(
after_series,
expected_values_by_group={
"svc-after-1": {
int(after_1.timestamp() * 1000): 1,
},
"svc-after-2": {
int(after_2.timestamp() * 1000): 1,
},
},
)
spanning_series = _query_grouped_trace_series(signoz, token, before_2, after_2 + timedelta(minutes=1))
assert_grouped_series(
spanning_series,
expected_values_by_group={
"svc-before-2": {
int(before_2.timestamp() * 1000): 1,
},
"svc-before-1": {
int(before_1.timestamp() * 1000): 1,
},
"svc-after-1": {
int(after_1.timestamp() * 1000): 1,
},
"svc-after-2": {
int(after_2.timestamp() * 1000): 1,
},
},
)
# query to check aggregation on the resource field like count_distinct(service.name)
aggregation_series = _query_grouped_trace_series(
signoz,
token,
before_2,
after_2 + timedelta(minutes=1),
group_by="deployment.environment",
aggregation="count_distinct(service.name)",
)
assert_grouped_series(
aggregation_series,
expected_values_by_group={
"integration": {
int(before_2.timestamp() * 1000): 1,
int(before_1.timestamp() * 1000): 1,
int(after_1.timestamp() * 1000): 1,
int(after_2.timestamp() * 1000): 1,
},
},
)
def test_traces_resource_evolution(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
_test_traces_resource_evolution(signoz, token, insert_traces)

View File

@@ -25,22 +25,13 @@ returnSpansFrom="A"
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from typing import Any
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
assert_grouped_scalar,
assert_raw_row_subset,
assert_scalar_value,
format_timestamp,
generate_traces_with_corrupt_metadata,
get_rows,
make_query_request,
)
from fixtures.querier import get_rows
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
@@ -443,173 +434,3 @@ def test_trace_operator(
)
assert response.status_code == HTTPStatus.OK, f"HTTP {response.status_code}: {response.text}"
assert case["validate"](response), f"validation failed: {response.json()}"
def _expected_trace_subset(trace: Traces) -> dict[str, Any]:
return {
"duration_nano": trace.duration_nano,
"name": trace.name,
"parent_span_id": trace.parent_span_id,
"span_id": trace.span_id,
"timestamp": format_timestamp(trace.timestamp),
"trace_id": trace.trace_id,
}
@pytest.mark.parametrize(
"payload_factory,request_type,assert_result",
[
# Case 1: CTE filter uses the deprecated intrinsic field `durationNano`.
pytest.param(
lambda traces: [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "traces",
"filter": {"expression": 'durationNano = "3s"'},
},
},
{
"type": "builder_query",
"spec": {
"name": "B",
"signal": "traces",
"filter": {"expression": 'durationNano = "5s"'},
},
},
{
"type": "builder_trace_operator",
"spec": {
"name": "C",
"expression": "A => B",
"limit": 1,
},
},
],
"raw",
lambda response, traces: assert_raw_row_subset(response, "C", _expected_trace_subset(traces[0])),
id="deprecated-intrinsic-filter",
),
# Case 2: CTE filter uses the deprecated calculated field `responseStatusCode`.
pytest.param(
lambda traces: [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "traces",
"filter": {"expression": 'responseStatusCode = "200"'},
},
},
{
"type": "builder_query",
"spec": {
"name": "B",
"signal": "traces",
"filter": {"expression": 'durationNano = "5s"'},
},
},
{
"type": "builder_trace_operator",
"spec": {
"name": "C",
"expression": "A => B",
"limit": 1,
},
},
],
"raw",
lambda response, traces: assert_raw_row_subset(response, "C", _expected_trace_subset(traces[0])),
id="deprecated-calculated-filter",
),
# Case 3: order by uses `count_` with fieldContext `span`, which has
# to be rewritten to the aggregation alias `span.count_`.
pytest.param(
lambda traces: [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "traces",
"aggregations": [{"expression": "count()"}],
},
},
{
"type": "builder_trace_operator",
"spec": {
"name": "C",
"expression": "A",
"aggregations": [{"expression": "count()", "alias": "span.count_"}],
"order": [{"key": {"name": "count_", "fieldContext": "span"}, "direction": "desc"}],
},
},
],
"scalar",
lambda response, traces: assert_scalar_value(response, "C", len(traces)),
id="context-prefixed-aggregation-alias-order",
),
# Case 4: group by lists `cloud.provider` twice (once with a resource
# context, once without).
pytest.param(
lambda traces: [
{
"type": "builder_query",
"spec": {
"name": "A",
"signal": "traces",
"disabled": True,
"aggregations": [{"expression": "count()"}],
},
},
{
"type": "builder_trace_operator",
"spec": {
"name": "C",
"expression": "A",
"aggregations": [{"expression": "count()"}],
"groupBy": [
{"name": "cloud.provider", "fieldContext": "resource"},
{"name": "cloud.provider"},
],
},
},
],
"scalar",
lambda response, traces: assert_grouped_scalar(response, "C", expected_groups=1, expected_columns=2, last_col_value=len(traces)),
id="duplicate-group-by-deduplicated",
),
],
)
def test_trace_operator_with_adjusted_keys(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
payload_factory: Callable[[list[Traces]], list[dict[str, Any]]],
request_type: str,
assert_result: Callable[[requests.Response, list[Traces]], None],
) -> None:
"""
Trace operators build a CTE per referenced builder query and an outer
query on top. Both layers need the same key adjustment as regular trace
queries, otherwise deprecated keys and context-prefixed aliases don't
resolve.
"""
traces = generate_traces_with_corrupt_metadata()
insert_traces(traces)
payload = payload_factory(traces)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
start_ms=int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000),
end_ms=int(datetime.now(tz=UTC).timestamp() * 1000),
request_type=request_type,
queries=payload,
)
assert response.status_code == HTTPStatus.OK, response.text
assert_result(response, traces)

View File

@@ -1,105 +0,0 @@
"""
Transparency check for the skip_resource_fingerprint optimization (traces and logs).
At or above the configured fingerprint threshold the optimization pushes resource
conditions onto the main spans/logs table instead of the fingerprint CTE. That
rewrite must change ClickHouse performance, never the rows: each test runs the same
query against the primary instance (optimization on, threshold=2) and
`signoz_fingerprint` (optimization off) and asserts the responses are identical.
"""
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import (
BuilderQuery,
OrderBy,
TelemetryFieldKey,
assert_identical_query_response,
get_rows,
make_query_request,
)
from fixtures.traces import Traces
def test_skip_resource_fingerprint_traces_fallback_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
) -> None:
"""A >= 2-fingerprint filter drives the fallback path; rows must match the fingerprint baseline."""
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
# 3 distinct services share one env (3 fingerprints > threshold 2 -> fallback);
# the 4th has a different env and must be excluded.
env = {"deployment.environment": "skip-fallback"}
insert_traces(
[
Traces(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-fb-svc-a", **env}),
Traces(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-fb-svc-b", **env}),
Traces(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-fb-svc-c", **env}),
Traces(timestamp=now - timedelta(seconds=7), resources={"service.name": "skip-fb-other", "deployment.environment": "skip-other"}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="traces",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="deployment.environment = 'skip-fallback'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 3
assert_identical_query_response(optimized, fingerprint)
def test_skip_resource_fingerprint_logs_fallback_matches_fingerprint(
signoz: types.SigNoz,
signoz_fingerprint: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
"""A >= 2-fingerprint filter drives the fallback path; rows must match the fingerprint baseline."""
now = datetime.now(tz=UTC)
# 3 distinct services share one env (3 fingerprints > threshold 2 -> fallback);
# the 4th has a different env and must be excluded.
env = {"deployment.environment": "skip-logs-fallback"}
insert_logs(
[
Logs(timestamp=now - timedelta(seconds=10), resources={"service.name": "skip-logs-fb-svc-a", **env}, body="a"),
Logs(timestamp=now - timedelta(seconds=9), resources={"service.name": "skip-logs-fb-svc-b", **env}, body="b"),
Logs(timestamp=now - timedelta(seconds=8), resources={"service.name": "skip-logs-fb-svc-c", **env}, body="c"),
Logs(timestamp=now - timedelta(seconds=7), resources={"service.name": "skip-logs-fb-other", "deployment.environment": "skip-logs-other"}, body="noise"),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
query = BuilderQuery(
signal="logs",
limit=50,
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
filter_expression="deployment.environment = 'skip-logs-fallback'",
select_fields=[TelemetryFieldKey("service.name", "string", "resource"), TelemetryFieldKey("body")],
).to_dict()
start_ms = int((datetime.now(tz=UTC) - timedelta(minutes=5)).timestamp() * 1000)
end_ms = int(datetime.now(tz=UTC).timestamp() * 1000)
optimized = make_query_request(signoz, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
fingerprint = make_query_request(signoz_fingerprint, token, start_ms=start_ms, end_ms=end_ms, request_type="raw", queries=[query])
assert len(get_rows(optimized)) == 3
assert_identical_query_response(optimized, fingerprint)

View File

@@ -1,71 +0,0 @@
import pytest
from testcontainers.core.container import Network
from fixtures import types
from fixtures.signoz import create_signoz
@pytest.fixture(name="signoz", scope="package")
def signoz_skip_resource_fingerprint(
network: Network,
migrator: types.Operation, # pylint: disable=unused-argument
zeus: types.TestContainerDocker,
gateway: types.TestContainerDocker,
sqlstore: types.TestContainerSQL,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.SigNoz:
"""
Package-scoped SigNoz instance with the skip_resource_fingerprint
optimization enabled and a low threshold so the fallback resolver path is
exercised (filters matching >= 2 fingerprints skip the fingerprint CTE).
"""
return create_signoz(
network=network,
zeus=zeus,
gateway=gateway,
sqlstore=sqlstore,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
cache_key="signoz-skip-resource-fingerprint",
env_overrides={
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_ENABLED": True,
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_THRESHOLD": 2,
},
)
@pytest.fixture(name="signoz_fingerprint", scope="package")
def signoz_fingerprint(
network: Network,
migrator: types.Operation, # pylint: disable=unused-argument
zeus: types.TestContainerDocker,
gateway: types.TestContainerDocker,
sqlstore: types.TestContainerSQL,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.SigNoz:
"""
A second SigNoz instance with the optimization disabled, so every query
resolves resource filters through the fingerprint CTE (the path the primary
also takes below the threshold). It shares the same ClickHouse (telemetry)
and sqlstore (users) as the primary instance, which lets a single admin
token authenticate against both and lets the tests diff the optimized
result against this fingerprint baseline for the same query and data.
"""
return create_signoz(
network=network,
zeus=zeus,
gateway=gateway,
sqlstore=sqlstore,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
cache_key="signoz-skip-resource-fingerprint-baseline",
env_overrides={
"SIGNOZ_QUERIER_SKIP__RESOURCE__FINGERPRINT_ENABLED": False,
},
)