mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-19 03:10:40 +01:00
Compare commits
10 Commits
ns/scope
...
tvats-drop
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a6f3d20239 | ||
|
|
73abf12446 | ||
|
|
0f3b3dfb07 | ||
|
|
7e2cd441f2 | ||
|
|
098448330d | ||
|
|
eb01617c15 | ||
|
|
7bcfaab35e | ||
|
|
5b62b31d34 | ||
|
|
f6a9b4b1f6 | ||
|
|
b46f099966 |
@@ -8,12 +8,19 @@ import {
|
||||
|
||||
import ChangelogRenderer from '../components/ChangelogRenderer';
|
||||
|
||||
// Mock react-markdown to just render children as plain text
|
||||
// Mock react-markdown to render children as plain text and a sample
|
||||
// anchor through the `components.a` override
|
||||
jest.mock(
|
||||
'react-markdown',
|
||||
() =>
|
||||
function ReactMarkdown({ children }: any) {
|
||||
return <div>{children}</div>;
|
||||
function ReactMarkdown({ children, components }: any) {
|
||||
const Anchor = components?.a;
|
||||
return (
|
||||
<div>
|
||||
{children}
|
||||
{Anchor && <Anchor href="https://signoz.io/docs">docs</Anchor>}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -62,4 +69,14 @@ describe('ChangelogRenderer', () => {
|
||||
expect(screen.getByAltText('Media')).toBeInTheDocument();
|
||||
expect(screen.getByText('Description for feature 1')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders markdown links that open in a new tab', () => {
|
||||
render(<ChangelogRenderer changelog={mockChangelog} />);
|
||||
const links = screen.getAllByRole('link', { name: 'docs' });
|
||||
expect(links.length).toBeGreaterThan(0);
|
||||
links.forEach((link) => {
|
||||
expect(link).toHaveAttribute('target', '_blank');
|
||||
expect(link).toHaveAttribute('rel', 'noopener noreferrer');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,6 +13,19 @@ interface Props {
|
||||
changelog: ChangelogSchema;
|
||||
}
|
||||
|
||||
interface LinkProps {
|
||||
href?: string;
|
||||
children?: React.ReactNode;
|
||||
}
|
||||
|
||||
function Link({ href, children }: LinkProps): JSX.Element {
|
||||
return (
|
||||
<a href={href} target="_blank" rel="noopener noreferrer">
|
||||
{children}
|
||||
</a>
|
||||
);
|
||||
}
|
||||
|
||||
function renderMedia(media: Media): JSX.Element | null {
|
||||
if (SupportedImageTypes.includes(media.ext)) {
|
||||
return (
|
||||
@@ -62,7 +75,9 @@ function ChangelogRenderer({ changelog }: Props): JSX.Element {
|
||||
<div key={feature.id}>
|
||||
<div className="changelog-renderer-section-title">{feature.title}</div>
|
||||
{feature.media && renderMedia(feature.media)}
|
||||
<ReactMarkdown>{feature.description}</ReactMarkdown>
|
||||
<ReactMarkdown components={{ a: Link }}>
|
||||
{feature.description}
|
||||
</ReactMarkdown>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
@@ -71,7 +86,9 @@ function ChangelogRenderer({ changelog }: Props): JSX.Element {
|
||||
<div className="changelog-renderer-bug-fixes">
|
||||
<div className="changelog-renderer-section-title">Bug Fixes</div>
|
||||
{changelog.bug_fixes && (
|
||||
<ReactMarkdown>{changelog.bug_fixes}</ReactMarkdown>
|
||||
<ReactMarkdown components={{ a: Link }}>
|
||||
{changelog.bug_fixes}
|
||||
</ReactMarkdown>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
@@ -79,7 +96,9 @@ function ChangelogRenderer({ changelog }: Props): JSX.Element {
|
||||
<div className="changelog-renderer-maintenance">
|
||||
<div className="changelog-renderer-section-title">Maintenance</div>
|
||||
{changelog.maintenance && (
|
||||
<ReactMarkdown>{changelog.maintenance}</ReactMarkdown>
|
||||
<ReactMarkdown components={{ a: Link }}>
|
||||
{changelog.maintenance}
|
||||
</ReactMarkdown>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -100,6 +100,7 @@ function LogDetailInner({
|
||||
// Don't close if clicking on drawer content, overlays, or portal elements
|
||||
if (
|
||||
target.closest('[data-log-detail-ignore="true"]') ||
|
||||
target.closest('.log-detail-drawer') ||
|
||||
target.closest('.cm-tooltip-autocomplete') ||
|
||||
target.closest('.drawer-popover') ||
|
||||
target.closest('.query-status-popover') ||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import { renderHook } from '@testing-library/react';
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import MockQueryClientProvider from 'providers/test/MockQueryClientProvider';
|
||||
import { Query } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
import useUpdatedQuery from '../useResolveQuery';
|
||||
|
||||
const mockGetSubstituteVars = jest.fn();
|
||||
const mockDynamicVariables: unknown[] = [];
|
||||
|
||||
jest.mock('api/dashboard/substitute_vars', () => ({
|
||||
getSubstituteVars: (...args: unknown[]): unknown =>
|
||||
mockGetSubstituteVars(...args),
|
||||
}));
|
||||
|
||||
jest.mock('api/v5/v5', () => ({
|
||||
prepareQueryRangePayloadV5: (): { queryPayload: unknown } => ({
|
||||
queryPayload: { start: 0, end: 1 },
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock(
|
||||
'lib/newQueryBuilder/queryBuilderMappers/mapQueryDataFromApi',
|
||||
() => ({
|
||||
mapQueryDataFromApi: (): Query => ({ resolved: true }) as unknown as Query,
|
||||
}),
|
||||
);
|
||||
|
||||
jest.mock('hooks/dashboard/useDashboardVariablesByType', () => ({
|
||||
useDashboardVariablesByType: (): unknown[] => mockDynamicVariables,
|
||||
}));
|
||||
|
||||
jest.mock('react-redux', () => ({
|
||||
...jest.requireActual('react-redux'),
|
||||
useSelector: (): unknown => ({
|
||||
selectedTime: 'GLOBAL_TIME',
|
||||
}),
|
||||
}));
|
||||
|
||||
const QUERY = { builder: { queryData: [] } } as unknown as Query;
|
||||
|
||||
const WIDGET_CONFIG = {
|
||||
query: QUERY,
|
||||
panelTypes: PANEL_TYPES.TIME_SERIES,
|
||||
timePreferance: 'GLOBAL_TIME' as const,
|
||||
};
|
||||
|
||||
describe('useResolveQuery', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockDynamicVariables.length = 0;
|
||||
});
|
||||
|
||||
it('skips the substitute_vars round-trip when there are no variables', async () => {
|
||||
const { result } = renderHook(() => useUpdatedQuery(), {
|
||||
wrapper: MockQueryClientProvider,
|
||||
});
|
||||
|
||||
const resolved = await result.current.getUpdatedQuery({
|
||||
widgetConfig: WIDGET_CONFIG,
|
||||
});
|
||||
|
||||
expect(mockGetSubstituteVars).not.toHaveBeenCalled();
|
||||
expect(resolved).toBe(QUERY);
|
||||
});
|
||||
|
||||
it('resolves through substitute_vars when the dashboard has variables', async () => {
|
||||
mockGetSubstituteVars.mockResolvedValue({
|
||||
httpStatusCode: 200,
|
||||
data: { compositeQuery: {} },
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useUpdatedQuery(), {
|
||||
wrapper: MockQueryClientProvider,
|
||||
});
|
||||
|
||||
const resolved = await result.current.getUpdatedQuery({
|
||||
widgetConfig: WIDGET_CONFIG,
|
||||
dashboardData: {
|
||||
data: {
|
||||
variables: {
|
||||
env: { name: 'env', selectedValue: 'prod' },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(mockGetSubstituteVars).toHaveBeenCalledTimes(1);
|
||||
expect(resolved).toStrictEqual({ resolved: true });
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { useCallback } from 'react';
|
||||
import { useMutation } from 'react-query';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { useSelector } from 'react-redux';
|
||||
import { isEmpty } from 'lodash-es';
|
||||
import { getSubstituteVars } from 'api/dashboard/substitute_vars';
|
||||
import { prepareQueryRangePayloadV5 } from 'api/v5/v5';
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
@@ -46,13 +47,21 @@ function useUpdatedQuery(): UseUpdatedQueryResult {
|
||||
widgetConfig,
|
||||
dashboardData,
|
||||
}: UseUpdatedQueryOptions): Promise<Query> => {
|
||||
const variables = getDashboardVariables(dashboardData?.data?.variables);
|
||||
|
||||
// `/substitute_vars` only rewrites `$variable` references, so on surfaces with no
|
||||
// dashboard behind them (APM, Celery, API monitoring) the round-trip is a no-op.
|
||||
if (isEmpty(variables) && isEmpty(dashboardDynamicVariables)) {
|
||||
return widgetConfig.query;
|
||||
}
|
||||
|
||||
// Prepare query payload with resolved variables
|
||||
const { queryPayload } = prepareQueryRangePayloadV5({
|
||||
query: widgetConfig.query,
|
||||
graphType: getGraphType(widgetConfig.panelTypes),
|
||||
selectedTime: widgetConfig.timePreferance,
|
||||
globalSelectedInterval,
|
||||
variables: getDashboardVariables(dashboardData?.data?.variables),
|
||||
variables,
|
||||
originalGraphType: widgetConfig.panelTypes,
|
||||
dynamicVariables: dashboardDynamicVariables,
|
||||
});
|
||||
|
||||
@@ -37,7 +37,7 @@ import { useInfraMonitoringFontSize } from './useInfraMonitoringTablePreferences
|
||||
import styles from './K8sExpandedRow.module.scss';
|
||||
import { buildExpressionFromGroupMeta } from './utils';
|
||||
import { logInfraColumnSortedEvent } from 'container/InfraMonitoringK8sV2/Base/events';
|
||||
import { getUnstableCurrentSearchParams } from 'container/TopNav/DateTimeSelectionV2/utils/getUnstableCurrentSearchParams';
|
||||
import { getUnstableCurrentSearchParams } from 'utils/getUnstableCurrentSearchParams';
|
||||
import { QueryParams } from 'constants/query';
|
||||
|
||||
const EXPANDED_ROW_LIMIT = 10;
|
||||
|
||||
@@ -9,7 +9,11 @@ function Overview(): JSX.Element {
|
||||
|
||||
return (
|
||||
<div className={styles.overview} data-testid="llm-observability-overview">
|
||||
<DashboardContainer dashboard={dashboard} refetch={refetch} />
|
||||
<DashboardContainer
|
||||
dashboard={dashboard}
|
||||
refetch={refetch}
|
||||
canEditDashboardOverride={false}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"id": "llm-observability-overview",
|
||||
"orgId": "",
|
||||
"locked": true,
|
||||
"locked": false,
|
||||
"name": "AI Observability Overview",
|
||||
"schemaVersion": "v6",
|
||||
"source": "system",
|
||||
@@ -1146,4 +1146,4 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -124,6 +124,9 @@ function Application(): JSX.Element {
|
||||
start: minTime,
|
||||
end: maxTime,
|
||||
}),
|
||||
// the time range is part of the key, so without this every window change blanks the
|
||||
// operations list and the widgets below are rebuilt with an empty `operation in []`
|
||||
keepPreviousData: true,
|
||||
});
|
||||
|
||||
const selectedTraceTags: string = JSON.stringify(
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { renderHook, waitFor } from '@testing-library/react';
|
||||
import { Query } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
import useBaseAggregateOptions from '../useBaseAggregateOptions';
|
||||
|
||||
const mockGetUpdatedQuery = jest.fn();
|
||||
const mockNotificationsError = jest.fn();
|
||||
|
||||
jest.mock('container/GridCardLayout/useResolveQuery', () => ({
|
||||
__esModule: true,
|
||||
default: (): unknown => ({
|
||||
getUpdatedQuery: mockGetUpdatedQuery,
|
||||
isLoading: false,
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock('hooks/useNotifications', () => ({
|
||||
useNotifications: (): unknown => ({
|
||||
notifications: { error: mockNotificationsError },
|
||||
}),
|
||||
}));
|
||||
|
||||
jest.mock('providers/Dashboard/store/useDashboardStore', () => ({
|
||||
useDashboardStore: (): unknown => ({ dashboardData: undefined }),
|
||||
}));
|
||||
|
||||
jest.mock('hooks/dashboard/useContextVariables', () => ({
|
||||
__esModule: true,
|
||||
default: (): unknown => ({ processedVariables: {} }),
|
||||
}));
|
||||
|
||||
jest.mock('hooks/useSafeNavigate', () => ({
|
||||
useSafeNavigate: (): unknown => ({ safeNavigate: jest.fn() }),
|
||||
}));
|
||||
|
||||
jest.mock('react-router-dom', () => ({
|
||||
...jest.requireActual('react-router-dom'),
|
||||
useLocation: (): { pathname: string } => ({ pathname: '/services/socky-api' }),
|
||||
}));
|
||||
|
||||
const QUERY = {
|
||||
builder: {
|
||||
queryData: [{ queryName: 'A', dataSource: 'traces', aggregations: [] }],
|
||||
},
|
||||
} as unknown as Query;
|
||||
|
||||
const AGGREGATE_DATA = { queryName: 'A', filters: [] };
|
||||
|
||||
const renderOptions = (): ReturnType<typeof renderHook> =>
|
||||
renderHook(() =>
|
||||
useBaseAggregateOptions({
|
||||
query: QUERY,
|
||||
onClose: jest.fn(),
|
||||
subMenu: '',
|
||||
setSubMenu: jest.fn(),
|
||||
aggregateData: AGGREGATE_DATA,
|
||||
fieldVariables: {},
|
||||
}),
|
||||
);
|
||||
|
||||
describe('useBaseAggregateOptions', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
it('notifies and keeps the unresolved query when variable resolution fails', async () => {
|
||||
mockGetUpdatedQuery.mockRejectedValue(
|
||||
new Error('syntax errors in expression'),
|
||||
);
|
||||
|
||||
renderOptions();
|
||||
|
||||
await waitFor(() =>
|
||||
expect(mockNotificationsError).toHaveBeenCalledWith({
|
||||
message: 'Unable to resolve variables',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not notify when variable resolution succeeds', async () => {
|
||||
mockGetUpdatedQuery.mockResolvedValue(QUERY);
|
||||
|
||||
renderOptions();
|
||||
|
||||
await waitFor(() => expect(mockGetUpdatedQuery).toHaveBeenCalled());
|
||||
expect(mockNotificationsError).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -6,6 +6,7 @@ import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import useUpdatedQuery from 'container/GridCardLayout/useResolveQuery';
|
||||
import { processContextLinks } from 'container/NewWidget/RightContainer/ContextLinks/utils';
|
||||
import useContextVariables from 'hooks/dashboard/useContextVariables';
|
||||
import { useNotifications } from 'hooks/useNotifications';
|
||||
import ContextMenu from 'periscope/components/ContextMenu';
|
||||
import { useDashboardStore } from 'providers/Dashboard/store/useDashboardStore';
|
||||
import { ContextLinksData } from 'types/api/dashboard/getAll';
|
||||
@@ -50,23 +51,25 @@ const useBaseAggregateOptions = ({
|
||||
const { getUpdatedQuery, isLoading: isResolveQueryLoading } =
|
||||
useUpdatedQuery();
|
||||
const { dashboardData } = useDashboardStore();
|
||||
const { notifications } = useNotifications();
|
||||
|
||||
useEffect(() => {
|
||||
if (!aggregateData) {
|
||||
return;
|
||||
}
|
||||
const resolveQuery = async (): Promise<void> => {
|
||||
const updatedQuery = await getUpdatedQuery({
|
||||
widgetConfig: {
|
||||
query,
|
||||
panelTypes: panelType || PANEL_TYPES.TIME_SERIES,
|
||||
timePreferance: 'GLOBAL_TIME',
|
||||
},
|
||||
dashboardData,
|
||||
getUpdatedQuery({
|
||||
widgetConfig: {
|
||||
query,
|
||||
panelTypes: panelType || PANEL_TYPES.TIME_SERIES,
|
||||
timePreferance: 'GLOBAL_TIME',
|
||||
},
|
||||
dashboardData,
|
||||
})
|
||||
.then(setResolvedQuery)
|
||||
.catch(() => {
|
||||
setResolvedQuery(query);
|
||||
notifications.error({ message: 'Unable to resolve variables' });
|
||||
});
|
||||
setResolvedQuery(updatedQuery);
|
||||
};
|
||||
resolveQuery();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [query, aggregateData, panelType]);
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import DateTimeSelection from '../index';
|
||||
import {
|
||||
__resetSearchParamsGetter,
|
||||
__setSearchParamsGetterForTest,
|
||||
} from '../utils/getUnstableCurrentSearchParams';
|
||||
} from 'utils/getUnstableCurrentSearchParams';
|
||||
import { queryClient, TestWrapper } from './testUtils';
|
||||
|
||||
const mockSafeNavigate = jest.fn();
|
||||
|
||||
@@ -10,7 +10,7 @@ import DateTimeSelection from '../index';
|
||||
import {
|
||||
__resetSearchParamsGetter,
|
||||
__setSearchParamsGetterForTest,
|
||||
} from '../utils/getUnstableCurrentSearchParams';
|
||||
} from 'utils/getUnstableCurrentSearchParams';
|
||||
import { queryClient, TestWrapper } from './testUtils';
|
||||
|
||||
const mockSafeNavigate = jest.fn();
|
||||
|
||||
@@ -24,7 +24,7 @@ import DateTimeSelection from '../index';
|
||||
import {
|
||||
__resetSearchParamsGetter,
|
||||
__setSearchParamsGetterForTest,
|
||||
} from '../utils/getUnstableCurrentSearchParams';
|
||||
} from 'utils/getUnstableCurrentSearchParams';
|
||||
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
|
||||
@@ -5,7 +5,7 @@ import DateTimeSelection from '../index';
|
||||
import {
|
||||
__resetSearchParamsGetter,
|
||||
__setSearchParamsGetterForTest,
|
||||
} from '../utils/getUnstableCurrentSearchParams';
|
||||
} from 'utils/getUnstableCurrentSearchParams';
|
||||
import { queryClient, TestWrapper } from './testUtils';
|
||||
|
||||
const mockSafeNavigate = jest.fn();
|
||||
|
||||
@@ -10,7 +10,7 @@ import DateTimeSelection from '../index';
|
||||
import {
|
||||
__resetSearchParamsGetter,
|
||||
__setSearchParamsGetterForTest,
|
||||
} from '../utils/getUnstableCurrentSearchParams';
|
||||
} from 'utils/getUnstableCurrentSearchParams';
|
||||
import { queryClient, TestWrapper, createMockMoment } from './testUtils';
|
||||
|
||||
const mockSafeNavigate = jest.fn();
|
||||
|
||||
@@ -54,7 +54,7 @@ import {
|
||||
Time,
|
||||
TimeRange,
|
||||
} from './types';
|
||||
import { getUnstableCurrentSearchParams } from './utils/getUnstableCurrentSearchParams';
|
||||
import { getUnstableCurrentSearchParams } from 'utils/getUnstableCurrentSearchParams';
|
||||
|
||||
import './DateTimeSelectionV2.styles.scss';
|
||||
|
||||
|
||||
@@ -189,7 +189,8 @@ function DashboardActions({
|
||||
onClick: (): void => void handleClone(),
|
||||
});
|
||||
}
|
||||
if (isAuthor || user.role === USER_ROLES.ADMIN) {
|
||||
|
||||
if (canEditDashboard && (isAuthor || user.role === USER_ROLES.ADMIN)) {
|
||||
dashboardGroup.push({
|
||||
key: 'lock',
|
||||
label: isDashboardLocked ? 'Unlock dashboard' : 'Lock dashboard',
|
||||
|
||||
@@ -46,23 +46,11 @@ beforeAll(() => {
|
||||
});
|
||||
|
||||
// jest.config maps the real hook to a no-op mock; this suite needs real navigation.
|
||||
jest.mock('hooks/useSafeNavigate', () => {
|
||||
const { useHistory } = jest.requireActual('react-router-dom');
|
||||
return {
|
||||
useSafeNavigate: (): unknown => {
|
||||
const history = useHistory();
|
||||
return {
|
||||
safeNavigate: (to: string, opts?: { replace?: boolean }): void => {
|
||||
if (opts?.replace) {
|
||||
history.replace(to);
|
||||
} else {
|
||||
history.push(to);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
jest.mock('hooks/useSafeNavigate', () =>
|
||||
jest
|
||||
.requireActual('tests/browser-history-safe-navigate')
|
||||
.createBrowserHistorySafeNavigateMock(),
|
||||
);
|
||||
|
||||
jest.mock('api/querySuggestions/getKeySuggestions', () => ({
|
||||
getKeySuggestions: jest
|
||||
@@ -204,9 +192,12 @@ function Harness(): JSX.Element {
|
||||
);
|
||||
}
|
||||
|
||||
const INITIAL_ROUTE = '/dashboard/dash-1';
|
||||
|
||||
const renderHarness = (): void => {
|
||||
window.history.replaceState(null, '', INITIAL_ROUTE);
|
||||
render(
|
||||
<MemoryRouter initialEntries={['/dashboard/dash-1']}>
|
||||
<MemoryRouter initialEntries={[INITIAL_ROUTE]}>
|
||||
<CompatRouter>
|
||||
<QueryClientProvider client={new QueryClient()}>
|
||||
<ReduxProvider store={configureStore([])(appStore.getState())}>
|
||||
|
||||
@@ -16,23 +16,11 @@ import ViewPanelModal from '../ViewPanelModal/ViewPanelModal';
|
||||
import { useViewPanel } from '../hooks/useViewPanel';
|
||||
|
||||
// jest.config maps the real hook to a no-op mock; this suite needs real navigation.
|
||||
jest.mock('hooks/useSafeNavigate', () => {
|
||||
const { useHistory } = jest.requireActual('react-router-dom');
|
||||
return {
|
||||
useSafeNavigate: (): unknown => {
|
||||
const history = useHistory();
|
||||
return {
|
||||
safeNavigate: (to: string, opts?: { replace?: boolean }): void => {
|
||||
if (opts?.replace) {
|
||||
history.replace(to);
|
||||
} else {
|
||||
history.push(to);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
jest.mock('hooks/useSafeNavigate', () =>
|
||||
jest
|
||||
.requireActual('tests/browser-history-safe-navigate')
|
||||
.createBrowserHistorySafeNavigateMock(),
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'pages/DashboardPageV2/DashboardContainer/hooks/usePanelQuery',
|
||||
@@ -150,9 +138,12 @@ function Harness(): JSX.Element {
|
||||
);
|
||||
}
|
||||
|
||||
const INITIAL_ROUTE = '/dashboard/dash-1';
|
||||
|
||||
const renderHarness = (): void => {
|
||||
window.history.replaceState(null, '', INITIAL_ROUTE);
|
||||
render(
|
||||
<MemoryRouter initialEntries={['/dashboard/dash-1']}>
|
||||
<MemoryRouter initialEntries={[INITIAL_ROUTE]}>
|
||||
<CompatRouter>
|
||||
<QueryClientProvider client={new QueryClient()}>
|
||||
<ReduxProvider store={configureStore([])(appStore.getState())}>
|
||||
|
||||
@@ -14,23 +14,11 @@ import { useViewPanelMode } from '../ViewPanelModal/useViewPanelMode';
|
||||
import { useViewPanel } from '../hooks/useViewPanel';
|
||||
|
||||
// jest.config maps the real hook to a no-op mock; this suite needs real navigation.
|
||||
jest.mock('hooks/useSafeNavigate', () => {
|
||||
const { useHistory } = jest.requireActual('react-router-dom');
|
||||
return {
|
||||
useSafeNavigate: (): unknown => {
|
||||
const history = useHistory();
|
||||
return {
|
||||
safeNavigate: (to: string, opts?: { replace?: boolean }): void => {
|
||||
if (opts?.replace) {
|
||||
history.replace(to);
|
||||
} else {
|
||||
history.push(to);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
});
|
||||
jest.mock('hooks/useSafeNavigate', () =>
|
||||
jest
|
||||
.requireActual('tests/browser-history-safe-navigate')
|
||||
.createBrowserHistorySafeNavigateMock(),
|
||||
);
|
||||
|
||||
jest.mock(
|
||||
'pages/DashboardPageV2/DashboardContainer/hooks/usePanelQuery',
|
||||
@@ -184,9 +172,12 @@ function Harness(): JSX.Element {
|
||||
);
|
||||
}
|
||||
|
||||
const INITIAL_ROUTE = '/dashboard/dash-1';
|
||||
|
||||
const renderHarness = (): void => {
|
||||
window.history.replaceState(null, '', INITIAL_ROUTE);
|
||||
render(
|
||||
<MemoryRouter initialEntries={['/dashboard/dash-1']}>
|
||||
<MemoryRouter initialEntries={[INITIAL_ROUTE]}>
|
||||
<CompatRouter>
|
||||
<QueryBuilderProvider>
|
||||
<Harness />
|
||||
|
||||
@@ -19,11 +19,20 @@ import { resolveDashboardImage } from 'pages/DashboardPageV2/DashboardContainer/
|
||||
interface DashboardContainerProps {
|
||||
dashboard: DashboardtypesGettableDashboardV2DTO;
|
||||
refetch: () => void;
|
||||
/**
|
||||
* @deprecated
|
||||
* `canEditDashboardOverride` is a temporary solution to allow the dashboard to be view only.
|
||||
* This is only used for LLM Observability.
|
||||
* It will be removed in the future.
|
||||
* TODO: @Ashwin / @Abhi — remove when the final solution is implemented.
|
||||
*/
|
||||
canEditDashboardOverride?: boolean;
|
||||
}
|
||||
|
||||
function DashboardContainer({
|
||||
dashboard,
|
||||
refetch,
|
||||
canEditDashboardOverride,
|
||||
}: DashboardContainerProps): JSX.Element {
|
||||
const spec = dashboard.spec;
|
||||
const image = resolveDashboardImage(dashboard.image);
|
||||
@@ -45,10 +54,11 @@ function DashboardContainer({
|
||||
// Seed during render (not an effect) so the first Panel render already sees the id —
|
||||
// useDashboardFetchRequired throws on a missing id. setEditContext self-guards.
|
||||
const setEditContext = useDashboardStore((s) => s.setEditContext);
|
||||
|
||||
setEditContext({
|
||||
dashboardId: dashboard.id,
|
||||
isLocked,
|
||||
canEditDashboard,
|
||||
canEditDashboard: canEditDashboardOverride ?? canEditDashboard,
|
||||
refetch,
|
||||
});
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ import { OptionsQuery } from 'container/OptionsMenu/types';
|
||||
import { useGetCompositeQueryParam } from 'hooks/queryBuilder/useGetCompositeQueryParam';
|
||||
import { updateStepInterval } from 'hooks/queryBuilder/useStepInterval';
|
||||
import { useSafeNavigate } from 'hooks/useSafeNavigate';
|
||||
import useUrlQuery from 'hooks/useUrlQuery';
|
||||
import { createIdFromObjectFields } from 'lib/createIdFromObjectFields';
|
||||
import { createNewBuilderItemName } from 'lib/newQueryBuilder/createNewBuilderItemName';
|
||||
import { getOperatorsBySourceAndPanelType } from 'lib/newQueryBuilder/getOperatorsBySourceAndPanelType';
|
||||
@@ -66,6 +65,7 @@ import {
|
||||
} from 'types/common/queryBuilder';
|
||||
import { sanitizeOrderByForExplorer } from 'utils/sanitizeOrderBy';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { getUnstableCurrentSearchParams } from 'utils/getUnstableCurrentSearchParams';
|
||||
|
||||
export const QueryBuilderContext = createContext<QueryBuilderContextType>({
|
||||
currentQuery: initialQueriesMap.metrics,
|
||||
@@ -105,7 +105,6 @@ export const QueryBuilderContext = createContext<QueryBuilderContextType>({
|
||||
export function QueryBuilderProvider({
|
||||
children,
|
||||
}: PropsWithChildren): JSX.Element {
|
||||
const urlQuery = useUrlQuery();
|
||||
const location = useLocation();
|
||||
|
||||
const currentPathnameRef = useRef<string | null>(location.pathname);
|
||||
@@ -122,7 +121,7 @@ export function QueryBuilderProvider({
|
||||
null,
|
||||
);
|
||||
|
||||
const panelTypeQueryParams = urlQuery.get(
|
||||
const panelTypeQueryParams = getUnstableCurrentSearchParams().get(
|
||||
QueryParams.panelTypes,
|
||||
) as PANEL_TYPES | null;
|
||||
|
||||
@@ -976,6 +975,7 @@ export function QueryBuilderProvider({
|
||||
unit: query.unit || initialQueryState.unit,
|
||||
};
|
||||
|
||||
const urlQuery = getUnstableCurrentSearchParams();
|
||||
const pagination = urlQuery.get(QueryParams.pagination);
|
||||
|
||||
if (pagination) {
|
||||
@@ -1014,7 +1014,7 @@ export function QueryBuilderProvider({
|
||||
|
||||
safeNavigate(generatedUrl, { newTab });
|
||||
},
|
||||
[location.pathname, safeNavigate, urlQuery],
|
||||
[location.pathname, safeNavigate],
|
||||
);
|
||||
|
||||
const handleSetConfig = useCallback(
|
||||
|
||||
54
frontend/src/tests/browser-history-safe-navigate.ts
Normal file
54
frontend/src/tests/browser-history-safe-navigate.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
// Mock factory for suites that need `useSafeNavigate` to navigate for real.
|
||||
//
|
||||
// `jest.config.ts` maps every `hooks/useSafeNavigate` import to the no-op
|
||||
// `__tests__/safeNavigateMock.ts`, so a suite that drives navigation has to opt
|
||||
// out with its own `jest.mock`.
|
||||
//
|
||||
// In production `safeNavigate` goes through `createBrowserHistory`, which writes
|
||||
// `window.location` as well as notifying the router. `MemoryRouter` never touches
|
||||
// `window`, so anything reading `getUnstableCurrentSearchParams()` sees an empty
|
||||
// search and drops the params the test just navigated with. This mock writes both.
|
||||
//
|
||||
// The `jest.mock` factory is hoisted above imports, so require it inside:
|
||||
//
|
||||
// jest.mock('hooks/useSafeNavigate', () =>
|
||||
// jest
|
||||
// .requireActual('tests/browser-history-safe-navigate')
|
||||
// .createBrowserHistorySafeNavigateMock(),
|
||||
// );
|
||||
|
||||
import type { History } from 'history';
|
||||
|
||||
interface SafeNavigateOptions {
|
||||
replace?: boolean;
|
||||
}
|
||||
|
||||
interface UseSafeNavigateModule {
|
||||
useSafeNavigate: () => {
|
||||
safeNavigate: (to: string, options?: SafeNavigateOptions) => void;
|
||||
};
|
||||
}
|
||||
|
||||
export function createBrowserHistorySafeNavigateMock(): UseSafeNavigateModule {
|
||||
const { useHistory } = jest.requireActual<{ useHistory: () => History }>(
|
||||
'react-router-dom',
|
||||
);
|
||||
|
||||
return {
|
||||
useSafeNavigate: () => {
|
||||
const history = useHistory();
|
||||
|
||||
return {
|
||||
safeNavigate: (to: string, options?: SafeNavigateOptions): void => {
|
||||
if (options?.replace) {
|
||||
window.history.replaceState(null, '', to);
|
||||
history.replace(to);
|
||||
} else {
|
||||
window.history.pushState(null, '', to);
|
||||
history.push(to);
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
|
||||
@@ -186,18 +185,7 @@ func (n *Notifier) Notify(ctx context.Context, alerts ...*types.Alert) (bool, er
|
||||
}
|
||||
}
|
||||
|
||||
// Thread same-rule alerts together: threadKey is a stable hash of the
|
||||
// alert group key. Changing a rule's grouping starts a new thread.
|
||||
u, err := url.Parse(n.conf.WebhookURL.String())
|
||||
if err != nil {
|
||||
return false, errors.WrapInternalf(err, errors.CodeInternal, "parse google chat webhook url")
|
||||
}
|
||||
q := u.Query()
|
||||
q.Set("threadKey", key.Hash())
|
||||
q.Set("messageReplyOption", "REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD")
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
resp, err := notify.PostJSON(ctx, n.client, u.String(), buf) //nolint:bodyclose
|
||||
resp, err := notify.PostJSON(ctx, n.client, n.conf.WebhookURL.String(), buf) //nolint:bodyclose
|
||||
if err != nil {
|
||||
return true, notify.RedactURL(err)
|
||||
}
|
||||
|
||||
@@ -245,7 +245,7 @@ func TestGoogleChatMessageSizeLimit(t *testing.T) {
|
||||
assert.LessOrEqual(t, bodyLen, maxMessageBytes, "posted body must be within the size limit")
|
||||
}
|
||||
|
||||
func TestGoogleChatThreading(t *testing.T) {
|
||||
func TestGoogleChatWebhookURLVerbatim(t *testing.T) {
|
||||
var query url.Values
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
query = r.URL.Query()
|
||||
@@ -253,25 +253,11 @@ func TestGoogleChatThreading(t *testing.T) {
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
cases := []struct{ name, groupKey string }{
|
||||
{"rule a", "{ruleId=\"aaa\"}"},
|
||||
{"rule b", "{ruleId=\"bbb\"}"},
|
||||
}
|
||||
seen := map[string]string{}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
n := newTestNotifier(t, server.URL, "T", "")
|
||||
ctx := notify.WithGroupKey(context.Background(), c.groupKey)
|
||||
_, err := n.Notify(ctx, newTestAlerts("X")...)
|
||||
require.NoError(t, err)
|
||||
n := newTestNotifier(t, server.URL+"?key=abc&token=xyz", "T", "")
|
||||
_, err := n.Notify(newTestContext(), newTestAlerts("X")...)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "REPLY_MESSAGE_FALLBACK_TO_NEW_THREAD", query.Get("messageReplyOption"))
|
||||
threadKey := query.Get("threadKey")
|
||||
assert.Equal(t, notify.Key(c.groupKey).Hash(), threadKey, "threadKey must be the group key hash")
|
||||
seen[c.name] = threadKey
|
||||
})
|
||||
}
|
||||
assert.NotEqual(t, seen["rule a"], seen["rule b"], "distinct group keys must yield distinct threadKeys")
|
||||
assert.Equal(t, url.Values{"key": {"abc"}, "token": {"xyz"}}, query, "configured webhook URL must be posted verbatim, with no params added")
|
||||
}
|
||||
|
||||
func TestGoogleChatCustomTemplateMarkdown(t *testing.T) {
|
||||
|
||||
@@ -51,6 +51,28 @@
|
||||
},
|
||||
"name": "Region"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "ListVariable",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "FunctionName",
|
||||
"description": "Name of the Lambda function"
|
||||
},
|
||||
"allowAllValue": true,
|
||||
"allowMultiple": true,
|
||||
"customAllValue": "",
|
||||
"capturingRegexp": "",
|
||||
"sort": "none",
|
||||
"plugin": {
|
||||
"kind": "signoz/DynamicVariable",
|
||||
"spec": {
|
||||
"name": "FunctionName",
|
||||
"signal": "metrics"
|
||||
}
|
||||
},
|
||||
"name": "FunctionName"
|
||||
}
|
||||
}
|
||||
],
|
||||
"panels": {
|
||||
@@ -118,7 +140,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -218,7 +240,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -318,7 +340,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -418,7 +440,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -518,7 +540,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -618,7 +640,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -718,7 +740,7 @@
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
@@ -831,4 +853,4 @@
|
||||
"refreshInterval": "",
|
||||
"links": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,6 +242,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewRestructureAuthDomainConfigFactory(sqlstore),
|
||||
sqlmigration.NewFixSavedViewSelectFieldsFactory(sqlstore),
|
||||
sqlmigration.NewDeleteOrphanUserRolesFactory(),
|
||||
sqlmigration.NewMigrateLambdaDashboardsFactory(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
160
pkg/sqlmigration/116_migrate_lambda_dashboards.go
Normal file
160
pkg/sqlmigration/116_migrate_lambda_dashboards.go
Normal file
@@ -0,0 +1,160 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
//go:embed 116_migrate_lambda_dashboards
|
||||
var lambdaDashboardFiles embed.FS
|
||||
|
||||
// These values mirror the cloud integration and dashboard packages but are duplicated
|
||||
// here so this migration keeps targeting and writing the same rows even if those
|
||||
// constants are later renamed or changed.
|
||||
const (
|
||||
lambdaDashboardFile = "116_migrate_lambda_dashboards/aws/lambda/overview.json"
|
||||
|
||||
lambdaDashboardSlug = "aws-lambda-overview"
|
||||
cloudIntegrationDashboardProvider = "cloud_integration"
|
||||
integrationDashboardSource = "integration"
|
||||
dashboardSchemaVersion = "v6"
|
||||
)
|
||||
|
||||
type migrateLambdaDashboards struct{}
|
||||
|
||||
type lambdaDashboardRow struct {
|
||||
bun.BaseModel `bun:"table:dashboard,alias:dashboard"`
|
||||
|
||||
ID string `bun:"id"`
|
||||
Data string `bun:"data"`
|
||||
}
|
||||
|
||||
// lambdaDashboardDefinition is the part of the embedded dashboard this migration reads:
|
||||
// its spec, which is what the cloud integration stores under data.spec.
|
||||
type lambdaDashboardDefinition struct {
|
||||
Spec map[string]any `json:"spec"`
|
||||
}
|
||||
|
||||
func NewMigrateLambdaDashboardsFactory() factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("migrate_lambda_dashboards"),
|
||||
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return &migrateLambdaDashboards{}, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (m *migrateLambdaDashboards) Register(migrations *migrate.Migrations) error {
|
||||
return migrations.Register(m.Up, m.Down)
|
||||
}
|
||||
|
||||
// Up rewrites the spec of every provisioned AWS Lambda overview dashboard to the
|
||||
// embedded revision that added the FunctionName variable. Cloud integration dashboards
|
||||
// are provisioned once and never updated afterwards, so existing installs only pick up
|
||||
// this change through a migration. Only the spec is replaced; the row keeps its id, name,
|
||||
// tags and metadata, so the dashboard is updated in place rather than recreated.
|
||||
func (m *migrateLambdaDashboards) Up(ctx context.Context, db *bun.DB) error {
|
||||
spec, err := m.loadSpec()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
var rows []*lambdaDashboardRow
|
||||
if err := tx.NewSelect().
|
||||
Model(&rows).
|
||||
Join("JOIN integration_dashboard AS id ON id.dashboard_id = dashboard.id").
|
||||
Where("id.provider = ?", cloudIntegrationDashboardProvider).
|
||||
Where("id.slug = ?", lambdaDashboardSlug).
|
||||
Where("dashboard.source = ?", integrationDashboardSource).
|
||||
Scan(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
data := map[string]any{}
|
||||
if err := json.Unmarshal([]byte(row.Data), &data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The embedded spec is v6-shaped, so only rewrite a row already carrying a v6 spec;
|
||||
// anything else is left alone rather than turned into a broken mix of versions.
|
||||
if !m.hasV6Spec(data) {
|
||||
continue
|
||||
}
|
||||
data["spec"] = spec
|
||||
|
||||
encoded, err := m.marshalUnescaped(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Skip rows already carrying this spec so a re-run does not needlessly rewrite them.
|
||||
if string(encoded) == row.Data {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := tx.NewUpdate().
|
||||
Model((*lambdaDashboardRow)(nil)).
|
||||
Set("data = ?", string(encoded)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Where("id = ?", row.ID).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (m *migrateLambdaDashboards) Down(context.Context, *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// hasV6Spec reports whether the stored data is a v6 dashboard with a spec object, which
|
||||
// is the shape whose spec this migration replaces.
|
||||
func (m *migrateLambdaDashboards) hasV6Spec(data map[string]any) bool {
|
||||
metadata, _ := data["metadata"].(map[string]any)
|
||||
version, _ := metadata["schemaVersion"].(string)
|
||||
if version != dashboardSchemaVersion {
|
||||
return false
|
||||
}
|
||||
_, ok := data["spec"].(map[string]any)
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *migrateLambdaDashboards) marshalUnescaped(v any) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
encoder := json.NewEncoder(&buf)
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bytes.TrimRight(buf.Bytes(), "\n"), nil
|
||||
}
|
||||
|
||||
func (m *migrateLambdaDashboards) loadSpec() (map[string]any, error) {
|
||||
raw, err := lambdaDashboardFiles.ReadFile(lambdaDashboardFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var dashboard lambdaDashboardDefinition
|
||||
if err := json.Unmarshal(raw, &dashboard); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return dashboard.Spec, nil
|
||||
}
|
||||
@@ -0,0 +1,856 @@
|
||||
{
|
||||
"schemaVersion": "v6",
|
||||
"image": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iODAwcHgiIGhlaWdodD0iODAwcHgiIHZpZXdCb3g9IjAgMCAxNiAxNiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBmaWxsPSJub25lIj48cGF0aCBmaWxsPSIjRkE3RTE0IiBkPSJNNy45ODMgOC4zN2MtLjA1My4wNzMtLjA5OC4xMzMtLjE0MS4xOTRMNS43NzUgMTEuNWMtLjY0LjkxLTEuMjgyIDEuODItMS45MjQgMi43M2EuMTI4LjEyOCAwIDAxLS4wOTIuMDUxYy0uOTA2LS4wMDctMS44MTMtLjAxNy0yLjcxOS0uMDI4LS4wMSAwLS4wMi0uMDAzLS4wNC0uMDA2YS40NTUuNDU1IDAgMDEuMDI1LS4wNTMgMTM5NzcuNDk2IDEzOTc3LjQ5NiAwIDAxNS40NDYtOC4xNDZjLjA5Mi0uMTM4LjE4OC0uMjczLjI3NS0uNDEzYS4xNjUuMTY1IDAgMDAuMDE4LS4xMjRjLS4xNjctLjUxNS0uMzM4LTEuMDMtLjUwOC0xLjU0My0uMDczLS4yMi0uMTUtLjQ0LS4yMTgtLjY2LS4wMjItLjA3Mi0uMDU5LS4wOTQtLjEzNC0uMDkzLS41Ny4wMDItMS4xMzYuMDAxLTEuNzA0LjAwMS0uMTA4IDAtLjEwOCAwLS4xMDgtLjEwMyAwLS42NzQgMC0xLjM0Ny0uMDAyLTIuMDIxIDAtLjA3NS4wMjYtLjA5Mi4wOTktLjA5MiAxLjE0My4wMDIgMi4yODYuMDAyIDMuNDMgMGEuMTEzLjExMyAwIDAxLjA3Ni4wMTcuMTA3LjEwNyAwIDAxLjA0NS4wNjEgMTgyNjYuMTg0IDE4MjY2LjE4NCAwIDAwMy45MiA5LjUxYy4yMTguNTMuNDM4IDEuMDU5LjY1NCAxLjU5LjAyNi4wNjQuMDUzLjA3Ni4xMi4wNTYuNi0uMTc4IDEuMi0uMzUyIDEuOC0uNTMxLjA3NS0uMDIzLjEwMi0uMDA4LjEyNi4wNjQuMjA0LjYyLjQxMiAxLjIzOS42MiAxLjg1OGwuMDIuMDczYy0uMDQzLjAxNS0uMDgzLjAzMi0uMTI0LjA0M2wtNC4wODUgMS4yNWMtLjA2NS4wMi0uMDg1IDAtLjEwNi0uMDU0bC0xLjI1LTMuMDQ4LTEuMjI2LTIuOTg0LS4xODMtLjQ0OWMtLjAxLS4wMjYtLjAyMy0uMDQ4LS4wNDMtLjA4N3oiLz48L3N2Zz4=",
|
||||
"name": "",
|
||||
"generateName": true,
|
||||
"tags": [],
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "AWS Lambda Overview",
|
||||
"description": "Overview of AWS Lambda functions"
|
||||
},
|
||||
"variables": [
|
||||
{
|
||||
"kind": "ListVariable",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Account",
|
||||
"description": "AWS Account"
|
||||
},
|
||||
"allowAllValue": false,
|
||||
"allowMultiple": false,
|
||||
"customAllValue": "",
|
||||
"capturingRegexp": "",
|
||||
"sort": "none",
|
||||
"plugin": {
|
||||
"kind": "signoz/QueryVariable",
|
||||
"spec": {
|
||||
"queryValue": "SELECT JSONExtractString(labels, 'cloud.account.id') as `cloud.account.id`\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE \n metric_name like 'aws_Lambda_Invocations_sum'\nGROUP BY `cloud.account.id`\n\n"
|
||||
}
|
||||
},
|
||||
"name": "Account"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "ListVariable",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Region",
|
||||
"description": "AWS Region"
|
||||
},
|
||||
"allowAllValue": false,
|
||||
"allowMultiple": false,
|
||||
"customAllValue": "",
|
||||
"capturingRegexp": "",
|
||||
"sort": "none",
|
||||
"plugin": {
|
||||
"kind": "signoz/QueryVariable",
|
||||
"spec": {
|
||||
"queryValue": "SELECT JSONExtractString(labels, 'cloud.region') as `cloud.region`\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE \n metric_name like 'aws_Lambda_Invocations_sum'\n and JSONExtractString(labels, 'cloud.account.id') IN {{.Account}}\nGROUP BY `cloud.region`\n"
|
||||
}
|
||||
},
|
||||
"name": "Region"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "ListVariable",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "FunctionName",
|
||||
"description": "Name of the Lambda function"
|
||||
},
|
||||
"allowAllValue": true,
|
||||
"allowMultiple": true,
|
||||
"customAllValue": "",
|
||||
"capturingRegexp": "",
|
||||
"sort": "none",
|
||||
"plugin": {
|
||||
"kind": "signoz/DynamicVariable",
|
||||
"spec": {
|
||||
"name": "FunctionName",
|
||||
"signal": "metrics"
|
||||
}
|
||||
},
|
||||
"name": "FunctionName"
|
||||
}
|
||||
}
|
||||
],
|
||||
"panels": {
|
||||
"2516c785-b025-49b3-aeb4-a4735ccb2709": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Errors",
|
||||
"description": "The number of invocations that result in a function error. Function errors include exceptions that your code throws and exceptions that the Lambda runtime throws. The runtime returns errors for issues such as timeouts and configuration errors. To calculate the error rate, divide the value of Errors by the value of Invocations. Note that the timestamp on an error metric reflects when the function was invoked, not when the error occurred.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "none",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_Errors_sum",
|
||||
"temporality": "",
|
||||
"timeAggregation": "sum",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
},
|
||||
"4119a1e5-32a8-4859-96e9-a5451114782b": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Async events dropped",
|
||||
"description": "The number of events that are dropped without successfully executing the function. If you configure a dead-letter queue (DLQ) or OnFailure destination, then events are sent there before they're dropped. Events are dropped for various reasons. For example, events can exceed the maximum event age or exhaust the maximum retry attempts, or reserved concurrency might be set to 0. To troubleshoot why events are dropped, look at the Errors metric to identify function errors and the Throttles metric to identify concurrency issues.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "none",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_AsyncEventsDropped_sum",
|
||||
"temporality": "",
|
||||
"timeAggregation": "sum",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
},
|
||||
"6354ea62-e82b-4323-a33d-eef92519e843": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Throttles",
|
||||
"description": "The number of invocation requests that are throttled. When all function instances are processing requests and no concurrency is available to scale up, Lambda rejects additional requests with a TooManyRequestsException error. Throttled requests and other invocation errors don't count as either Invocations or Errors.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "none",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_Throttles_sum",
|
||||
"temporality": "",
|
||||
"timeAggregation": "sum",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
},
|
||||
"853d3a92-b396-4064-8762-18d7487989e0": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Async events received",
|
||||
"description": "The number of events that Lambda successfully queues for processing. This metric provides insight into the number of events that a Lambda function receives. Monitor this metric and set alarms for thresholds to check for issues. For example, to detect an undesirable number of events sent to Lambda, and to quickly diagnose issues resulting from incorrect trigger or function configurations. Mismatches between AsyncEventsReceived and Invocations can indicate a disparity in processing, events being dropped, or a potential queue backlog.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "none",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_AsyncEventsReceived_sum",
|
||||
"temporality": "",
|
||||
"timeAggregation": "sum",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
},
|
||||
"877bb5c8-331c-492f-b666-2054c2ae39bd": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Invocations",
|
||||
"description": "The number of times that your function code is invoked, including successful invocations and invocations that result in a function error. Invocations aren't recorded if the invocation request is throttled or otherwise results in an invocation error. The value of Invocations equals the number of requests billed.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "none",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_Invocations_sum",
|
||||
"temporality": "",
|
||||
"timeAggregation": "sum",
|
||||
"spaceAggregation": "sum",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
},
|
||||
"ae6d7c81-d921-4d4c-95ec-6b42d900ea45": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Max Async Event Age",
|
||||
"description": "The time between when Lambda successfully queues the event and when the function is invoked. The value of this metric increases when events are being retried due to invocation failures or throttling. Monitor this metric and set alarms for thresholds on different statistics for when a queue buildup occurs. To troubleshoot an increase in this metric, look at the Errors metric to identify function errors and the Throttles metric to identify concurrency issues.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "ms",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_AsyncEventAge_max",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"spaceAggregation": "max",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
},
|
||||
"b038520d-0756-4e46-a915-12a2f19a0254": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"display": {
|
||||
"name": "Max Duration",
|
||||
"description": "The amount of time that your function code spends processing an event. The billed duration for an invocation is the value of Duration rounded up to the nearest millisecond. Duration does not include cold start time.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
|
||||
},
|
||||
"plugin": {
|
||||
"kind": "signoz/TimeSeriesPanel",
|
||||
"spec": {
|
||||
"visualization": {
|
||||
"timePreference": "global_time",
|
||||
"fillSpans": false
|
||||
},
|
||||
"formatting": {
|
||||
"unit": "ms",
|
||||
"decimalPrecision": "2"
|
||||
},
|
||||
"chartAppearance": {
|
||||
"lineInterpolation": "spline",
|
||||
"showPoints": false,
|
||||
"lineStyle": "solid",
|
||||
"fillMode": "none",
|
||||
"spanGaps": {
|
||||
"fillOnlyBelow": false,
|
||||
"fillLessThan": ""
|
||||
}
|
||||
},
|
||||
"axes": {
|
||||
"softMin": 0,
|
||||
"softMax": 0,
|
||||
"isLogScale": false
|
||||
},
|
||||
"legend": {
|
||||
"position": "bottom",
|
||||
"mode": "list",
|
||||
"customColors": null
|
||||
},
|
||||
"thresholds": null
|
||||
}
|
||||
},
|
||||
"queries": [
|
||||
{
|
||||
"kind": "time_series",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"plugin": {
|
||||
"kind": "signoz/BuilderQuery",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"stepInterval": 60,
|
||||
"signal": "metrics",
|
||||
"source": "",
|
||||
"aggregations": [
|
||||
{
|
||||
"metricName": "aws_Lambda_Duration_max",
|
||||
"temporality": "",
|
||||
"timeAggregation": "max",
|
||||
"spaceAggregation": "max",
|
||||
"reduceTo": "avg"
|
||||
}
|
||||
],
|
||||
"disabled": false,
|
||||
"filter": {
|
||||
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
|
||||
},
|
||||
"groupBy": [
|
||||
{
|
||||
"name": "cloud.account.id",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "cloud.region",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
},
|
||||
{
|
||||
"name": "FunctionName",
|
||||
"signal": "",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "string"
|
||||
}
|
||||
],
|
||||
"order": [],
|
||||
"having": {
|
||||
"expression": ""
|
||||
},
|
||||
"functions": [],
|
||||
"legend": "{{FunctionName}}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"links": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"layouts": [
|
||||
{
|
||||
"kind": "Grid",
|
||||
"spec": {
|
||||
"items": [
|
||||
{
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/877bb5c8-331c-492f-b666-2054c2ae39bd"
|
||||
}
|
||||
},
|
||||
{
|
||||
"x": 6,
|
||||
"y": 0,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/b038520d-0756-4e46-a915-12a2f19a0254"
|
||||
}
|
||||
},
|
||||
{
|
||||
"x": 0,
|
||||
"y": 6,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/2516c785-b025-49b3-aeb4-a4735ccb2709"
|
||||
}
|
||||
},
|
||||
{
|
||||
"x": 6,
|
||||
"y": 6,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/6354ea62-e82b-4323-a33d-eef92519e843"
|
||||
}
|
||||
},
|
||||
{
|
||||
"x": 0,
|
||||
"y": 12,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/853d3a92-b396-4064-8762-18d7487989e0"
|
||||
}
|
||||
},
|
||||
{
|
||||
"x": 6,
|
||||
"y": 12,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/ae6d7c81-d921-4d4c-95ec-6b42d900ea45"
|
||||
}
|
||||
},
|
||||
{
|
||||
"x": 0,
|
||||
"y": 18,
|
||||
"width": 6,
|
||||
"height": 6,
|
||||
"content": {
|
||||
"$ref": "#/spec/panels/4119a1e5-32a8-4859-96e9-a5451114782b"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"duration": "",
|
||||
"refreshInterval": "",
|
||||
"links": []
|
||||
}
|
||||
}
|
||||
@@ -462,8 +462,8 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
Limit: 10,
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (JSON_VALUE(body, '$.\"status\"') = ? AND JSON_EXISTS(body, '$.\"status\"')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"success", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSON_VALUE(body, '$.\"status\"') = ? AND LOWER(body) LIKE LOWER(?)) AND (JSON_EXISTS(body, '$.\"status\"') AND LOWER(body) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"success", "%success%", "%\"status\"%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Warnings: []string{querybuilder.NewKeyNotFoundWarning("status")},
|
||||
},
|
||||
expectedErr: nil,
|
||||
@@ -481,8 +481,8 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
Limit: 10,
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSON_VALUE(body, '$.\"user_names\"[*]') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (((JSON_VALUE(body, '$.\"user_names\"[*]') = ? AND LOWER(body) LIKE LOWER(?))) AND (JSON_EXISTS(body, '$.\"user_names\"[*]') AND LOWER(body) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"john_doe", "%john\\_doe%", "%\"user\\_names\"%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")},
|
||||
},
|
||||
expectedErr: nil,
|
||||
@@ -498,8 +498,8 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
Limit: 10,
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (has(JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$.\"user_names\"') = ? AND JSONType(body, 'user_names') NOT IN ('Array', 'Object', 'Null')), false)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"john_doe", "john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((has(JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$.\"user_names\"') = ? AND JSONType(body, 'user_names') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"john_doe", "john_doe", "%\"user\\_names\"%", "%john\\_doe%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")},
|
||||
},
|
||||
expectedErr: nil,
|
||||
@@ -1011,8 +1011,8 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
},
|
||||
enableUseJSONBody: false,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (body = ? AND LOWER(body) = LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"", "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
|
||||
@@ -322,6 +322,38 @@ func TestStatementBuilder(t *testing.T) {
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
name: "test_bool_label_filter",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.MetricAggregation{
|
||||
{
|
||||
MetricName: "signoz_calls_total",
|
||||
Type: metrictypes.SumType,
|
||||
Temporality: metrictypes.Cumulative,
|
||||
TimeAggregation: metrictypes.TimeAggregationRate,
|
||||
SpaceAggregation: metrictypes.SpaceAggregationSum,
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "success = true",
|
||||
},
|
||||
GroupBy: []qbtypes.GroupByKey{
|
||||
{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __temporal_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(30)) AS ts, `__GROUP_BY_KEY_0_service.name`, max(value) AS per_series_value FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'service.name') AS `__GROUP_BY_KEY_0_service.name` FROM signoz_metrics.time_series_v4_6hrs WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ? GROUP BY fingerprint, `__GROUP_BY_KEY_0_service.name`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `__GROUP_BY_KEY_0_service.name` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `__GROUP_BY_KEY_0_service.name`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `__GROUP_BY_KEY_0_service.name`) SELECT * FROM __spatial_aggregation_cte ORDER BY `__GROUP_BY_KEY_0_service.name`, ts",
|
||||
Args: []any{"signoz_calls_total", uint64(1747936800000), uint64(1747983420000), "cumulative", true, "signoz_calls_total", uint64(1747947360000), uint64(1747983420000), 0},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
}
|
||||
|
||||
fm := metricstelemetryschema.NewFieldMapper()
|
||||
|
||||
@@ -31,6 +31,14 @@
|
||||
"signal": "metrics"
|
||||
}
|
||||
],
|
||||
"success": [
|
||||
{
|
||||
"name": "success",
|
||||
"fieldContext": "attribute",
|
||||
"fieldDataType": "bool",
|
||||
"signal": "metrics"
|
||||
}
|
||||
],
|
||||
"materialized.key.name": [
|
||||
{
|
||||
"name": "materialized.key.name",
|
||||
|
||||
@@ -105,14 +105,14 @@ func (c *conditionBuilder) conditionForArrayFunction(
|
||||
"function `%s` supports only body JSON search", operator.FunctionName()).WithUrl(functionBodyJSONSearchDocURL)
|
||||
}
|
||||
|
||||
needle := value
|
||||
element := value
|
||||
if args, ok := value.([]any); ok && len(args) > 0 {
|
||||
needle = args[0]
|
||||
element = args[0]
|
||||
}
|
||||
|
||||
if c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
// JSON access plan: data-type collision handling, nested array paths.
|
||||
valueType, needle := InferDataType(needle, operator, key)
|
||||
valueType, element := InferDataType(element, operator, key)
|
||||
// A not-found (synthesized) body path carries no metadata plan; build an exhaustive
|
||||
// one so the query runs against the underlying data (with the not-found warning)
|
||||
// instead of erroring, matching the regular-operator path.
|
||||
@@ -123,21 +123,21 @@ func (c *conditionBuilder) conditionForArrayFunction(
|
||||
}
|
||||
key = keyCopy
|
||||
}
|
||||
return NewJSONConditionBuilder(key, valueType).buildArrayFunctionCondition(operator, needle, sb)
|
||||
return NewJSONConditionBuilder(key, valueType).buildArrayFunctionCondition(operator, element, sb)
|
||||
}
|
||||
|
||||
// legacy string-body path: type-matched array extraction, OR-ed with a scalar comparison
|
||||
// for a scalar body value (coalesced to false so NOT has() matches missing-key rows).
|
||||
elemType := legacyElemType(needle)
|
||||
elemType := legacyElemType(element)
|
||||
arrayExpr := getBodyJSONArrayKey(key, elemType)
|
||||
scalarExpr, scalarGuard, hasScalar := getBodyJSONScalarKey(key, elemType)
|
||||
if list, ok := needle.([]any); ok {
|
||||
if list, ok := element.([]any); ok {
|
||||
vals := make([]any, len(list))
|
||||
for i, v := range list {
|
||||
vals[i] = legacyCoerceNeedle(v, elemType)
|
||||
vals[i] = legacyCoerceElement(v, elemType)
|
||||
}
|
||||
// Pin the needle array type to the haystack; scalar fallback below coerces value-level.
|
||||
arrayCond := fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), arrayExpr, castNeedleArray(elemType, sb.Var(vals)))
|
||||
// Pin the element array type to the array it is tested against; scalar fallback below coerces value-level.
|
||||
arrayCond := fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), arrayExpr, castElementArray(elemType, sb.Var(vals)))
|
||||
if !hasScalar {
|
||||
return arrayCond, nil
|
||||
}
|
||||
@@ -153,17 +153,17 @@ func (c *conditionBuilder) conditionForArrayFunction(
|
||||
}
|
||||
return fmt.Sprintf("(%s OR ifNull(%s, false))", arrayCond, sb.And(membership, scalarGuard)), nil
|
||||
}
|
||||
typedNeedle := legacyCoerceNeedle(needle, elemType)
|
||||
arrayCond := fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), arrayExpr, sb.Var(typedNeedle))
|
||||
typedElement := legacyCoerceElement(element, elemType)
|
||||
arrayCond := fmt.Sprintf("%s(%s, %s)", operator.FunctionName(), arrayExpr, sb.Var(typedElement))
|
||||
if !hasScalar {
|
||||
return arrayCond, nil
|
||||
}
|
||||
return fmt.Sprintf("(%s OR ifNull(%s, false))", arrayCond, sb.And(sb.E(scalarExpr, typedNeedle), scalarGuard)), nil
|
||||
return fmt.Sprintf("(%s OR ifNull(%s, false))", arrayCond, sb.And(sb.E(scalarExpr, typedElement), scalarGuard)), nil
|
||||
}
|
||||
|
||||
// castNeedleArray pins an Int64 needle array to Array(Int64) so it matches the Array(Nullable(Int64))
|
||||
// haystack; without it a needle >= 2^32 binds as Array(UInt64) and hasAny/hasAll error (code 386).
|
||||
func castNeedleArray(elemType telemetrytypes.FieldDataType, arg string) string {
|
||||
// castElementArray pins an Int64 element array to Array(Int64) so it matches the Array(Nullable(Int64))
|
||||
// it is tested against; without it an element >= 2^32 binds as Array(UInt64) and hasAny/hasAll error (code 386).
|
||||
func castElementArray(elemType telemetrytypes.FieldDataType, arg string) string {
|
||||
if elemType == telemetrytypes.FieldDataTypeInt64 {
|
||||
return fmt.Sprintf("CAST(%s AS Array(Int64))", arg)
|
||||
}
|
||||
@@ -191,24 +191,24 @@ func (c *conditionBuilder) conditionForHasToken(
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
// hasToken takes a single needle; unwrap it from the function-argument slice.
|
||||
needle := value
|
||||
// hasToken takes a single token; unwrap it from the function-argument slice.
|
||||
token := value
|
||||
if args, ok := value.([]any); ok && len(args) > 0 {
|
||||
needle = args[0]
|
||||
token = args[0]
|
||||
}
|
||||
|
||||
// hasToken matches string tokens only.
|
||||
needleStr, ok := needle.(string)
|
||||
tokenStr, ok := token.(string)
|
||||
if !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"function `hasToken` expects value parameter to be a string").WithUrl(hasTokenFunctionDocURL)
|
||||
}
|
||||
|
||||
// A multi-token needle makes CH hasToken error (code 36); reject up front as a 400. Both modes flow here.
|
||||
if sep, found := firstTokenSeparator(needleStr); found {
|
||||
// A multi-token value makes CH hasToken error (code 36); reject up front as a 400. Both modes flow here.
|
||||
if sep, found := firstTokenSeparator(tokenStr); found {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"function `hasToken` matches a single whole token, but %q contains the separator %q; use a substring filter (e.g. `body CONTAINS '%s'`) to search across separators",
|
||||
needleStr, sep, needleStr).WithUrl(hasTokenFunctionDocURL)
|
||||
tokenStr, sep, tokenStr).WithUrl(hasTokenFunctionDocURL)
|
||||
}
|
||||
|
||||
bodyJSONEnabled := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))
|
||||
@@ -219,7 +219,7 @@ func (c *conditionBuilder) conditionForHasToken(
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"function `hasToken` only supports body field as first parameter").WithUrl(hasTokenFunctionDocURL)
|
||||
}
|
||||
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", LogsV2BodyColumn, sb.Var(needle)), nil
|
||||
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", LogsV2BodyColumn, sb.Var(token)), nil
|
||||
}
|
||||
|
||||
// JSON mode: a bare body/body.message key searches the body.message column; any other body
|
||||
@@ -228,7 +228,7 @@ func (c *conditionBuilder) conditionForHasToken(
|
||||
// falls through and emits dynamicElement over the already-typed String column, which errors.
|
||||
if key.Name == LogsV2BodyColumn || key.Name == bodyMessageField ||
|
||||
(key.FieldContext == telemetrytypes.FieldContextBody && key.Name == messageSubField) {
|
||||
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", bodyMessageField, sb.Var(needle)), nil
|
||||
return fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", bodyMessageField, sb.Var(token)), nil
|
||||
}
|
||||
if key.FieldContext == telemetrytypes.FieldContextBody {
|
||||
// A not-found (synthesized) body path carries no metadata plan; build an exhaustive
|
||||
@@ -240,7 +240,7 @@ func (c *conditionBuilder) conditionForHasToken(
|
||||
}
|
||||
key = keyCopy
|
||||
}
|
||||
return NewJSONConditionBuilder(key, telemetrytypes.FieldDataTypeString).buildTokenFunctionCondition(needle, sb)
|
||||
return NewJSONConditionBuilder(key, telemetrytypes.FieldDataTypeString).buildTokenFunctionCondition(token, sb)
|
||||
}
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput,
|
||||
"function `hasToken` only supports the body field or a body JSON string field as first parameter").WithUrl(hasTokenFunctionDocURL)
|
||||
@@ -254,12 +254,21 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
) (condition string, err error) {
|
||||
// hasToken resolves from the key name + flag alone (no column resolution), so handle it first.
|
||||
if operator == qbtypes.FilterOperatorHasToken {
|
||||
return c.conditionForHasToken(ctx, orgID, key, value, sb)
|
||||
}
|
||||
|
||||
// What the legacy body JSON path implies over the indexed LOWER(body), ANDed onto whichever
|
||||
// condition the operator builds below — which still decides the row.
|
||||
var bodyIndexPredicates []string
|
||||
defer func() {
|
||||
if err == nil && len(bodyIndexPredicates) > 0 {
|
||||
condition = sb.And(append([]string{condition}, bodyIndexPredicates...)...)
|
||||
}
|
||||
}()
|
||||
|
||||
columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) && key.FieldContext == telemetrytypes.FieldContextUnspecified {
|
||||
key = telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextBody, key.FieldDataType)
|
||||
@@ -269,14 +278,20 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
return "", err
|
||||
}
|
||||
|
||||
useJSONBody := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID))
|
||||
legacyBodyJSONSearch := isBodyJSONSearch(key, columns) && !useJSONBody
|
||||
|
||||
// has/hasAny/hasAll take the body-JSON path, not the normal operator paths.
|
||||
if operator.IsArrayFunctionOperator() {
|
||||
if legacyBodyJSONSearch {
|
||||
bodyIndexPredicates = legacyBodyIndexPredicates(key, operator, value, sb)
|
||||
}
|
||||
return c.conditionForArrayFunction(ctx, orgID, key, operator, value, columns, sb)
|
||||
}
|
||||
|
||||
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
|
||||
for _, column := range columns {
|
||||
if column.Type.GetType() == schema.ColumnTypeEnumJSON && isBodyJSONSearch(key, columns) && c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) && key.Name != messageSubField {
|
||||
if column.Type.GetType() == schema.ColumnTypeEnumJSON && isBodyJSONSearch(key, columns) && useJSONBody && key.Name != messageSubField {
|
||||
valueType, value := InferDataType(value, operator, key)
|
||||
if len(key.JSONPlan) == 0 {
|
||||
keyCopy := telemetrytypes.NewTelemetryFieldKey(key.Name, key.FieldContext, key.FieldDataType)
|
||||
@@ -305,8 +320,9 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
}
|
||||
|
||||
// Check if this is a body JSON search (legacy string-body path, JSON flag off).
|
||||
if isBodyJSONSearch(key, columns) && !c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
if legacyBodyJSONSearch {
|
||||
fieldExpression, value = GetBodyJSONKey(ctx, key, operator, value)
|
||||
bodyIndexPredicates = legacyBodyIndexPredicates(key, operator, value, sb)
|
||||
}
|
||||
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
@@ -314,10 +330,21 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
// make use of case insensitive index for body
|
||||
if fieldExpression == "body" || fieldExpression == messageSubColumn {
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
// Bloom filters index lower(body), not the column; `=` still decides the row.
|
||||
if _, ok := value.(string); ok && fieldExpression == LogsV2BodyColumn {
|
||||
return sb.And(
|
||||
sb.E(fieldExpression, value),
|
||||
fmt.Sprintf("LOWER(%s) = LOWER(%s)", fieldExpression, sb.Var(value)),
|
||||
), nil
|
||||
}
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
if _, ok := value.(string); ok && fieldExpression == LogsV2BodyColumn {
|
||||
return sb.And(
|
||||
sb.Like(fieldExpression, value),
|
||||
sb.ILike(fieldExpression, value),
|
||||
), nil
|
||||
}
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
@@ -356,7 +383,7 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
if isBodyJSONSearch(key, columns) && !c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
if legacyBodyJSONSearch {
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return GetBodyJSONKeyForExists(ctx, key, operator, value), nil
|
||||
}
|
||||
|
||||
@@ -168,9 +168,9 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: "error message",
|
||||
expectedSQL: "body = ?",
|
||||
expectedArgs: []any{"error message"},
|
||||
value: "Error Message",
|
||||
expectedSQL: "(body = ? AND LOWER(body) = LOWER(?))",
|
||||
expectedArgs: []any{"Error Message", "Error Message"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -207,8 +207,8 @@ func TestConditionFor(t *testing.T) {
|
||||
},
|
||||
operator: qbtypes.FilterOperatorLike,
|
||||
value: "%error%",
|
||||
expectedSQL: "LOWER(body) LIKE LOWER(?)",
|
||||
expectedArgs: []any{"%error%"},
|
||||
expectedSQL: "(body LIKE ? AND LOWER(body) LIKE LOWER(?))",
|
||||
expectedArgs: []any{"%error%", "%error%"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -219,7 +219,7 @@ func TestConditionFor(t *testing.T) {
|
||||
},
|
||||
operator: qbtypes.FilterOperatorNotLike,
|
||||
value: "%error%",
|
||||
expectedSQL: "LOWER(body) NOT LIKE LOWER(?)",
|
||||
expectedSQL: "body NOT LIKE ?",
|
||||
expectedArgs: []any{"%error%"},
|
||||
expectedError: nil,
|
||||
},
|
||||
@@ -619,8 +619,8 @@ func TestConditionForMultipleKeys(t *testing.T) {
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: "error message",
|
||||
expectedSQL: "body = ? AND severity_text = ?",
|
||||
expectedArgs: []any{"error message", "error message"},
|
||||
expectedSQL: "(body = ? AND LOWER(body) = LOWER(?)) AND severity_text = ?",
|
||||
expectedArgs: []any{"error message", "error message", "error message"},
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
@@ -906,8 +906,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// IN on the body column routes each value back through the `=` path; the SQL it produces
|
||||
// must stay what the shared IN handling produced before, including for a mixed-type list.
|
||||
// IN on the body column routes each value back through the `=` path, so every arm picks up
|
||||
// the lower(body) companion — including the values a mixed-type list stringifies.
|
||||
func TestConditionForBodyIn(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
@@ -918,14 +918,14 @@ func TestConditionForBodyIn(t *testing.T) {
|
||||
{
|
||||
name: "strings",
|
||||
values: []any{"alpha", "beta"},
|
||||
expectedSQL: "(body = ? OR body = ?)",
|
||||
expectedArgs: []any{"alpha", "beta"},
|
||||
expectedSQL: "((body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)))",
|
||||
expectedArgs: []any{"alpha", "alpha", "beta", "beta"},
|
||||
},
|
||||
{
|
||||
name: "mixed types are stringified before they reach the column",
|
||||
values: []any{"alpha", float64(1), true},
|
||||
expectedSQL: "(body = ? OR body = ? OR body = ?)",
|
||||
expectedArgs: []any{"alpha", "1", "true"},
|
||||
expectedSQL: "((body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)) OR (body = ? AND LOWER(body) = LOWER(?)))",
|
||||
expectedArgs: []any{"alpha", "alpha", "1", "1", "true", "true"},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -954,3 +954,213 @@ func TestConditionForBodyIn(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ClickHouse treats `\` as an escape only before `%`, `_` and itself.
|
||||
func TestLikePatternLiterals(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
pattern string
|
||||
expected []string
|
||||
}{
|
||||
{"contains wraps a plain value", "%error%", []string{"error"}},
|
||||
{"wildcards split runs", "%foo%bar%", []string{"foo", "bar"}},
|
||||
{"underscore splits too", "a_b", []string{"a", "b"}},
|
||||
{"escaped wildcards stay literal", `%100\%\_off%`, []string{`100%_off`}},
|
||||
{"escaped backslash collapses", `%C:\\tmp%`, []string{`C:\tmp`}},
|
||||
{"backslash before other chars is literal", `%C:\tmp%`, []string{`C:\tmp`}},
|
||||
{"trailing backslash is literal", `%path\`, []string{`path\`}},
|
||||
{"no literals at all", "%_%", nil},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, likePatternLiterals(tc.pattern))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The literals have to hold whichever encoder wrote the body, so the runs stop at every byte
|
||||
// one of them may rewrite.
|
||||
func TestJSONTextRuns(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
value string
|
||||
expected []string
|
||||
}{
|
||||
{"plain text is one run", "checkout failed", []string{"checkout failed"}},
|
||||
{"shorter than an ngram yields nothing", "abc", nil},
|
||||
{"quote splits the run", `say "hello there"`, []string{"say ", "hello there"}},
|
||||
{"slash splits the run, PHP escapes it", "/api/v1/users", []string{"users"}},
|
||||
{"ampersand and angles split, Go escapes them", "a&b<c>dddd", []string{"dddd"}},
|
||||
{"non-ascii splits, Python escapes it", "order café latte", []string{"order caf", " latte"}},
|
||||
{"newline splits", "line one\nline two", []string{"line one", "line two"}},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.expected, jsonTextRuns(tc.value))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBodyPathLiterals(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
key string
|
||||
expected []string
|
||||
}{
|
||||
{"quoting lifts a short name over the ngram size", "id", []string{`"id"`}},
|
||||
{"one literal per component", "response.status_code", []string{`"response"`, `"status_code"`}},
|
||||
{"array suffixes are trimmed", "items[*].sku", []string{`"items"`, `"sku"`}},
|
||||
{"single-character components are dropped", "a.b.count", []string{`"count"`}},
|
||||
{"a component an encoder may rewrite is dropped", "user/name.email", []string{`"email"`}},
|
||||
{"nothing usable", "a.b", nil},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
key := telemetrytypes.NewTelemetryFieldKey(tc.key, telemetrytypes.FieldContextBody, telemetrytypes.FieldDataTypeUnspecified)
|
||||
assert.Equal(t, tc.expected, bodyPathLiterals(key))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The path literals ride on the existence assertion and the value literals on the comparison, so
|
||||
// a filter carries each at most once. Nothing rides on a negated operator: it matches rows
|
||||
// without the path, which say nothing about the body text.
|
||||
func TestLegacyBodyIndexPredicates(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
key string
|
||||
operator qbtypes.FilterOperator
|
||||
value any
|
||||
expected string
|
||||
expectedArgs []any
|
||||
}{
|
||||
{
|
||||
name: "exists carries the path",
|
||||
key: "user_id",
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
expected: `LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%"user\_id"%`},
|
||||
},
|
||||
{
|
||||
name: "equality carries the value",
|
||||
key: "status",
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: "timeout_error",
|
||||
expected: `LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%timeout\_error%`},
|
||||
},
|
||||
{
|
||||
name: "contains carries the value",
|
||||
key: "message",
|
||||
operator: qbtypes.FilterOperatorContains,
|
||||
value: "upstream refused",
|
||||
expected: `LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%upstream refused%`},
|
||||
},
|
||||
{
|
||||
name: "like carries one literal per run of the pattern",
|
||||
key: "message",
|
||||
operator: qbtypes.FilterOperatorLike,
|
||||
value: "%conn%refused%",
|
||||
expected: `LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%conn%refused%`},
|
||||
},
|
||||
{
|
||||
name: "has carries the path and the element",
|
||||
key: "tags[*]",
|
||||
operator: qbtypes.FilterOperatorHas,
|
||||
value: []any{"production"},
|
||||
// The element rides on its own predicate rather than being pinned next to the key:
|
||||
// has() over the extracted array says nothing about where in the text it sits.
|
||||
expected: `LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%"tags"%`, "%production%"},
|
||||
},
|
||||
{
|
||||
name: "hasAll carries one literal per element",
|
||||
key: "tags[*]",
|
||||
operator: qbtypes.FilterOperatorHasAll,
|
||||
value: []any{[]any{"production", "webserver"}},
|
||||
expected: `LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%"tags"%`, "%production%", "%webserver%"},
|
||||
},
|
||||
{
|
||||
// hasAny asks for one of the elements, so the arms are ORed — ANDing them would
|
||||
// demand every element be present.
|
||||
name: "hasAny ORs the element literals",
|
||||
key: "tags[*]",
|
||||
operator: qbtypes.FilterOperatorHasAny,
|
||||
value: []any{[]any{"production", "webserver"}},
|
||||
expected: `LOWER(body) LIKE LOWER(?) AND (LOWER(body) LIKE LOWER(?) OR LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{`%"tags"%`, "%production%", "%webserver%"},
|
||||
},
|
||||
{
|
||||
// one element with no usable literal voids the whole OR: the filter can still match
|
||||
// through that element, so nothing about the text is implied.
|
||||
name: "hasAny drops the OR when an element is too short",
|
||||
key: "tags[*]",
|
||||
operator: qbtypes.FilterOperatorHasAny,
|
||||
value: []any{[]any{"production", "web"}},
|
||||
expected: `LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%"tags"%`},
|
||||
},
|
||||
{
|
||||
name: "numeric elements carry nothing",
|
||||
key: "ids[*]",
|
||||
operator: qbtypes.FilterOperatorHasAny,
|
||||
value: []any{[]any{"9007199254740993", "9007199254740994"}},
|
||||
expected: `LOWER(body) LIKE LOWER(?)`,
|
||||
expectedArgs: []any{`%"ids"%`},
|
||||
},
|
||||
{
|
||||
name: "a number carries nothing",
|
||||
key: "user_id",
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: int64(123),
|
||||
},
|
||||
{
|
||||
name: "IN leaves it to the equalities it delegates to",
|
||||
key: "status",
|
||||
operator: qbtypes.FilterOperatorIn,
|
||||
value: []any{"timeout_error", "conn_refused"},
|
||||
},
|
||||
{
|
||||
name: "not equal carries nothing",
|
||||
key: "status",
|
||||
operator: qbtypes.FilterOperatorNotEqual,
|
||||
value: "timeout_error",
|
||||
},
|
||||
{
|
||||
name: "not exists carries nothing",
|
||||
key: "user_id",
|
||||
operator: qbtypes.FilterOperatorNotExists,
|
||||
},
|
||||
{
|
||||
name: "not contains carries nothing",
|
||||
key: "message",
|
||||
operator: qbtypes.FilterOperatorNotContains,
|
||||
value: "upstream refused",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("1").From("t")
|
||||
key := telemetrytypes.NewTelemetryFieldKey(tc.key, telemetrytypes.FieldContextBody, telemetrytypes.FieldDataTypeUnspecified)
|
||||
|
||||
predicates := legacyBodyIndexPredicates(key, tc.operator, tc.value, sb)
|
||||
if tc.expected == "" {
|
||||
assert.Empty(t, predicates)
|
||||
return
|
||||
}
|
||||
|
||||
sb.Where(predicates...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, query, tc.expected)
|
||||
assert.Equal(t, tc.expectedArgs, args)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,168 +44,169 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
category: "json",
|
||||
query: "has(body.requestor_list[*], 'index_service')",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."requestor_list"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."requestor_list"') = ? AND JSONType(body, 'requestor_list') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{"index_service", "index_service"},
|
||||
expectedQuery: `WHERE ((has(JSONExtract(JSON_QUERY(body, '$."requestor_list"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."requestor_list"') = ? AND JSONType(body, 'requestor_list') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{"index_service", "index_service", "%\"requestor\\_list\"%", "%index\\_service%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "has(body.int_numbers[*], 2)",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."int_numbers"[*]'), 'Array(Nullable(Float64))'), ?) OR ifNull((JSONExtract(JSON_VALUE(body, '$."int_numbers"'), 'Nullable(Float64)') = ? AND JSONType(body, 'int_numbers') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{float64(2), float64(2)},
|
||||
expectedQuery: `WHERE ((has(JSONExtract(JSON_QUERY(body, '$."int_numbers"[*]'), 'Array(Nullable(Float64))'), ?) OR ifNull((JSONExtract(JSON_VALUE(body, '$."int_numbers"'), 'Nullable(Float64)') = ? AND JSONType(body, 'int_numbers') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{float64(2), float64(2), "%\"int\\_numbers\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "has(body.bool[*], true)",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."bool"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."bool"') = ? AND JSONType(body, 'bool') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{"true", "true"},
|
||||
expectedQuery: `WHERE ((has(JSONExtract(JSON_QUERY(body, '$."bool"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."bool"') = ? AND JSONType(body, 'bool') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{"true", "true", "%\"bool\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "NOT has(body.nested_num[*].float_nums[*], 2.2)",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE NOT (has(JSONExtract(JSON_QUERY(body, '$."nested_num"[*]."float_nums"[*]'), 'Array(Nullable(Float64))'), ?))`,
|
||||
expectedArgs: []any{float64(2.2)},
|
||||
expectedQuery: `WHERE NOT ((has(JSONExtract(JSON_QUERY(body, '$."nested_num"[*]."float_nums"[*]'), 'Array(Nullable(Float64))'), ?) AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{float64(2.2), "%\"nested\\_num\"%\"float\\_nums\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "has(body.tags, 'production')",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."tags"') = ? AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{"production", "production"},
|
||||
expectedQuery: `WHERE ((has(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."tags"') = ? AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{"production", "production", "%\"tags\"%", "%production%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "hasAny(body.tags, ['critical', 'test'])",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (hasAny(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."tags"') IN (?, ?) AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{[]any{"critical", "test"}, "critical", "test"},
|
||||
expectedQuery: `WHERE ((hasAny(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull((JSON_VALUE(body, '$."tags"') IN (?, ?) AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?) AND (LOWER(body) LIKE LOWER(?) OR LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{[]any{"critical", "test"}, "critical", "test", "%\"tags\"%", "%critical%", "%test%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "hasAll(body.tags, ['production', 'web'])",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (hasAll(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull(((JSON_VALUE(body, '$."tags"') = ? AND JSON_VALUE(body, '$."tags"') = ?) AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{[]any{"production", "web"}, "production", "web"},
|
||||
expectedQuery: `WHERE ((hasAll(JSONExtract(JSON_QUERY(body, '$."tags"[*]'), 'Array(Nullable(String))'), ?) OR ifNull(((JSON_VALUE(body, '$."tags"') = ? AND JSON_VALUE(body, '$."tags"') = ?) AND JSONType(body, 'tags') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{[]any{"production", "web"}, "production", "web", "%\"tags\"%", "%production%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "has(body.ids, \"200\")",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (has(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), ?) OR ifNull((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ? AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{int64(200), int64(200)},
|
||||
expectedQuery: `WHERE ((has(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), ?) OR ifNull((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ? AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{int64(200), int64(200), "%\"ids\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
// Big-int needle CAST to Array(Int64) to match the haystack (else 386).
|
||||
// Big-int element CAST to Array(Int64) to match the array it is tested against (else 386).
|
||||
category: "json",
|
||||
query: `hasAny(body.ids, ['9007199254740993', '9007199254740994'])`,
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (hasAny(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), CAST(? AS Array(Int64))) OR ifNull((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') IN (?, ?) AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{[]any{int64(9007199254740993), int64(9007199254740994)}, int64(9007199254740993), int64(9007199254740994)},
|
||||
expectedQuery: `WHERE ((hasAny(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), CAST(? AS Array(Int64))) OR ifNull((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') IN (?, ?) AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{[]any{int64(9007199254740993), int64(9007199254740994)}, int64(9007199254740993), int64(9007199254740994), "%\"ids\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: `hasAll(body.ids, ['9007199254740993', '9007199254740994'])`,
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (hasAll(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), CAST(? AS Array(Int64))) OR ifNull(((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ? AND JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ?) AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false))`,
|
||||
expectedArgs: []any{[]any{int64(9007199254740993), int64(9007199254740994)}, int64(9007199254740993), int64(9007199254740994)},
|
||||
expectedQuery: `WHERE ((hasAll(JSONExtract(JSON_QUERY(body, '$."ids"[*]'), 'Array(Nullable(Int64))'), CAST(? AS Array(Int64))) OR ifNull(((JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ? AND JSONExtract(JSON_VALUE(body, '$."ids"'), 'Nullable(Int64)') = ?) AND JSONType(body, 'ids') NOT IN ('Array', 'Object', 'Null')), false)) AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{[]any{int64(9007199254740993), int64(9007199254740994)}, int64(9007199254740993), int64(9007199254740994), "%\"ids\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.message = hello",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (JSON_VALUE(body, '$."message"') = ? AND JSON_EXISTS(body, '$."message"'))`,
|
||||
expectedArgs: []any{"hello"},
|
||||
expectedQuery: `WHERE ((JSON_VALUE(body, '$."message"') = ? AND LOWER(body) LIKE LOWER(?)) AND (JSON_EXISTS(body, '$."message"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{"hello", "%hello%", "%\"message\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.status = 1",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."status"'), 'Float64') = ? AND JSON_EXISTS(body, '$."status"'))`,
|
||||
expectedArgs: []any{float64(1)},
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."status"'), 'Float64') = ? AND (JSON_EXISTS(body, '$."status"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{float64(1), "%\"status\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.status = 1.1",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."status"'), 'Float64') = ? AND JSON_EXISTS(body, '$."status"'))`,
|
||||
expectedArgs: []any{float64(1.1)},
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."status"'), 'Float64') = ? AND (JSON_EXISTS(body, '$."status"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{float64(1.1), "%\"status\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.boolkey = true",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."boolkey"'), 'Bool') = ? AND JSON_EXISTS(body, '$."boolkey"'))`,
|
||||
expectedArgs: []any{true},
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."boolkey"'), 'Bool') = ? AND (JSON_EXISTS(body, '$."boolkey"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{true, "%\"boolkey\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.status > 200",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."status"'), 'Float64') > ? AND JSON_EXISTS(body, '$."status"'))`,
|
||||
expectedArgs: []any{float64(200)},
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."status"'), 'Float64') > ? AND (JSON_EXISTS(body, '$."status"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{float64(200), "%\"status\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.message REGEXP 'a*'",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (match(JSON_VALUE(body, '$."message"'), ?) AND JSON_EXISTS(body, '$."message"'))`,
|
||||
expectedArgs: []any{"a*"},
|
||||
expectedQuery: `WHERE (match(JSON_VALUE(body, '$."message"'), ?) AND (JSON_EXISTS(body, '$."message"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{"a*", "%\"message\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: `body.message CONTAINS "hello 'world'"`,
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (LOWER(JSON_VALUE(body, '$."message"')) LIKE LOWER(?) AND JSON_EXISTS(body, '$."message"'))`,
|
||||
expectedArgs: []any{"%hello 'world'%"},
|
||||
expectedQuery: `WHERE ((LOWER(JSON_VALUE(body, '$."message"')) LIKE LOWER(?) AND LOWER(body) LIKE LOWER(?)) AND (JSON_EXISTS(body, '$."message"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{"%hello 'world'%", "%hello 'world'%", "%\"message\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: `body.message EXISTS`,
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE JSON_EXISTS(body, '$."message"')`,
|
||||
expectedQuery: `WHERE (JSON_EXISTS(body, '$."message"') AND LOWER(body) LIKE LOWER(?))`,
|
||||
expectedArgs: []any{"%\"message\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: `body.name IN ('hello', 'world')`,
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE ((JSON_VALUE(body, '$."name"') = ? OR JSON_VALUE(body, '$."name"') = ?) AND JSON_EXISTS(body, '$."name"'))`,
|
||||
expectedArgs: []any{"hello", "world"},
|
||||
expectedQuery: `WHERE (((JSON_VALUE(body, '$."name"') = ? AND LOWER(body) LIKE LOWER(?)) OR (JSON_VALUE(body, '$."name"') = ? AND LOWER(body) LIKE LOWER(?))) AND (JSON_EXISTS(body, '$."name"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{"hello", "%hello%", "world", "%world%", "%\"name\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: `body.value IN (200, 300)`,
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE ((JSONExtract(JSON_VALUE(body, '$."value"'), 'Float64') = ? OR JSONExtract(JSON_VALUE(body, '$."value"'), 'Float64') = ?) AND JSON_EXISTS(body, '$."value"'))`,
|
||||
expectedArgs: []any{float64(200), float64(300)},
|
||||
expectedQuery: `WHERE ((JSONExtract(JSON_VALUE(body, '$."value"'), 'Float64') = ? OR JSONExtract(JSON_VALUE(body, '$."value"'), 'Float64') = ?) AND (JSON_EXISTS(body, '$."value"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{float64(200), float64(300), "%\"value\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
category: "json",
|
||||
query: "body.key-with-hyphen = true",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."key-with-hyphen"'), 'Bool') = ? AND JSON_EXISTS(body, '$."key-with-hyphen"'))`,
|
||||
expectedArgs: []any{true},
|
||||
expectedQuery: `WHERE (JSONExtract(JSON_VALUE(body, '$."key-with-hyphen"'), 'Bool') = ? AND (JSON_EXISTS(body, '$."key-with-hyphen"') AND LOWER(body) LIKE LOWER(?)))`,
|
||||
expectedArgs: []any{true, "%\"key-with-hyphen\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -737,8 +737,8 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
category: "Key-operator-value boundary",
|
||||
query: "greater>than",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE ((attributes_string['greater'] > ? AND mapContains(attributes_string, 'greater')) OR (JSON_VALUE(body, '$."greater"') > ? AND JSON_EXISTS(body, '$."greater"')))`,
|
||||
expectedArgs: []any{"than", "than"},
|
||||
expectedQuery: `WHERE ((attributes_string['greater'] > ? AND mapContains(attributes_string, 'greater')) OR (JSON_VALUE(body, '$."greater"') > ? AND (JSON_EXISTS(body, '$."greater"') AND LOWER(body) LIKE LOWER(?))))`,
|
||||
expectedArgs: []any{"than", "than", "%\"greater\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
@@ -753,8 +753,8 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
category: "Key-operator-value boundary",
|
||||
query: "less<than",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE ((attributes_string['less'] < ? AND mapContains(attributes_string, 'less')) OR (JSON_VALUE(body, '$."less"') < ? AND JSON_EXISTS(body, '$."less"')))`,
|
||||
expectedArgs: []any{"than", "than"},
|
||||
expectedQuery: `WHERE ((attributes_string['less'] < ? AND mapContains(attributes_string, 'less')) OR (JSON_VALUE(body, '$."less"') < ? AND (JSON_EXISTS(body, '$."less"') AND LOWER(body) LIKE LOWER(?))))`,
|
||||
expectedArgs: []any{"than", "than", "%\"less\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
@@ -809,8 +809,8 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
category: "Key-operator-value boundary",
|
||||
query: "user=admin",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE ((attributes_string['user'] = ? AND mapContains(attributes_string, 'user')) OR (JSON_VALUE(body, '$."user"') = ? AND JSON_EXISTS(body, '$."user"')))`,
|
||||
expectedArgs: []any{"admin", "admin"},
|
||||
expectedQuery: `WHERE ((attributes_string['user'] = ? AND mapContains(attributes_string, 'user')) OR ((JSON_VALUE(body, '$."user"') = ? AND LOWER(body) LIKE LOWER(?)) AND (JSON_EXISTS(body, '$."user"') AND LOWER(body) LIKE LOWER(?))))`,
|
||||
expectedArgs: []any{"admin", "admin", "%admin%", "%\"user\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
@@ -1561,15 +1561,15 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
expectedArgs: []any{"download"},
|
||||
expectedErrorContains: "function `hasToken` expects value parameter to be a string",
|
||||
},
|
||||
// A multi-token needle (separator/whitespace) is a clean 400, not a CH execution error.
|
||||
// A multi-token value (separator/whitespace) is a clean 400, not a CH execution error.
|
||||
{
|
||||
category: "hasTokenUnderscoreNeedle",
|
||||
category: "hasTokenUnderscoreSeparator",
|
||||
query: "hasToken(body, \"user_id\")",
|
||||
shouldPass: false,
|
||||
expectedErrorContains: "function `hasToken` matches a single whole token",
|
||||
},
|
||||
{
|
||||
category: "hasTokenWhitespaceNeedle",
|
||||
category: "hasTokenWhitespaceSeparator",
|
||||
query: "hasToken(body, \"production node\")",
|
||||
shouldPass: false,
|
||||
expectedErrorContains: "function `hasToken` matches a single whole token",
|
||||
@@ -2052,8 +2052,8 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
category: "Nested object paths",
|
||||
query: "response.body.data.items[].id=123",
|
||||
shouldPass: true,
|
||||
expectedQuery: `WHERE ((toFloat64(attributes_number['response.body.data.items[].id']) = ? AND mapContains(attributes_number, 'response.body.data.items[].id')) OR (JSONExtract(JSON_VALUE(body, '$."response"."body"."data"."items"[*]."id"'), 'Float64') = ? AND JSON_EXISTS(body, '$."response"."body"."data"."items"[*]."id"')))`,
|
||||
expectedArgs: []any{float64(123), float64(123)},
|
||||
expectedQuery: `WHERE ((toFloat64(attributes_number['response.body.data.items[].id']) = ? AND mapContains(attributes_number, 'response.body.data.items[].id')) OR (JSONExtract(JSON_VALUE(body, '$."response"."body"."data"."items"[*]."id"'), 'Float64') = ? AND (JSON_EXISTS(body, '$."response"."body"."data"."items"[*]."id"') AND LOWER(body) LIKE LOWER(?))))`,
|
||||
expectedArgs: []any{float64(123), float64(123), "%\"response\"%\"body\"%\"data\"%\"items\"%\"id\"%"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
{
|
||||
@@ -2521,7 +2521,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
query: "body NOT LIKE 'done'",
|
||||
shouldPass: true,
|
||||
// lower index search on body even for LIKE
|
||||
expectedQuery: "WHERE (LOWER(body) NOT LIKE LOWER(?) AND attributes_string['body'] NOT LIKE ?)",
|
||||
expectedQuery: "WHERE (body NOT LIKE ? AND attributes_string['body'] NOT LIKE ?)",
|
||||
expectedArgs: []any{"done", "done"},
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
|
||||
@@ -439,27 +439,27 @@ func (c *jsonConditionBuilder) arrayFuncScalarLeaf(node *telemetrytypes.JSONAcce
|
||||
// buildTokenFunctionCondition builds a hasToken search over a body JSON string field:
|
||||
// hasToken(LOWER(<elem>), LOWER(?)) wrapped in arrayExists over any array hops between the
|
||||
// root and the terminal. The field must resolve to a String leaf or a String array.
|
||||
func (c *jsonConditionBuilder) buildTokenFunctionCondition(needle any, sb *sqlbuilder.SelectBuilder) (string, error) {
|
||||
func (c *jsonConditionBuilder) buildTokenFunctionCondition(token any, sb *sqlbuilder.SelectBuilder) (string, error) {
|
||||
if len(c.key.JSONPlan) == 0 {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "function `hasToken` could not resolve a JSON access plan for field `%s`", c.key.Name)
|
||||
}
|
||||
|
||||
return c.buildOredRootChains(func(node *telemetrytypes.JSONAccessNode) (string, error) {
|
||||
return c.tokenLeaf(node, needle, sb)
|
||||
return c.tokenLeaf(node, token, sb)
|
||||
}, sb)
|
||||
}
|
||||
|
||||
// tokenLeaf builds the hasToken match at a terminal node: a direct match for a String leaf
|
||||
// (coalesced to false, as in arrayFuncScalarLeaf), or an arrayExists over the elements for a
|
||||
// String array leaf. hasToken is string-only, so any other element type is rejected.
|
||||
func (c *jsonConditionBuilder) tokenLeaf(node *telemetrytypes.JSONAccessNode, needle any, sb *sqlbuilder.SelectBuilder) (string, error) {
|
||||
func (c *jsonConditionBuilder) tokenLeaf(node *telemetrytypes.JSONAccessNode, token any, sb *sqlbuilder.SelectBuilder) (string, error) {
|
||||
switch node.TerminalConfig.ElemType {
|
||||
case telemetrytypes.String:
|
||||
fieldExpr := fmt.Sprintf("dynamicElement(%s, 'String')", node.FieldPath())
|
||||
return fmt.Sprintf("ifNull(hasToken(LOWER(%s), LOWER(%s)), false)", fieldExpr, sb.Var(needle)), nil
|
||||
return fmt.Sprintf("ifNull(hasToken(LOWER(%s), LOWER(%s)), false)", fieldExpr, sb.Var(token)), nil
|
||||
case telemetrytypes.ArrayString:
|
||||
arrayExpr := fmt.Sprintf("dynamicElement(%s, '%s')", node.FieldPath(), node.TerminalConfig.ElemType.StringValue())
|
||||
return fmt.Sprintf("arrayExists(x -> hasToken(LOWER(x), LOWER(%s)), %s)", sb.Var(needle), arrayExpr), nil
|
||||
return fmt.Sprintf("arrayExists(x -> hasToken(LOWER(x), LOWER(%s)), %s)", sb.Var(token), arrayExpr), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "function `hasToken` only supports string fields; field `%s` is `%s`", c.key.Name, node.TerminalConfig.Key.FieldDataType.StringValue())
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
func parseStrValue(valueStr string, operator qbtypes.FilterOperator) (telemetrytypes.FieldDataType, any) {
|
||||
@@ -92,6 +94,200 @@ func InferDataType(value any, operator qbtypes.FilterOperator, key *telemetrytyp
|
||||
return closure(value, key)
|
||||
}
|
||||
|
||||
// bodyNgramSize is the n of the ngrambf_v1 index over lower(body); a shorter literal yields no
|
||||
// ngram for it to check.
|
||||
const bodyNgramSize = 4
|
||||
|
||||
// likePatternLiterals returns the runs of pattern between unescaped wildcards, with `\` escapes
|
||||
// resolved; every value the pattern matches holds each run verbatim. ClickHouse treats `\` as an
|
||||
// escape only before `%`, `_` and itself, so dropping it elsewhere would yield a run it never requires.
|
||||
func likePatternLiterals(pattern string) []string {
|
||||
var (
|
||||
literals []string
|
||||
run strings.Builder
|
||||
)
|
||||
for i := 0; i < len(pattern); i++ {
|
||||
switch c := pattern[i]; c {
|
||||
case '%', '_':
|
||||
if run.Len() > 0 {
|
||||
literals = append(literals, run.String())
|
||||
run.Reset()
|
||||
}
|
||||
case '\\':
|
||||
if i+1 >= len(pattern) {
|
||||
run.WriteByte('\\')
|
||||
continue
|
||||
}
|
||||
i++
|
||||
if escaped := pattern[i]; escaped != '%' && escaped != '_' && escaped != '\\' {
|
||||
run.WriteByte('\\')
|
||||
}
|
||||
run.WriteByte(pattern[i])
|
||||
default:
|
||||
run.WriteByte(c)
|
||||
}
|
||||
}
|
||||
if run.Len() > 0 {
|
||||
literals = append(literals, run.String())
|
||||
}
|
||||
return literals
|
||||
}
|
||||
|
||||
// jsonEscapable reports whether a JSON encoder is free to rewrite r: `"` and `\` always, `/` by
|
||||
// PHP, `<` `>` `&` by Go, non-printable ASCII by Python's ensure_ascii. The legacy body holds the
|
||||
// producer's own text, so a literal spanning one of these may not be there to find.
|
||||
func jsonEscapable(r rune) bool {
|
||||
return r < 0x20 || r > 0x7e || strings.ContainsRune(`"\/<>&`, r)
|
||||
}
|
||||
|
||||
// jsonTextRuns splits s at every byte an encoder may rewrite, keeping the runs long enough for
|
||||
// the ngram index. A body whose JSON holds s contains each returned run verbatim, in order.
|
||||
func jsonTextRuns(s string) []string {
|
||||
var runs []string
|
||||
for _, run := range strings.FieldsFunc(s, jsonEscapable) {
|
||||
if len(run) >= bodyNgramSize {
|
||||
runs = append(runs, run)
|
||||
}
|
||||
}
|
||||
return runs
|
||||
}
|
||||
|
||||
// bodyPathLiterals returns one literal per component of key's JSON path, quoted the way JSON writes
|
||||
// an object key — which also lets a two-character name reach the ngram size. A component holding a
|
||||
// byte an encoder may rewrite is dropped; JSON writes a parent first, so the order carries.
|
||||
func bodyPathLiterals(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
var literals []string
|
||||
for _, part := range strings.Split(key.Name, ".") {
|
||||
if idx := strings.Index(part, "["); idx >= 0 {
|
||||
part = part[:idx]
|
||||
}
|
||||
if literal := `"` + part + `"`; len(literal) >= bodyNgramSize && !strings.ContainsFunc(part, jsonEscapable) {
|
||||
literals = append(literals, literal)
|
||||
}
|
||||
}
|
||||
return literals
|
||||
}
|
||||
|
||||
// bodyValueLiterals returns the literals a comparison implies in the body text. Only string
|
||||
// comparisons qualify: a number is compared after JSONExtract parses it, which reads 1.23e2
|
||||
// as 123, so the digits of the filter value need not appear in the body at all.
|
||||
func bodyValueLiterals(operator qbtypes.FilterOperator, value any) []string {
|
||||
str, ok := value.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorEqual, qbtypes.FilterOperatorContains:
|
||||
return jsonTextRuns(str)
|
||||
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorILike:
|
||||
var literals []string
|
||||
for _, literal := range likePatternLiterals(str) {
|
||||
literals = append(literals, jsonTextRuns(literal)...)
|
||||
}
|
||||
return literals
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// escapeLikeLiteral escapes the LIKE metacharacters so s matches as literal text. Backslash
|
||||
// goes first, being the escape character itself.
|
||||
func escapeLikeLiteral(s string) string {
|
||||
s = strings.ReplaceAll(s, `\`, `\\`)
|
||||
s = strings.ReplaceAll(s, "%", `\%`)
|
||||
return strings.ReplaceAll(s, "_", `\_`)
|
||||
}
|
||||
|
||||
// bodyIndexPredicate asserts the raw body text holds the literals in order. ILike renders as
|
||||
// LOWER(body) LIKE LOWER(?) on the ClickHouse flavor — the expression both bloom filters index.
|
||||
// They are plain text and backslash-free by construction, so only the LIKE wildcards need escaping.
|
||||
func bodyIndexPredicate(literals []string, sb *sqlbuilder.SelectBuilder) string {
|
||||
if len(literals) == 0 {
|
||||
return ""
|
||||
}
|
||||
escaped := make([]string, 0, len(literals))
|
||||
for _, literal := range literals {
|
||||
escaped = append(escaped, escapeLikeLiteral(literal))
|
||||
}
|
||||
pattern := "%" + strings.Join(escaped, "%") + "%"
|
||||
return sb.ILike(LogsV2BodyColumn, pattern)
|
||||
}
|
||||
|
||||
// legacyBodyIndexPredicates returns what a legacy body JSON filter implies over LOWER(body),
|
||||
// which nothing it compares matches. Path literals ride on the existence assertion and value
|
||||
// literals on the comparison, so each appears once; a negation carries none, matching rows that
|
||||
// lack the path entirely.
|
||||
func legacyBodyIndexPredicates(key *telemetrytypes.TelemetryFieldKey, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) []string {
|
||||
var predicates []string
|
||||
if operator == qbtypes.FilterOperatorExists || operator.IsArrayFunctionOperator() {
|
||||
if predicate := bodyIndexPredicate(bodyPathLiterals(key), sb); predicate != "" {
|
||||
predicates = append(predicates, predicate)
|
||||
}
|
||||
}
|
||||
if operator.IsArrayFunctionOperator() {
|
||||
return append(predicates, bodyArrayFunctionPredicates(operator, value, sb)...)
|
||||
}
|
||||
if predicate := bodyIndexPredicate(bodyValueLiterals(operator, value), sb); predicate != "" {
|
||||
predicates = append(predicates, predicate)
|
||||
}
|
||||
return predicates
|
||||
}
|
||||
|
||||
// bodyArrayFunctionPredicates returns what a has-family filter implies about the body text. has
|
||||
// and hasAll require every element, so each becomes its own predicate; hasAny requires one, so its
|
||||
// arms are ORed, and an element yielding no literal leaves that OR unassertable.
|
||||
func bodyArrayFunctionPredicates(operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) []string {
|
||||
element := value
|
||||
if args, ok := value.([]any); ok && len(args) > 0 {
|
||||
element = args[0]
|
||||
}
|
||||
// the has family compares at the element type it infers, so a quoted number is still a
|
||||
// number here and its digits need not appear in the body — same reason `=` skips them
|
||||
if legacyElemType(element) != telemetrytypes.FieldDataTypeString {
|
||||
return nil
|
||||
}
|
||||
|
||||
values, ok := element.([]any)
|
||||
if !ok {
|
||||
values = []any{element}
|
||||
}
|
||||
|
||||
if operator == qbtypes.FilterOperatorHasAny {
|
||||
// resolve every value before binding anything: one unusable value voids the whole OR
|
||||
runSets := make([][]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
str, ok := v.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
runs := jsonTextRuns(str)
|
||||
if len(runs) == 0 {
|
||||
return nil
|
||||
}
|
||||
runSets = append(runSets, runs)
|
||||
}
|
||||
if len(runSets) == 0 {
|
||||
return nil
|
||||
}
|
||||
arms := make([]string, 0, len(runSets))
|
||||
for _, runs := range runSets {
|
||||
arms = append(arms, bodyIndexPredicate(runs, sb))
|
||||
}
|
||||
return []string{sb.Or(arms...)}
|
||||
}
|
||||
|
||||
var predicates []string
|
||||
for _, v := range values {
|
||||
str, ok := v.(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if predicate := bodyIndexPredicate(jsonTextRuns(str), sb); predicate != "" {
|
||||
predicates = append(predicates, predicate)
|
||||
}
|
||||
}
|
||||
return predicates
|
||||
}
|
||||
|
||||
func getBodyJSONPath(key *telemetrytypes.TelemetryFieldKey) string {
|
||||
parts := strings.Split(key.Name, ".")
|
||||
newParts := []string{}
|
||||
@@ -139,14 +335,14 @@ func GetBodyJSONKeyForExists(_ context.Context, key *telemetrytypes.TelemetryFie
|
||||
return fmt.Sprintf("JSON_EXISTS(body, '$.%s')", getBodyJSONPath(key))
|
||||
}
|
||||
|
||||
// legacyElemType infers the has-family element type from the needle (legacy has no schema). It
|
||||
// scans EVERY value so the chosen array type and all coerced needles agree — else ClickHouse
|
||||
// legacyElemType infers the has-family element type from the arg (legacy has no schema). It
|
||||
// scans EVERY value so the chosen array type and all coerced args agree — else ClickHouse
|
||||
// raises "no supertype ... String" (code 386). Int64 stays distinct from Float64 so a quoted
|
||||
// integer is exact past 2^53 (unquoted literals already arrive as float64, parsed upstream).
|
||||
func legacyElemType(needle any) telemetrytypes.FieldDataType {
|
||||
list, ok := needle.([]any)
|
||||
func legacyElemType(arg any) telemetrytypes.FieldDataType {
|
||||
list, ok := arg.([]any)
|
||||
if !ok {
|
||||
list = []any{needle}
|
||||
list = []any{arg}
|
||||
}
|
||||
if len(list) == 0 {
|
||||
return telemetrytypes.FieldDataTypeString
|
||||
@@ -167,7 +363,7 @@ func legacyElemType(needle any) telemetrytypes.FieldDataType {
|
||||
}
|
||||
default:
|
||||
// booleans (and anything else) -> String; a bool renders to 'true'/'false', so a
|
||||
// bool needle only matches genuine JSON booleans, not truthy numbers/strings.
|
||||
// bool arg only matches genuine JSON booleans, not truthy numbers/strings.
|
||||
allInt, allNumeric = false, false
|
||||
}
|
||||
}
|
||||
@@ -181,9 +377,9 @@ func legacyElemType(needle any) telemetrytypes.FieldDataType {
|
||||
}
|
||||
}
|
||||
|
||||
// legacyCoerceNeedle coerces a needle to elem type dt so its bound-arg type matches the
|
||||
// legacyCoerceElement coerces an element to elem type dt so its bound-arg type matches the
|
||||
// extracted column (legacyElemType guarantees it's coercible).
|
||||
func legacyCoerceNeedle(v any, dt telemetrytypes.FieldDataType) any {
|
||||
func legacyCoerceElement(v any, dt telemetrytypes.FieldDataType) any {
|
||||
switch dt {
|
||||
case telemetrytypes.FieldDataTypeInt64:
|
||||
if s, ok := v.(string); ok {
|
||||
@@ -199,7 +395,7 @@ func legacyCoerceNeedle(v any, dt telemetrytypes.FieldDataType) any {
|
||||
}
|
||||
return v
|
||||
default:
|
||||
return bodyArrayNeedleString(v)
|
||||
return bodyArrayElementString(v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,7 +438,7 @@ func getBodyJSONScalarKey(key *telemetrytypes.TelemetryFieldKey, dt telemetrytyp
|
||||
return expr, guard, true
|
||||
}
|
||||
|
||||
func bodyArrayNeedleString(v any) string {
|
||||
func bodyArrayElementString(v any) string {
|
||||
switch t := v.(type) {
|
||||
case string:
|
||||
return t
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -22,6 +23,28 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
return &conditionBuilder{fm: fm}
|
||||
}
|
||||
|
||||
// Labels read back as String from the `labels` JSON whatever type the metadata claims, so the
|
||||
// collision is always String vs the literal; intrinsic columns keep their own type.
|
||||
func resolveTypeCollisionForFieldName(fieldExpression string, value any) string {
|
||||
if col, isColumn := timeSeriesV4Columns[fieldExpression]; isColumn {
|
||||
columnType := col.Type.GetType()
|
||||
if lowCardinality, ok := col.Type.(schema.LowCardinalityColumnType); ok {
|
||||
columnType = lowCardinality.ElementType.GetType()
|
||||
}
|
||||
if columnType != schema.ColumnTypeEnumString {
|
||||
return fieldExpression
|
||||
}
|
||||
}
|
||||
|
||||
switch value.(type) {
|
||||
case bool:
|
||||
return fmt.Sprintf("accurateCastOrNull(%s, 'Bool')", fieldExpression)
|
||||
case float64:
|
||||
return fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
}
|
||||
return fieldExpression
|
||||
}
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
@@ -42,17 +65,8 @@ func (c *conditionBuilder) conditionFor(
|
||||
return "", err
|
||||
}
|
||||
|
||||
// TODO(srikanthccv): use the same data type collision handling when metrics schemas are updated
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
case []any:
|
||||
if len(v) > 0 && (operator == qbtypes.FilterOperatorBetween || operator == qbtypes.FilterOperatorNotBetween) {
|
||||
if _, ok := v[0].(float64); ok {
|
||||
fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
}
|
||||
}
|
||||
}
|
||||
// TODO(srikanthccv): use querybuilder.DataTypeCollisionHandledFieldName when metrics schemas are updated
|
||||
fieldExpression = resolveTypeCollisionForFieldName(fieldExpression, value)
|
||||
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
@@ -100,6 +114,8 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
// both bounds share one expression, so the lower bound picks the cast
|
||||
fieldExpression = resolveTypeCollisionForFieldName(fieldExpression, values[0])
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -109,6 +125,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
fieldExpression = resolveTypeCollisionForFieldName(fieldExpression, values[0])
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
@@ -117,13 +134,23 @@ func (c *conditionBuilder) conditionFor(
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
return sb.In(fieldExpression, values), nil
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, item := range values {
|
||||
conditions = append(conditions, sb.E(resolveTypeCollisionForFieldName(fieldExpression, item), item))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
return sb.NotIn(fieldExpression, values), nil
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, item := range values {
|
||||
conditions = append(conditions, sb.NE(resolveTypeCollisionForFieldName(fieldExpression, item), item))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
// exists and not exists
|
||||
// in the UI based query builder, `exists` and `not exists` are used for
|
||||
|
||||
@@ -119,8 +119,8 @@ func TestConditionFor(t *testing.T) {
|
||||
},
|
||||
operator: qbtypes.FilterOperatorIn,
|
||||
value: []any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"},
|
||||
expectedSQL: "metric_name IN (?)",
|
||||
expectedArgs: []any{[]any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"}},
|
||||
expectedSQL: "(metric_name = ? OR metric_name = ? OR metric_name = ?)",
|
||||
expectedArgs: []any{"http.server.duration", "http.server.request.duration", "http.server.response.duration"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -155,8 +155,8 @@ func TestConditionFor(t *testing.T) {
|
||||
},
|
||||
operator: qbtypes.FilterOperatorNotIn,
|
||||
value: []any{"debug", "info", "trace"},
|
||||
expectedSQL: "metric_name NOT IN (?)",
|
||||
expectedArgs: []any{[]any{"debug", "info", "trace"}},
|
||||
expectedSQL: "(metric_name <> ? AND metric_name <> ? AND metric_name <> ?)",
|
||||
expectedArgs: []any{"debug", "info", "trace"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -227,6 +227,120 @@ func TestConditionFor(t *testing.T) {
|
||||
expectedSQL: "",
|
||||
expectedError: qbtypes.ErrColumnNotFound,
|
||||
},
|
||||
{
|
||||
name: "Equal operator - bool label casts the JSON read to Bool",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "success",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: true,
|
||||
expectedSQL: "accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Not Equal operator - bool label casts the JSON read to Bool",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "success",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorNotEqual,
|
||||
value: false,
|
||||
expectedSQL: "accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') <> ?",
|
||||
expectedArgs: []any{false},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Equal operator - bool value on a label the metadata calls a string",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "success",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: true,
|
||||
expectedSQL: "accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "In operator - all-bool set casts the JSON read to Bool",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "success",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorIn,
|
||||
value: []any{true, false},
|
||||
expectedSQL: "(accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ? OR accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ?)",
|
||||
expectedArgs: []any{true, false},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "In operator - a mixed set casts each value on its own",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "success",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorIn,
|
||||
value: []any{true, "maybe"},
|
||||
expectedSQL: "(accurateCastOrNull(JSONExtractString(labels, 'success'), 'Bool') = ? OR JSONExtractString(labels, 'success') = ?)",
|
||||
expectedArgs: []any{true, "maybe"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Greater Than operator - a numeric column is compared without a cast",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unix_milli",
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorGreaterThan,
|
||||
value: float64(1747947419000),
|
||||
expectedSQL: "unix_milli > ?",
|
||||
expectedArgs: []any{float64(1747947419000)},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Equal operator - the is_monotonic column is already Bool, no cast",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "is_monotonic",
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: true,
|
||||
expectedSQL: "is_monotonic = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Between operator - the bounds cast the JSON read to Float64",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "latency",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeFloat64,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorBetween,
|
||||
value: []any{float64(10), float64(20)},
|
||||
expectedSQL: "toFloat64OrNull(JSONExtractString(labels, 'latency')) BETWEEN ? AND ?",
|
||||
expectedArgs: []any{float64(10), float64(20)},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Between operator - a numeric column is compared without a cast",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unix_milli",
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
},
|
||||
operator: qbtypes.FilterOperatorBetween,
|
||||
value: []any{float64(1747947419000), float64(1747947429000)},
|
||||
expectedSQL: "unix_milli BETWEEN ? AND ?",
|
||||
expectedArgs: []any{float64(1747947419000), float64(1747947429000)},
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
|
||||
21
tests/fixtures/querier.py
vendored
21
tests/fixtures/querier.py
vendored
@@ -253,6 +253,27 @@ def get_preview_sql(response: requests.Response, name: str) -> str:
|
||||
return statements[0]["db.statement.query"]
|
||||
|
||||
|
||||
def get_preview_skip_indexes(response: requests.Response, name: str) -> dict[str, dict[str, Any]]:
|
||||
"""The skip-index steps of the named query's read funnel, keyed by index name.
|
||||
|
||||
Needs a verbose preview. ClickHouse lists a skip index only when the predicate matches its
|
||||
expression, so an absent entry means it was never consulted."""
|
||||
statements = get_preview_statements(response, name)
|
||||
assert len(statements) == 1, f"expected 1 statement for query {name}, got {len(statements)}"
|
||||
granules = statements[0]["granules"]
|
||||
assert granules is not None, f"query {name} reads no MergeTree table: {statements[0]}"
|
||||
return {step["name"]: step for read in granules["reads"] for step in read["steps"] if step["type"] == "Skip"}
|
||||
|
||||
|
||||
def get_preview_selected_granules(response: requests.Response, name: str) -> int:
|
||||
"""Granules surviving every index step of the named query's read funnel."""
|
||||
statements = get_preview_statements(response, name)
|
||||
assert len(statements) == 1, f"expected 1 statement for query {name}, got {len(statements)}"
|
||||
granules = statements[0]["granules"]
|
||||
assert granules is not None, f"query {name} reads no MergeTree table: {statements[0]}"
|
||||
return granules["selected"]
|
||||
|
||||
|
||||
def aligned_epoch(ago: timedelta, step_seconds: int = DEFAULT_STEP_INTERVAL) -> int:
|
||||
"""Epoch seconds for `now - ago`, floored to a step boundary so seeded
|
||||
points land exactly on the query's toStartOfInterval buckets."""
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
|
||||
|
||||
LOWER = "alpha"
|
||||
UPPER = "ALPHA"
|
||||
PLAIN = "beta"
|
||||
NON_ASCII = "Mixed CASE Ünïcode"
|
||||
SLASH = "GET /api/v1/users"
|
||||
SUPERSTRING = "GET /api/v1/users/42"
|
||||
QUOTE = 'say "hi" now'
|
||||
BACKSLASH = "C:\\tmp\\log"
|
||||
LIKE_META = "100% _off"
|
||||
TAB = "tab\there"
|
||||
CTRL = "ctrl\x01here"
|
||||
|
||||
BODIES = [LOWER, UPPER, PLAIN, NON_ASCII, SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, TAB, CTRL]
|
||||
|
||||
|
||||
# querierlogs/16_body_equality.py with use_json_body on: `body` resolves to body_v2.message,
|
||||
# which the lower(body) companion skips, and the same expressions must still answer alike.
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_bodies",
|
||||
[
|
||||
pytest.param(f"body = '{LOWER}'", {LOWER}, id="equality_exact"),
|
||||
pytest.param(f"body = '{UPPER}'", {UPPER}, id="equality_other_case"),
|
||||
pytest.param("body = 'Alpha'", set(), id="equality_case_must_match"),
|
||||
pytest.param(f"body = '{NON_ASCII}'", {NON_ASCII}, id="equality_non_ascii"),
|
||||
pytest.param("body = 'gamma'", set(), id="equality_no_match"),
|
||||
pytest.param(f"body = '{SLASH}'", {SLASH}, id="equality_slash"),
|
||||
pytest.param("body = 'say \"hi\" now'", {QUOTE}, id="equality_quote"),
|
||||
pytest.param(r"body = 'C:\\tmp\\log'", {BACKSLASH}, id="equality_backslash"),
|
||||
pytest.param(f"body = '{LIKE_META}'", {LIKE_META}, id="equality_like_metacharacters"),
|
||||
pytest.param("body = 'tab\there'", {TAB}, id="equality_tab"),
|
||||
pytest.param("body = 'ctrl\x01here'", {CTRL}, id="equality_control_char"),
|
||||
pytest.param("body = 'GET /api/v1'", set(), id="equality_prefix_does_not_match"),
|
||||
pytest.param(f"body IN ('{LOWER}', '{PLAIN}')", {LOWER, PLAIN}, id="in_excludes_other_case"),
|
||||
pytest.param(f"body IN ('{SLASH}', '{LIKE_META}')", {SLASH, LIKE_META}, id="in_escaped_values"),
|
||||
pytest.param(f"body NOT IN ('{LOWER}', '{UPPER}')", set(BODIES) - {LOWER, UPPER}, id="not_in"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_equality_json(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_bodies: set[str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
# body_v2 comes back parsed; a plain-string body is {"message": <body>}.
|
||||
assert {row["data"]["body"]["message"] for row in get_rows(response)} == expected_bodies
|
||||
92
tests/integration/tests/querierlogs/16_body_equality.py
Normal file
92
tests/integration/tests/querierlogs/16_body_equality.py
Normal file
@@ -0,0 +1,92 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_column_data_from_response, make_query_request
|
||||
|
||||
LOWER = "alpha"
|
||||
UPPER = "ALPHA"
|
||||
PLAIN = "beta"
|
||||
NON_ASCII = "Mixed CASE Ünïcode"
|
||||
SLASH = "GET /api/v1/users"
|
||||
SUPERSTRING = "GET /api/v1/users/42"
|
||||
QUOTE = 'say "hi" now'
|
||||
BACKSLASH = "C:\\tmp\\log"
|
||||
LIKE_META = "100% _off"
|
||||
TAB = "tab\there"
|
||||
CTRL = "ctrl\x01here"
|
||||
|
||||
BODIES = [LOWER, UPPER, PLAIN, NON_ASCII, SLASH, SUPERSTRING, QUOTE, BACKSLASH, LIKE_META, TAB, CTRL]
|
||||
|
||||
|
||||
# `body = ?` carries a case-insensitive LOWER(body) companion for the bloom filters, so a
|
||||
# body differing only in case must still not come back.
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_bodies",
|
||||
[
|
||||
pytest.param(f"body = '{LOWER}'", {LOWER}, id="equality_exact"),
|
||||
pytest.param(f"body = '{UPPER}'", {UPPER}, id="equality_other_case"),
|
||||
pytest.param("body = 'Alpha'", set(), id="equality_case_must_match"),
|
||||
pytest.param(f"body = '{NON_ASCII}'", {NON_ASCII}, id="equality_non_ascii"),
|
||||
pytest.param("body = ''", set(), id="equality_empty"),
|
||||
pytest.param("body = 'gamma'", set(), id="equality_no_match"),
|
||||
# the companion is a LIKE-free equality, so none of these are metacharacters to it
|
||||
pytest.param(f"body = '{SLASH}'", {SLASH}, id="equality_slash"),
|
||||
pytest.param("body = 'say \"hi\" now'", {QUOTE}, id="equality_quote"),
|
||||
pytest.param(r"body = 'C:\\tmp\\log'", {BACKSLASH}, id="equality_backslash"),
|
||||
pytest.param(f"body = '{LIKE_META}'", {LIKE_META}, id="equality_like_metacharacters"),
|
||||
pytest.param("body = 'tab\there'", {TAB}, id="equality_tab"),
|
||||
pytest.param("body = 'ctrl\x01here'", {CTRL}, id="equality_control_char"),
|
||||
# a prefix of another body must not match it
|
||||
pytest.param("body = 'GET /api/v1'", set(), id="equality_prefix_does_not_match"),
|
||||
pytest.param(f"body IN ('{LOWER}', '{PLAIN}')", {LOWER, PLAIN}, id="in_excludes_other_case"),
|
||||
pytest.param(f"body IN ('{SLASH}', '{LIKE_META}')", {SLASH, LIKE_META}, id="in_escaped_values"),
|
||||
pytest.param(f"body NOT IN ('{LOWER}', '{UPPER}')", set(BODIES) - {LOWER, UPPER}, id="not_in"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_equality(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_bodies: set[str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
assert set(get_column_data_from_response(response.json(), "body")) == expected_bodies
|
||||
226
tests/integration/tests/querierlogs/17_index_pruning.py
Normal file
226
tests/integration/tests/querierlogs/17_index_pruning.py
Normal file
@@ -0,0 +1,226 @@
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
build_order_by,
|
||||
build_raw_query,
|
||||
get_preview_selected_granules,
|
||||
get_preview_skip_indexes,
|
||||
get_rows,
|
||||
make_preview_query_request,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
# The legacy body holds the text the producer wrote, so the same value reaches ClickHouse under
|
||||
# different encodings: PHP escapes `/`, Go escapes `&` `<` `>`, Python escapes non-ASCII. The
|
||||
# LOWER(body) predicates the filters carry for the bloom filters must find all of them.
|
||||
BODIES = {
|
||||
"plain": '{"tag":"plain","url":"https://signoz.io/docs","user_id":4242,"status":"timeout_error"}',
|
||||
"php": '{"tag":"php","url":"https:\\/\\/signoz.io\\/docs"}',
|
||||
"go": '{"tag":"go","note":"connection reset \\u0026 retry aborted"}',
|
||||
"python": '{"tag":"python","city":"caf\\u00e9 municipal district"}',
|
||||
"other_case": '{"tag":"other_case","status":"TIMEOUT_ERROR"}',
|
||||
"no_user_id": '{"tag":"no_user_id","status":"ok","url":"https://signoz.io/pricing"}',
|
||||
"tagged": '{"tag":"tagged","labels":["production","webserver"]}',
|
||||
"tagged_escaped": '{"tag":"tagged_escaped","labels":["batch \\u0026 stream","webserver"]}',
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_tags",
|
||||
[
|
||||
pytest.param("body.user_id = 4242", {"plain"}, id="numeric_equality"),
|
||||
pytest.param("body.user_id EXISTS", {"plain"}, id="exists"),
|
||||
# a negated comparison matches the rows without the path, so it carries no predicate
|
||||
pytest.param("body.user_id != 4242", set(BODIES) - {"plain"}, id="not_equal_keeps_pathless_rows"),
|
||||
pytest.param("body.status NOT EXISTS", {"php", "go", "python", "tagged", "tagged_escaped"}, id="not_exists"),
|
||||
pytest.param("body.url = 'https://signoz.io/docs'", {"plain", "php"}, id="equality_escaped_slashes"),
|
||||
pytest.param("body.url CONTAINS 'signoz.io/docs'", {"plain", "php"}, id="contains_escaped_slashes"),
|
||||
pytest.param("body.note = 'connection reset & retry aborted'", {"go"}, id="equality_escaped_ampersand"),
|
||||
pytest.param("body.city = 'café municipal district'", {"python"}, id="equality_escaped_non_ascii"),
|
||||
# the value predicate is case-insensitive where the equality is not
|
||||
pytest.param("body.status = 'timeout_error'", {"plain"}, id="equality_underscore"),
|
||||
pytest.param("body.status = 'TIMEOUT_ERROR'", {"other_case"}, id="equality_other_case"),
|
||||
pytest.param("body.status IN ('timeout_error', 'ok')", {"plain", "no_user_id"}, id="in_carries_one_value_per_arm"),
|
||||
# has and hasAll assert every element, hasAny only one of them
|
||||
pytest.param("has(body.labels[*], 'production')", {"tagged"}, id="has_element"),
|
||||
pytest.param("has(body.labels[*], 'batch & stream')", {"tagged_escaped"}, id="has_escaped_element"),
|
||||
pytest.param("hasAll(body.labels[*], ['production', 'webserver'])", {"tagged"}, id="has_all_needs_every_element"),
|
||||
pytest.param(
|
||||
"hasAny(body.labels[*], ['production', 'batch & stream'])",
|
||||
{"tagged", "tagged_escaped"},
|
||||
id="has_any_needs_one_element",
|
||||
),
|
||||
# 'webserver' is in both, so an ORed literal set must not exclude either row
|
||||
pytest.param(
|
||||
"hasAny(body.labels[*], ['webserver', 'nothing here'])",
|
||||
{"tagged", "tagged_escaped"},
|
||||
id="has_any_across_both",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_logs_body_json_index_predicates(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_tags: set[str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES.values())
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
assert {json.loads(row["data"]["body"])["tag"] for row in get_rows(response)} == expected_tags
|
||||
|
||||
|
||||
# JSON_VALUE matches no index expression, so the literals are what get the bloom filters consulted
|
||||
# at all; the read funnel is what catches one that stops matching.
|
||||
BODY_BLOOM_FILTERS = {"body_index_v2_token", "body_index_v2_ngram"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,prunes_every_granule",
|
||||
[
|
||||
pytest.param("body.status = 'timeout_error'", False, id="value_needle_present"),
|
||||
pytest.param("body.status = 'zz_no_seeded_body_holds_this'", True, id="value_needle_absent"),
|
||||
pytest.param("body.zz_no_seeded_body_holds_this EXISTS", True, id="path_needle_absent"),
|
||||
# a number is compared after JSONExtract parses it, so it carries no value literal - only
|
||||
# its path, which every row holding the key satisfies
|
||||
pytest.param("body.user_id = 999999", False, id="number_carries_only_its_path"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_json_index_prunes_granules(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
prunes_every_granule: bool,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=body,
|
||||
)
|
||||
for i, body in enumerate(BODIES.values())
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_preview_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
skip_indexes = get_preview_skip_indexes(response, "A")
|
||||
assert BODY_BLOOM_FILTERS <= set(skip_indexes), f"body bloom filters not consulted, only: {sorted(skip_indexes)}"
|
||||
|
||||
selected = get_preview_selected_granules(response, "A")
|
||||
if prunes_every_granule:
|
||||
assert selected == 0, f"expected every granule pruned, {selected} survived"
|
||||
else:
|
||||
assert selected > 0, "the granule holding the match must survive"
|
||||
|
||||
|
||||
# `body = ?` matches no index expression on its own; the lowered companion is what the filters
|
||||
# prune on, so its absence from the funnel is the regression this catches.
|
||||
@pytest.mark.parametrize(
|
||||
"expression,prunes_every_granule",
|
||||
[
|
||||
pytest.param("body = 'alpha'", False, id="equality_present_value"),
|
||||
pytest.param("body = 'zz_no_seeded_body_holds_this'", True, id="equality_absent_value"),
|
||||
# IN delegates to the equalities, so every arm carries its own companion
|
||||
pytest.param("body IN ('alpha', 'beta')", False, id="in_present_values"),
|
||||
pytest.param("body IN ('zz_absent_one', 'zz_absent_two')", True, id="in_absent_values"),
|
||||
],
|
||||
)
|
||||
def test_logs_body_equality_prunes_granules(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
prunes_every_granule: bool,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_logs([Logs(timestamp=now - timedelta(seconds=i + 1), resources={"service.name": "api"}, body=body) for i, body in enumerate(["alpha", "ALPHA", "beta"])])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_preview_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc"), build_order_by("id", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
skip_indexes = get_preview_skip_indexes(response, "A")
|
||||
assert BODY_BLOOM_FILTERS <= set(skip_indexes), f"body bloom filters not consulted, only: {sorted(skip_indexes)}"
|
||||
|
||||
selected = get_preview_selected_granules(response, "A")
|
||||
if prunes_every_granule:
|
||||
assert selected == 0, f"expected every granule pruned, {selected} survived"
|
||||
else:
|
||||
assert selected > 0, "the granule holding the match must survive"
|
||||
@@ -0,0 +1,66 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.metrics import Metrics
|
||||
|
||||
METRIC = "test.metric.boollabel"
|
||||
|
||||
|
||||
def test_metrics_filter_bool_label(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(
|
||||
[
|
||||
Metrics(
|
||||
metric_name=METRIC,
|
||||
labels=labels,
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=value,
|
||||
)
|
||||
for labels, value in [
|
||||
({"success": "true"}, 30.0),
|
||||
({"success": "false"}, 10.0),
|
||||
({"success": "1"}, 5.0),
|
||||
({"success": "maybe"}, 3.0),
|
||||
({"region": "us"}, 7.0),
|
||||
]
|
||||
]
|
||||
)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# `true` selects "true" and "1"; `false` selects only "false". "maybe" and the series
|
||||
# carrying no `success` label cast to NULL, so they are in neither result.
|
||||
for expr, expected in [
|
||||
("success = true", 35.0),
|
||||
("success = false", 10.0),
|
||||
("success != true", 10.0),
|
||||
("success IN [true]", 35.0),
|
||||
("success IN [true, false]", 45.0),
|
||||
]:
|
||||
response = querier.make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
querier.build_scalar_query(
|
||||
name="A",
|
||||
signal="metrics",
|
||||
aggregations=[querier.build_metrics_aggregation(METRIC, "latest", "sum", "unspecified", reduce_to="last")],
|
||||
filter_expression=expr,
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, f"{expr}: {response.text}"
|
||||
data = querier.get_scalar_table_data(response.json())
|
||||
assert len(data) == 1, f"{expr}: {data}"
|
||||
assert data[0][-1] == expected, f"{expr}: {data}"
|
||||
Reference in New Issue
Block a user