Compare commits

..

16 Commits

Author SHA1 Message Date
Srikanth Chekuri
8c736a7b51 Merge branch 'main' into SIG_8931 2026-02-20 08:03:40 +05:30
Ishan Uniyal
ed77517150 feat: value suggestion optimised 2026-02-19 19:27:14 +05:30
Ishan Uniyal
761a78f59c feat: key suggestion made common 2026-02-18 06:43:35 +05:30
Ishan
94525c538e Merge branch 'main' into SIG_8931 2026-02-17 12:51:55 +05:30
Ishan Uniyal
0327af3be4 feat: relatedValues and all values together with unchecked filter handling 2026-02-17 12:50:18 +05:30
Ishan
2ed43caf79 Merge branch 'main' into SIG_8931 2026-02-13 12:48:38 +05:30
Ishan Uniyal
939dfa8fc6 feat: revert env.ts 2026-01-30 15:03:22 +05:30
Ishan Uniyal
3c86c02a58 feat: merge conflict 2026-01-30 15:02:13 +05:30
Ishan
b1045bb1e0 Merge branch 'main' into SIG_8931 2026-01-30 14:46:36 +05:30
Ishan Uniyal
2c0ab52806 feat: pr comments signal source 2026-01-30 11:01:28 +05:30
Ishan Uniyal
dc512ab14c feat: code cleanup 2026-01-30 11:01:28 +05:30
Ishan Uniyal
cf80979dfc feat: css updated 2026-01-29 15:57:30 +05:30
Ishan Uniyal
1aae409931 feat: checkbox jitter updated 2026-01-29 15:57:30 +05:30
Ishan Uniyal
c160cf9b4e feat: testcases updated 2026-01-29 15:57:30 +05:30
Ishan Uniyal
799cd4bbca chore: removed env.ts 2026-01-29 15:57:30 +05:30
Ishan Uniyal
8b5a881641 feat: api migration and related values feature 2026-01-29 15:57:30 +05:30
43 changed files with 1346 additions and 1220 deletions

View File

@@ -1,5 +1,6 @@
import axios from 'api';
import { AxiosResponse } from 'axios';
import store from 'store';
import {
QueryKeyRequestProps,
QueryKeySuggestionsResponseProps,
@@ -17,6 +18,12 @@ export const getKeySuggestions = (
signalSource = '',
} = props;
const { globalTime } = store.getState();
const resolvedTimeRange = {
startUnixMilli: Math.floor(globalTime.minTime / 1000000),
endUnixMilli: Math.floor(globalTime.maxTime / 1000000),
};
const encodedSignal = encodeURIComponent(signal);
const encodedSearchText = encodeURIComponent(searchText);
const encodedMetricName = encodeURIComponent(metricName);
@@ -24,7 +31,14 @@ export const getKeySuggestions = (
const encodedFieldDataType = encodeURIComponent(fieldDataType);
const encodedSource = encodeURIComponent(signalSource);
return axios.get(
`/fields/keys?signal=${encodedSignal}&searchText=${encodedSearchText}&metricName=${encodedMetricName}&fieldContext=${encodedFieldContext}&fieldDataType=${encodedFieldDataType}&source=${encodedSource}`,
);
let url = `/fields/keys?signal=${encodedSignal}&searchText=${encodedSearchText}&metricName=${encodedMetricName}&fieldContext=${encodedFieldContext}&fieldDataType=${encodedFieldDataType}&source=${encodedSource}`;
if (resolvedTimeRange.startUnixMilli !== undefined) {
url += `&startUnixMilli=${resolvedTimeRange.startUnixMilli}`;
}
if (resolvedTimeRange.endUnixMilli !== undefined) {
url += `&endUnixMilli=${resolvedTimeRange.endUnixMilli}`;
}
return axios.get(url);
};

View File

@@ -1,5 +1,6 @@
import axios from 'api';
import { AxiosResponse } from 'axios';
import store from 'store';
import {
QueryKeyValueRequestProps,
QueryKeyValueSuggestionsResponseProps,
@@ -8,7 +9,20 @@ import {
export const getValueSuggestions = (
props: QueryKeyValueRequestProps,
): Promise<AxiosResponse<QueryKeyValueSuggestionsResponseProps>> => {
const { signal, key, searchText, signalSource, metricName } = props;
const {
signal,
key,
searchText,
signalSource,
metricName,
existingQuery,
} = props;
const { globalTime } = store.getState();
const resolvedTimeRange = {
startUnixMilli: Math.floor(globalTime.minTime / 1000000),
endUnixMilli: Math.floor(globalTime.maxTime / 1000000),
};
const encodedSignal = encodeURIComponent(signal);
const encodedKey = encodeURIComponent(key);
@@ -16,7 +30,17 @@ export const getValueSuggestions = (
const encodedSearchText = encodeURIComponent(searchText);
const encodedSource = encodeURIComponent(signalSource || '');
return axios.get(
`/fields/values?signal=${encodedSignal}&name=${encodedKey}&searchText=${encodedSearchText}&metricName=${encodedMetricName}&source=${encodedSource}`,
);
let url = `/fields/values?signal=${encodedSignal}&name=${encodedKey}&searchText=${encodedSearchText}&metricName=${encodedMetricName}&source=${encodedSource}`;
if (resolvedTimeRange.startUnixMilli !== undefined) {
url += `&startUnixMilli=${resolvedTimeRange.startUnixMilli}`;
}
if (resolvedTimeRange.endUnixMilli !== undefined) {
url += `&endUnixMilli=${resolvedTimeRange.endUnixMilli}`;
}
if (existingQuery) {
url += `&existingQuery=${encodeURIComponent(existingQuery)}`;
}
return axios.get(url);
};

View File

@@ -1,9 +1,12 @@
import { useEffect, useRef, useState } from 'react';
import { useQuery } from 'react-query';
import { useSelector } from 'react-redux';
import { Select, Spin } from 'antd';
import { getKeySuggestions } from 'api/querySuggestions/getKeySuggestions';
import { AppState } from 'store/reducers';
import { QueryKeyDataSuggestionsProps } from 'types/api/querySuggestions/types';
import { DataSource } from 'types/common/queryBuilder';
import { GlobalReducer } from 'types/reducer/globalTime';
import './ListViewOrderBy.styles.scss';
@@ -34,9 +37,20 @@ function ListViewOrderBy({
>([]);
const debounceTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
// Get time range from Redux global state
const globalTime = useSelector<AppState, GlobalReducer>(
(state) => state.globalTime,
);
// Fetch key suggestions based on debounced input
const { data, isLoading } = useQuery({
queryKey: ['orderByKeySuggestions', dataSource, debouncedInput],
queryKey: [
'orderByKeySuggestions',
dataSource,
debouncedInput,
globalTime.minTime,
globalTime.maxTime,
],
queryFn: async () => {
const response = await getKeySuggestions({
signal: dataSource,

View File

@@ -1,6 +1,7 @@
/* eslint-disable sonarjs/no-identical-functions */
/* eslint-disable sonarjs/cognitive-complexity */
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { useSelector } from 'react-redux';
import { CheckCircleFilled } from '@ant-design/icons';
import {
autocompletion,
@@ -33,6 +34,7 @@ import { useIsDarkMode } from 'hooks/useDarkMode';
import useDebounce from 'hooks/useDebounce';
import { debounce, isNull } from 'lodash-es';
import { Info, TriangleAlert } from 'lucide-react';
import { AppState } from 'store/reducers';
import {
IDetailedError,
IQueryContext,
@@ -41,6 +43,7 @@ import {
import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
import { QueryKeyDataSuggestionsProps } from 'types/api/querySuggestions/types';
import { DataSource } from 'types/common/queryBuilder';
import { GlobalReducer } from 'types/reducer/globalTime';
import {
getCurrentValueIndexAtCursor,
getQueryContextAtCursor,
@@ -112,6 +115,20 @@ function QuerySearch({
const [isFocused, setIsFocused] = useState(false);
const editorRef = useRef<EditorView | null>(null);
// Get time range from Redux global state
const globalTime = useSelector<AppState, GlobalReducer>(
(state) => state.globalTime,
);
// Convert nanoseconds to milliseconds - use useMemo with both deps to avoid multiple updates
const timeRange = useMemo(
() => ({
startUnixMilli: Math.floor(globalTime.minTime / 1000000),
endUnixMilli: Math.floor(globalTime.maxTime / 1000000),
}),
[globalTime.minTime, globalTime.maxTime],
);
const handleQueryValidation = useCallback((newExpression: string): void => {
try {
const validationResponse = validateQuery(newExpression);
@@ -271,7 +288,6 @@ function QuerySearch({
metricName: debouncedMetricName ?? undefined,
signalSource: signalSource as 'meter' | '',
});
if (response.data.data) {
const { keys } = response.data.data;
const options = generateOptions(keys);
@@ -313,7 +329,12 @@ function QuerySearch({
setKeySuggestions([]);
debouncedFetchKeySuggestions();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [dataSource, debouncedMetricName]);
}, [
dataSource,
debouncedMetricName,
timeRange.startUnixMilli,
timeRange.endUnixMilli,
]);
// Add a state for tracking editing mode
const [editingMode, setEditingMode] = useState<
@@ -431,6 +452,7 @@ function QuerySearch({
}
const sanitizedSearchText = searchText ? searchText?.trim() : '';
const existingQuery = queryData.filter?.expression || '';
try {
const response = await getValueSuggestions({
@@ -439,9 +461,9 @@ function QuerySearch({
signal: dataSource,
signalSource: signalSource as 'meter' | '',
metricName: debouncedMetricName ?? undefined,
});
existingQuery,
}); // Skip updates if component unmounted or key changed
// Skip updates if component unmounted or key changed
if (
!isMountedRef.current ||
lastKeyRef.current !== key ||
@@ -453,7 +475,9 @@ function QuerySearch({
// Process the response data
const responseData = response.data as any;
const values = responseData.data?.values || {};
const stringValues = values.stringValues || [];
const relatedValues = values.relatedValues || [];
const stringValues =
relatedValues.length > 0 ? relatedValues : values.stringValues || [];
const numberValues = values.numberValues || [];
// Generate options from string values - explicitly handle empty strings
@@ -528,11 +552,14 @@ function QuerySearch({
},
[
activeKey,
dataSource,
isLoadingSuggestions,
debouncedMetricName,
signalSource,
queryData.filter?.expression,
toggleSuggestions,
dataSource,
signalSource,
debouncedMetricName,
timeRange.startUnixMilli,
timeRange.endUnixMilli,
],
);
@@ -1249,19 +1276,17 @@ function QuerySearch({
if (!queryContext) {
return;
}
// Trigger suggestions based on context
if (editorRef.current) {
// Only trigger suggestions and fetch if editor is focused (i.e., user is interacting)
if (isFocused && editorRef.current) {
toggleSuggestions(10);
}
// Handle value suggestions for value context
if (queryContext.isInValue) {
const { keyToken, currentToken } = queryContext;
const key = keyToken || currentToken;
// Only fetch if needed and if we have a valid key
if (key && key !== activeKey && !isLoadingSuggestions) {
fetchValueSuggestions({ key });
// Handle value suggestions for value context
if (queryContext.isInValue) {
const { keyToken, currentToken } = queryContext;
const key = keyToken || currentToken;
// Only fetch if needed and if we have a valid key
if (key && key !== activeKey && !isLoadingSuggestions) {
fetchValueSuggestions({ key });
}
}
}
}, [
@@ -1270,6 +1295,7 @@ function QuerySearch({
isLoadingSuggestions,
activeKey,
fetchValueSuggestions,
isFocused,
]);
const getTooltipContent = (): JSX.Element => (

View File

@@ -138,6 +138,93 @@
cursor: pointer;
}
}
.search-prompt {
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
padding: 10px 12px;
margin-top: 4px;
border: 1px dashed var(--bg-amber-500);
border-radius: 10px;
color: var(--bg-amber-200);
background: linear-gradient(
90deg,
var(--bg-ink-500) 0%,
var(--bg-ink-400) 100%
);
cursor: pointer;
transition: all 0.16s ease, transform 0.12s ease;
text-align: left;
box-shadow: 0 2px 12px rgba(0, 0, 0, 0.35);
&:hover {
background: linear-gradient(
90deg,
var(--bg-ink-400) 0%,
var(--bg-ink-300) 100%
);
box-shadow: 0 4px 18px rgba(0, 0, 0, 0.45);
}
&:active {
transform: translateY(1px);
}
&__icon {
color: var(--bg-amber-400);
flex-shrink: 0;
}
&__text {
display: flex;
flex-direction: column;
gap: 2px;
}
&__title {
color: var(--bg-amber-200);
}
&__subtitle {
color: var(--bg-amber-300);
font-size: 12px;
}
}
.lightMode & {
.search-prompt {
border: 1px dashed var(--bg-amber-500);
color: var(--bg-amber-800);
background: linear-gradient(
90deg,
var(--bg-vanilla-200) 0%,
var(--bg-vanilla-100) 100%
);
box-shadow: 0 2px 12px rgba(184, 107, 0, 0.08);
&:hover {
background: linear-gradient(
90deg,
var(--bg-vanilla-100) 0%,
var(--bg-vanilla-50) 100%
);
box-shadow: 0 4px 16px rgba(184, 107, 0, 0.15);
}
&__icon {
color: var(--bg-amber-600);
}
&__title {
color: var(--bg-amber-800);
}
&__subtitle {
color: var(--bg-amber-800);
}
}
}
.go-to-docs {
display: flex;
flex-direction: column;

View File

@@ -2,8 +2,15 @@
/* eslint-disable sonarjs/no-identical-functions */
/* eslint-disable jsx-a11y/no-static-element-interactions */
/* eslint-disable jsx-a11y/click-events-have-key-events */
import { Fragment, useMemo, useState } from 'react';
import { Button, Checkbox, Input, Skeleton, Typography } from 'antd';
import {
Fragment,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { Button, Checkbox, Input, InputRef, Skeleton, Typography } from 'antd';
import cx from 'classnames';
import { removeKeysFromExpression } from 'components/QueryBuilderV2/utils';
import {
@@ -11,19 +18,14 @@ import {
QuickFiltersSource,
} from 'components/QuickFilters/types';
import { OPERATORS } from 'constants/antlrQueryConstants';
import {
DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY,
PANEL_TYPES,
} from 'constants/queryBuilder';
import { PANEL_TYPES } from 'constants/queryBuilder';
import { DEBOUNCE_DELAY } from 'constants/queryBuilderFilterConfig';
import { getOperatorValue } from 'container/QueryBuilder/filters/QueryBuilderSearch/utils';
import { useGetAggregateValues } from 'hooks/queryBuilder/useGetAggregateValues';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { useGetQueryKeyValueSuggestions } from 'hooks/querySuggestions/useGetQueryKeyValueSuggestions';
import useDebouncedFn from 'hooks/useDebouncedFunction';
import { cloneDeep, isArray, isEqual, isFunction } from 'lodash-es';
import { ChevronDown, ChevronRight } from 'lucide-react';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { AlertTriangle, ChevronDown, ChevronRight } from 'lucide-react';
import { Query, TagFilterItem } from 'types/api/queryBuilder/queryBuilderData';
import { DataSource } from 'types/common/queryBuilder';
import { v4 as uuid } from 'uuid';
@@ -81,6 +83,12 @@ export default function CheckboxFilter(props: ICheckboxProps): JSX.Element {
return lastUsedQuery || 0;
}, [isListView, source, lastUsedQuery]);
// Extract current filter expression for the active query
const currentFilterExpression = useMemo(() => {
const queryData = currentQuery.builder.queryData?.[activeQueryIndex];
return queryData?.filter?.expression || '';
}, [currentQuery.builder.queryData, activeQueryIndex]);
// Check if this filter has active filters in the query
const isSomeFilterPresentForCurrentAttribute = useMemo(
() =>
@@ -112,47 +120,126 @@ export default function CheckboxFilter(props: ICheckboxProps): JSX.Element {
filter.defaultOpen,
]);
const { data, isLoading } = useGetAggregateValues(
{
aggregateOperator: filter.aggregateOperator || 'noop',
dataSource: filter.dataSource || DataSource.LOGS,
aggregateAttribute: filter.aggregateAttribute || '',
attributeKey: filter.attributeKey.key,
filterAttributeKeyDataType: filter.attributeKey.dataType || DataTypes.EMPTY,
tagType: filter.attributeKey.type || '',
searchText: searchText ?? '',
},
{
enabled: isOpen && source !== QuickFiltersSource.METER_EXPLORER,
keepPreviousData: true,
},
);
const {
data: keyValueSuggestions,
isLoading: isLoadingKeyValueSuggestions,
refetch: refetchKeyValueSuggestions,
} = useGetQueryKeyValueSuggestions({
key: filter.attributeKey.key,
signal: filter.dataSource || DataSource.LOGS,
signalSource: 'meter',
signalSource: source === QuickFiltersSource.METER_EXPLORER ? 'meter' : '',
searchText: searchText || '',
existingQuery: currentFilterExpression,
options: {
enabled: isOpen && source === QuickFiltersSource.METER_EXPLORER,
enabled: isOpen,
keepPreviousData: true,
},
});
const attributeValues: string[] = useMemo(() => {
const dataType = filter.attributeKey.dataType || DataTypes.String;
const searchInputRef = useRef<InputRef | null>(null);
const searchContainerRef = useRef<HTMLDivElement | null>(null);
const previousFiltersItemsRef = useRef(
currentQuery.builder.queryData?.[activeQueryIndex]?.filters?.items,
);
if (source === QuickFiltersSource.METER_EXPLORER && keyValueSuggestions) {
// Process the response data
// Refetch when other filters change (not this filter)
// Watch for when filters.items is different from previous value, indicating other filters changed
useEffect(() => {
const currentFiltersItems =
currentQuery.builder.queryData?.[activeQueryIndex]?.filters?.items;
const previousFiltersItems = previousFiltersItemsRef.current;
// Check if filters items have changed (not the same)
const filtersChanged = !isEqual(previousFiltersItems, currentFiltersItems);
if (isOpen && filtersChanged) {
// Check if OTHER filters (not this filter) have changed
const currentOtherFilters = currentFiltersItems?.filter(
(item) => !isEqual(item.key?.key, filter.attributeKey.key),
);
const previousOtherFilters = previousFiltersItems?.filter(
(item) => !isEqual(item.key?.key, filter.attributeKey.key),
);
// Refetch if other filters changed (not just this filter's values)
const otherFiltersChanged = !isEqual(
currentOtherFilters,
previousOtherFilters,
);
// Only update ref if we have valid API data or if filters actually changed
// Don't update if search returned 0 results to preserve unchecked values
const hasValidData = keyValueSuggestions && !isLoadingKeyValueSuggestions;
if (otherFiltersChanged || hasValidData) {
previousFiltersItemsRef.current = currentFiltersItems;
}
if (otherFiltersChanged) {
refetchKeyValueSuggestions();
}
} else {
previousFiltersItemsRef.current = currentFiltersItems;
}
}, [
activeQueryIndex,
isOpen,
refetchKeyValueSuggestions,
filter.attributeKey.key,
currentQuery.builder.queryData,
keyValueSuggestions,
isLoadingKeyValueSuggestions,
]);
const handleSearchPromptClick = useCallback((): void => {
if (searchContainerRef.current) {
searchContainerRef.current.scrollIntoView({
behavior: 'smooth',
block: 'center',
});
}
if (searchInputRef.current) {
setTimeout(() => searchInputRef.current?.focus({ cursor: 'end' }), 120);
}
}, []);
const isDataComplete = useMemo(() => {
if (keyValueSuggestions) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const responseData = keyValueSuggestions?.data as any;
return responseData.data?.complete || false;
}
return false;
}, [keyValueSuggestions]);
const previousUncheckedValuesRef = useRef<string[]>([]);
const attributeValues: string[] = useMemo(() => {
// const dataType = filter.attributeKey.dataType || DataTypes.String;
if (keyValueSuggestions) {
// Process the response data for all pages
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const responseData = keyValueSuggestions?.data as any;
const values = responseData.data?.values || {};
const relatedValues = values.relatedValues || [];
const stringValues = values.stringValues || [];
const numberValues = values.numberValues || [];
// Combine relatedValues first, then unique stringValues
const valuesToUse = [
...relatedValues,
...stringValues.filter(
(value: string | null | undefined) =>
value !== null &&
value !== undefined &&
value !== '' &&
!relatedValues.includes(value),
),
];
// Generate options from string values - explicitly handle empty strings
const stringOptions = stringValues
const stringOptions = valuesToUse
// Strict filtering for empty string - we'll handle it as a special case if needed
.filter(
(value: string | null | undefined): value is string =>
@@ -167,15 +254,15 @@ export default function CheckboxFilter(props: ICheckboxProps): JSX.Element {
)
.map((value: number) => value.toString());
// Combine all options and make sure we don't have duplicate labels
return [...stringOptions, ...numberOptions];
const baseValues = [...stringOptions, ...numberOptions];
const previousUnchecked = previousUncheckedValuesRef.current || [];
const preservedUnchecked = previousUnchecked.filter(
(value) => !baseValues.includes(value),
);
return [...baseValues, ...preservedUnchecked];
}
const key = DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY[dataType];
return (data?.payload?.[key] || []).filter(
(val) => val !== undefined && val !== null,
);
}, [data?.payload, filter.attributeKey.dataType, keyValueSuggestions, source]);
return [];
}, [keyValueSuggestions]);
const setSearchTextDebounced = useDebouncedFn((...args) => {
setSearchText(args[0] as string);
@@ -249,22 +336,30 @@ export default function CheckboxFilter(props: ICheckboxProps): JSX.Element {
const isMultipleValuesTrueForTheKey =
Object.values(currentFilterState).filter((val) => val).length > 1;
// Sort checked items to the top, then unchecked items
const currentAttributeKeys = useMemo(() => {
// Sort checked items to the top; always show unchecked items beneath, regardless of pagination
const {
visibleCheckedValues,
uncheckedValues,
visibleCheckedCount,
hasMoreChecked,
} = useMemo(() => {
const checkedValues = attributeValues.filter(
(val) => currentFilterState[val],
);
const uncheckedValues = attributeValues.filter(
(val) => !currentFilterState[val],
);
return [...checkedValues, ...uncheckedValues].slice(0, visibleItemsCount);
const unchecked = attributeValues.filter((val) => !currentFilterState[val]);
const visibleChecked = checkedValues.slice(0, visibleItemsCount);
return {
visibleCheckedValues: visibleChecked,
uncheckedValues: unchecked,
visibleCheckedCount: visibleChecked.length,
hasMoreChecked: checkedValues.length > visibleChecked.length,
};
}, [attributeValues, currentFilterState, visibleItemsCount]);
// Count of checked values in the currently visible items
const checkedValuesCount = useMemo(
() => currentAttributeKeys.filter((val) => currentFilterState[val]).length,
[currentAttributeKeys, currentFilterState],
);
useEffect(() => {
previousUncheckedValuesRef.current = uncheckedValues;
}, [uncheckedValues]);
const handleClearFilterAttribute = (): void => {
const preparedQuery: Query = {
@@ -593,35 +688,90 @@ export default function CheckboxFilter(props: ICheckboxProps): JSX.Element {
)}
</section>
</section>
{isOpen &&
(isLoading || isLoadingKeyValueSuggestions) &&
!attributeValues.length && (
<section className="loading">
<Skeleton paragraph={{ rows: 4 }} />
</section>
)}
{isOpen && !isLoading && !isLoadingKeyValueSuggestions && (
{isOpen && isLoadingKeyValueSuggestions && !attributeValues.length && (
<section className="loading">
<Skeleton paragraph={{ rows: 4 }} />
</section>
)}
{isOpen && !isLoadingKeyValueSuggestions && (
<>
{!isEmptyStateWithDocsEnabled && (
<section className="search">
<section className="search" ref={searchContainerRef}>
<Input
placeholder="Filter values"
placeholder="Search values"
onChange={(e): void => setSearchTextDebounced(e.target.value)}
disabled={isFilterDisabled}
ref={searchInputRef}
/>
</section>
)}
{attributeValues.length > 0 ? (
<section className="values">
{currentAttributeKeys.map((value: string, index: number) => (
{visibleCheckedValues.map((value: string) => (
<Fragment key={value}>
{index === checkedValuesCount && checkedValuesCount > 0 && (
<div
key="separator"
className="filter-separator"
data-testid="filter-separator"
<div className="value">
<Checkbox
onChange={(e): void => onChange(value, e.target.checked, false)}
checked={currentFilterState[value]}
disabled={isFilterDisabled}
rootClassName="check-box"
/>
)}
<div
className={cx(
'checkbox-value-section',
isFilterDisabled ? 'filter-disabled' : '',
)}
onClick={(): void => {
if (isFilterDisabled) {
return;
}
onChange(value, currentFilterState[value], true);
}}
>
<div className={`${filter.title} label-${value}`} />
{filter.customRendererForValue ? (
filter.customRendererForValue(value)
) : (
<Typography.Text
className="value-string"
ellipsis={{ tooltip: { placement: 'top' } }}
>
{String(value)}
</Typography.Text>
)}
<Button type="text" className="only-btn">
{isSomeFilterPresentForCurrentAttribute
? currentFilterState[value] && !isMultipleValuesTrueForTheKey
? 'All'
: 'Only'
: 'Only'}
</Button>
<Button type="text" className="toggle-btn">
Toggle
</Button>
</div>
</div>
</Fragment>
))}
{hasMoreChecked && (
<section className="show-more">
<Typography.Text
className="show-more-text"
onClick={(): void => setVisibleItemsCount((prev) => prev + 10)}
>
Show More...
</Typography.Text>
</section>
)}
{visibleCheckedCount > 0 && uncheckedValues.length > 0 && (
<div className="filter-separator" data-testid="filter-separator" />
)}
{uncheckedValues.map((value: string) => (
<Fragment key={value}>
<div className="value">
<Checkbox
onChange={(e): void => onChange(value, e.target.checked, false)}
@@ -681,16 +831,18 @@ export default function CheckboxFilter(props: ICheckboxProps): JSX.Element {
<Typography.Text>No values found</Typography.Text>{' '}
</section>
)}
{visibleItemsCount < attributeValues?.length && (
<section className="show-more">
<Typography.Text
className="show-more-text"
onClick={(): void => setVisibleItemsCount((prev) => prev + 10)}
>
Show More...
</Typography.Text>
</section>
)}
{visibleItemsCount >= attributeValues?.length &&
attributeValues?.length > 0 &&
!isDataComplete && (
<section className="search-prompt" onClick={handleSearchPromptClick}>
<AlertTriangle size={16} className="search-prompt__icon" />
<span className="search-prompt__text">
<Typography.Text className="search-prompt__subtitle">
Tap to search and load more suggestions.
</Typography.Text>
</span>
</section>
)}
</>
)}
</div>

View File

@@ -127,6 +127,34 @@
align-items: center;
padding: 8px;
}
.filters-info {
display: flex;
align-items: center;
padding: 6px 10px 0 10px;
color: var(--bg-vanilla-400);
gap: 6px;
flex-wrap: wrap;
.filters-info-toggle {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 0;
height: auto;
color: var(--bg-vanilla-400);
&:hover {
color: var(--bg-robin-500);
}
}
.filters-info-text {
color: var(--bg-vanilla-400);
font-size: 13px;
line-height: 16px;
}
}
}
.perilin-bg {
@@ -180,5 +208,21 @@
}
}
}
.filters-info {
color: var(--bg-ink-400);
.filters-info-toggle {
color: var(--bg-ink-400);
&:hover {
color: var(--bg-ink-300);
}
}
.filters-info-text {
color: var(--bg-ink-400);
}
}
}
}

View File

@@ -12,7 +12,7 @@ import {
ComboboxList,
ComboboxTrigger,
} from '@signozhq/combobox';
import { Skeleton, Switch, Tooltip, Typography } from 'antd';
import { Button, Skeleton, Switch, Tooltip, Typography } from 'antd';
import getLocalStorageKey from 'api/browser/localstorage/get';
import setLocalStorageKey from 'api/browser/localstorage/set';
import logEvent from 'api/common/logEvent';
@@ -23,7 +23,7 @@ import { PANEL_TYPES } from 'constants/queryBuilder';
import { useApiMonitoringParams } from 'container/ApiMonitoring/queryParams';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { isFunction, isNull } from 'lodash-es';
import { Frown, Settings2 as SettingsIcon } from 'lucide-react';
import { Frown, Lightbulb, Settings2 as SettingsIcon } from 'lucide-react';
import { useAppContext } from 'providers/App/App';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
import { USER_ROLES } from 'types/roles';
@@ -51,6 +51,7 @@ export default function QuickFilters(props: IQuickFiltersProps): JSX.Element {
} = props;
const { user } = useAppContext();
const [isSettingsOpen, setIsSettingsOpen] = useState(false);
const [isFiltersInfoOpen, setIsFiltersInfoOpen] = useState(false);
const isAdmin = user.role === USER_ROLES.ADMIN;
const [params, setParams] = useApiMonitoringParams();
const showIP = params.showIP ?? true;
@@ -291,6 +292,21 @@ export default function QuickFilters(props: IQuickFiltersProps): JSX.Element {
/>
</div>
)}
<section className="filters-info">
<Button
type="link"
className="filters-info-toggle"
onClick={(): void => setIsFiltersInfoOpen((prev) => !prev)}
>
<Lightbulb size={15} />
{isFiltersInfoOpen ? 'Hide info' : 'Adaptive filters'}
</Button>
{isFiltersInfoOpen && (
<Typography.Text className="filters-info-text">
Filter values update automatically based on other selected filters
</Typography.Text>
)}
</section>
<section className="filters">
{filterConfig.map((filter) => {
switch (filter.type) {

View File

@@ -4,6 +4,7 @@ import {
useApiMonitoringParams,
} from 'container/ApiMonitoring/queryParams';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { useGetQueryKeyValueSuggestions } from 'hooks/querySuggestions/useGetQueryKeyValueSuggestions';
import {
otherFiltersResponse,
quickFiltersAttributeValuesResponse,
@@ -24,6 +25,8 @@ jest.mock('hooks/queryBuilder/useQueryBuilder', () => ({
}));
jest.mock('container/ApiMonitoring/queryParams');
jest.mock('hooks/querySuggestions/useGetQueryKeyValueSuggestions');
const handleFilterVisibilityChange = jest.fn();
const redirectWithQueryBuilderData = jest.fn();
const putHandler = jest.fn();
@@ -32,13 +35,15 @@ const mockSetApiMonitoringParams = jest.fn() as jest.MockedFunction<
>;
const mockUseApiMonitoringParams = jest.mocked(useApiMonitoringParams);
const mockUseGetQueryKeyValueSuggestions = jest.mocked(
useGetQueryKeyValueSuggestions,
);
const BASE_URL = ENVIRONMENT.baseURL;
const SIGNAL = SignalType.LOGS;
const quickFiltersListURL = `${BASE_URL}/api/v1/orgs/me/filters/${SIGNAL}`;
const saveQuickFiltersURL = `${BASE_URL}/api/v1/orgs/me/filters`;
const quickFiltersSuggestionsURL = `${BASE_URL}/api/v3/filter_suggestions`;
const quickFiltersAttributeValuesURL = `${BASE_URL}/api/v3/autocomplete/attribute_values`;
const fieldsValuesURL = `${BASE_URL}/api/v1/fields/values`;
const FILTER_OS_DESCRIPTION = 'os.description';
const FILTER_K8S_DEPLOYMENT_NAME = 'k8s.deployment.name';
@@ -62,10 +67,7 @@ const setupServer = (): void => {
putHandler(await req.json());
return res(ctx.status(200), ctx.json({}));
}),
rest.get(quickFiltersAttributeValuesURL, (_req, res, ctx) =>
res(ctx.status(200), ctx.json(quickFiltersAttributeValuesResponse)),
),
rest.get(fieldsValuesURL, (_req, res, ctx) =>
rest.get('*/api/v1/fields/values*', (_req, res, ctx) =>
res(ctx.status(200), ctx.json(quickFiltersAttributeValuesResponse)),
),
);
@@ -135,14 +137,24 @@ beforeEach(() => {
queryData: [
{
queryName: QUERY_NAME,
filters: { items: [{ key: 'test', value: 'value' }] },
filters: { items: [], op: 'AND' },
filter: { expression: '' },
},
],
},
},
lastUsedQuery: 0,
panelType: 'logs',
redirectWithQueryBuilderData,
});
// Mock the hook to return data with mq-kafka
mockUseGetQueryKeyValueSuggestions.mockReturnValue({
data: quickFiltersAttributeValuesResponse,
isLoading: false,
refetch: jest.fn(),
} as any);
mockUseApiMonitoringParams.mockReturnValue([
{ showIP: true } as ApiMonitoringParams,
mockSetApiMonitoringParams,
@@ -259,8 +271,9 @@ describe('Quick Filters', () => {
render(<TestQuickFilters />);
// Prefer role if possible; if label text isnt wired to input, clicking the label text is OK
const target = await screen.findByText('mq-kafka');
// Wait for the filter to load with data
const target = await screen.findByText('mq-kafka', {}, { timeout: 5000 });
await user.click(target);
await waitFor(() => {

View File

@@ -9,6 +9,7 @@ export const useGetQueryKeyValueSuggestions = ({
searchText,
signalSource,
metricName,
existingQuery,
options,
}: {
key: string;
@@ -20,6 +21,7 @@ export const useGetQueryKeyValueSuggestions = ({
AxiosError
>;
metricName?: string;
existingQuery?: string;
}): UseQueryResult<
AxiosResponse<QueryKeyValueSuggestionsResponseProps>,
AxiosError
@@ -40,6 +42,7 @@ export const useGetQueryKeyValueSuggestions = ({
searchText: searchText || '',
signalSource: signalSource as 'meter' | '',
metricName: metricName || '',
existingQuery,
}),
...options,
});

View File

@@ -118,13 +118,13 @@ export const otherFiltersResponse = {
export const quickFiltersAttributeValuesResponse = {
status: 'success',
data: {
stringAttributeValues: [
'mq-kafka',
'otel-demo',
'otlp-python',
'sample-flask',
],
numberAttributeValues: null,
boolAttributeValues: null,
data: {
values: {
relatedValues: ['mq-kafka', 'otel-demo', 'otlp-python', 'sample-flask'],
stringValues: ['mq-kafka', 'otel-demo', 'otlp-python', 'sample-flask'],
numberValues: [],
},
complete: true,
},
},
};

View File

@@ -592,6 +592,39 @@ body {
}
}
.ant-btn-text:hover,
.ant-btn-text:focus-visible {
background-color: var(--bg-vanilla-200) !important;
}
.ant-btn-link:hover,
.ant-btn-link:focus-visible {
background-color: transparent !important;
}
.ant-btn-default:hover,
.ant-btn-default:focus-visible,
.ant-btn-text:not(.ant-btn-primary):hover,
.ant-btn-text:not(.ant-btn-primary):focus-visible,
.ant-btn:not(.ant-btn-primary):not(.ant-btn-dangerous):hover,
.ant-btn:not(.ant-btn-primary):not(.ant-btn-dangerous):focus-visible {
background-color: var(--bg-vanilla-200) !important;
}
.ant-typography:hover,
.ant-typography:focus-visible {
background-color: transparent !important;
}
.ant-tooltip {
--antd-arrow-background-color: var(--bg-vanilla-300);
.ant-tooltip-inner {
background-color: var(--bg-vanilla-200);
color: var(--bg-ink-500);
}
}
// Enhanced legend light mode styles
.u-legend-enhanced {
// Light mode scrollbar styling

View File

@@ -47,6 +47,7 @@ export interface QueryKeyValueRequestProps {
searchText: string;
signalSource?: 'meter' | '';
metricName?: string;
existingQuery?: string;
}
export type SignalType = 'traces' | 'logs' | 'metrics';

287
go.mod
View File

@@ -3,22 +3,23 @@ module github.com/SigNoz/signoz
go 1.24.0
require (
dario.cat/mergo v1.0.2
dario.cat/mergo v1.0.1
github.com/AfterShip/clickhouse-sql-parser v0.4.16
github.com/ClickHouse/clickhouse-go/v2 v2.40.1
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
github.com/SigNoz/signoz-otel-collector v0.142.1-rc.1
github.com/SigNoz/signoz-otel-collector v0.129.10-rc.9
github.com/antlr4-go/antlr/v4 v4.13.1
github.com/antonmedv/expr v1.15.3
github.com/bytedance/sonic v1.14.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/coreos/go-oidc/v3 v3.16.0
github.com/coreos/go-oidc/v3 v3.14.1
github.com/dgraph-io/ristretto/v2 v2.3.0
github.com/dustin/go-humanize v1.0.1
github.com/gin-gonic/gin v1.11.0
github.com/go-co-op/gocron v1.30.1
github.com/go-openapi/runtime v0.28.0
github.com/go-openapi/strfmt v0.24.0
github.com/go-openapi/strfmt v0.23.0
github.com/go-redis/redismock/v9 v9.2.0
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/gojek/heimdall/v7 v7.0.3
@@ -29,22 +30,22 @@ require (
github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674
github.com/huandu/go-sqlbuilder v1.35.0
github.com/jackc/pgx/v5 v5.7.6
github.com/json-iterator/go v1.1.13-0.20220915233716-71ac16282d12
github.com/json-iterator/go v1.1.12
github.com/knadh/koanf v1.5.0
github.com/knadh/koanf/v2 v2.3.0
github.com/mailru/easyjson v0.9.0
github.com/open-telemetry/opamp-go v0.22.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.142.0
github.com/knadh/koanf/v2 v2.2.0
github.com/mailru/easyjson v0.7.7
github.com/open-telemetry/opamp-go v0.19.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.128.0
github.com/openfga/api/proto v0.0.0-20250909172242-b4b2a12f5c67
github.com/openfga/language/pkg/go v0.2.0-beta.2.0.20250428093642-7aeebe78bbfe
github.com/opentracing/opentracing-go v1.2.0
github.com/pkg/errors v0.9.1
github.com/prometheus/alertmanager v0.28.1
github.com/prometheus/client_golang v1.23.2
github.com/prometheus/common v0.67.4
github.com/prometheus/prometheus v0.308.0
github.com/prometheus/common v0.66.1
github.com/prometheus/prometheus v0.304.1
github.com/redis/go-redis/extra/redisotel/v9 v9.15.1
github.com/redis/go-redis/v9 v9.17.2
github.com/redis/go-redis/v9 v9.15.1
github.com/rs/cors v1.11.1
github.com/russellhaering/gosaml2 v0.9.0
github.com/russellhaering/goxmldsig v1.2.0
@@ -53,7 +54,7 @@ require (
github.com/sethvargo/go-password v0.2.0
github.com/smartystreets/goconvey v1.8.1
github.com/soheilhy/cmux v0.1.5
github.com/spf13/cobra v1.10.2
github.com/spf13/cobra v1.10.1
github.com/srikanthccv/ClickHouse-go-mock v0.13.0
github.com/stretchr/testify v1.11.1
github.com/swaggest/jsonschema-go v0.3.78
@@ -63,59 +64,43 @@ require (
github.com/uptrace/bun/dialect/pgdialect v1.2.9
github.com/uptrace/bun/dialect/sqlitedialect v1.2.9
github.com/uptrace/bun/extra/bunotel v1.2.9
go.opentelemetry.io/collector/confmap v1.48.0
go.opentelemetry.io/collector/otelcol v0.142.0
go.opentelemetry.io/collector/pdata v1.48.0
go.opentelemetry.io/collector/confmap v1.34.0
go.opentelemetry.io/collector/otelcol v0.128.0
go.opentelemetry.io/collector/pdata v1.34.0
go.opentelemetry.io/contrib/config v0.10.0
go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0
go.opentelemetry.io/otel v1.39.0
go.opentelemetry.io/otel/metric v1.39.0
go.opentelemetry.io/otel/sdk v1.39.0
go.opentelemetry.io/otel/trace v1.39.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0
go.opentelemetry.io/otel v1.38.0
go.opentelemetry.io/otel/metric v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.1
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.46.0
golang.org/x/exp v0.0.0-20251113190631-e25ba8c21ef6
golang.org/x/net v0.48.0
golang.org/x/oauth2 v0.33.0
golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b
golang.org/x/net v0.47.0
golang.org/x/oauth2 v0.30.0
golang.org/x/sync v0.19.0
golang.org/x/text v0.32.0
google.golang.org/protobuf v1.36.10
google.golang.org/protobuf v1.36.9
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/apimachinery v0.35.0-alpha.0
k8s.io/apimachinery v0.34.0
modernc.org/sqlite v1.39.1
)
require (
github.com/aws/aws-sdk-go-v2 v1.40.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.14 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.4 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.1 // indirect
github.com/aws/smithy-go v1.23.2 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.14.1 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
github.com/goccy/go-yaml v1.19.0 // indirect
github.com/goccy/go-yaml v1.18.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/prometheus/client_golang/exp v0.0.0-20250914183048-a974e0d45e0a // indirect
github.com/redis/go-redis/extra/rediscmd/v9 v9.15.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/swaggest/refl v1.4.0 // indirect
@@ -123,27 +108,24 @@ require (
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect
go.opentelemetry.io/collector/client v1.48.0 // indirect
go.opentelemetry.io/collector/config/configoptional v1.48.0 // indirect
go.opentelemetry.io/collector/config/configretry v1.48.0 // indirect
go.opentelemetry.io/collector/exporter/exporterhelper v0.142.0 // indirect
go.opentelemetry.io/collector/pdata/xpdata v0.142.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.opentelemetry.io/collector/config/configretry v1.34.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/tools/godoc v0.1.0-deprecated // indirect
modernc.org/libc v1.66.10 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
)
require (
cel.dev/expr v0.25.1 // indirect
cloud.google.com/go/auth v0.17.0 // indirect
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go/auth v0.16.1 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.9.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
cloud.google.com/go/compute/metadata v0.8.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect
github.com/ClickHouse/ch-go v0.67.0 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Yiling-J/theine-go v0.6.2 // indirect
@@ -151,35 +133,35 @@ require (
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/aws/aws-sdk-go v1.55.8 // indirect
github.com/aws/aws-sdk-go v1.55.7 // indirect
github.com/beevik/etree v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/coder/quartz v0.1.2 // indirect
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dennwc/varint v1.0.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/ebitengine/purego v0.9.1 // indirect
github.com/ebitengine/purego v0.8.4 // indirect
github.com/edsrzf/mmap-go v1.2.0 // indirect
github.com/elastic/lunes v0.2.0 // indirect
github.com/elastic/lunes v0.1.0 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/expr-lang/expr v1.17.7
github.com/expr-lang/expr v1.17.5
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-openapi/analysis v0.23.0 // indirect
github.com/go-openapi/errors v0.22.3 // indirect
github.com/go-openapi/errors v0.22.0 // indirect
github.com/go-openapi/jsonpointer v0.21.0 // indirect
github.com/go-openapi/jsonreference v0.21.0 // indirect
github.com/go-openapi/loads v0.22.0 // indirect
@@ -196,19 +178,19 @@ require (
github.com/google/btree v1.1.3 // indirect
github.com/google/cel-go v0.26.1 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.14.2 // indirect
github.com/gopherjs/gopherjs v1.17.2 // indirect
github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.3.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-sockaddr v1.0.7 // indirect
github.com/hashicorp/go-version v1.8.0 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/memberlist v0.5.1 // indirect
@@ -224,21 +206,21 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/compress v1.18.2 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/leodido/go-syslog/v4 v4.3.0 // indirect
github.com/leodido/go-syslog/v4 v4.2.0 // indirect
github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect
github.com/magefile/mage v1.15.0 // indirect
github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mdlayher/socket v0.5.1 // indirect
github.com/mdlayher/socket v0.4.1 // indirect
github.com/mdlayher/vsock v1.2.1 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/miekg/dns v1.1.68 // indirect
github.com/miekg/dns v1.1.65 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
@@ -247,14 +229,14 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/natefinch/wrap v0.2.0 // indirect
github.com/oklog/run v1.2.0 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/oklog/ulid/v2 v2.1.1
github.com/open-feature/go-sdk v1.17.0
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.142.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.142.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.142.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.142.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.128.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.128.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.128.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.128.0 // indirect
github.com/openfga/openfga v1.10.1
github.com/paulmach/orb v0.11.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
@@ -264,10 +246,10 @@ require (
github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect
github.com/pressly/goose/v3 v3.25.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/exporter-toolkit v0.15.0 // indirect
github.com/prometheus/otlptranslator v1.0.0 // indirect
github.com/prometheus/procfs v0.19.2 // indirect
github.com/prometheus/sigv4 v0.3.0 // indirect
github.com/prometheus/exporter-toolkit v0.14.0 // indirect
github.com/prometheus/otlptranslator v0.0.0-20250320144820-d800c8b0eb07 // indirect
github.com/prometheus/procfs v0.16.1 // indirect
github.com/prometheus/sigv4 v0.1.2 // indirect
github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/sagikazarmark/locafero v0.9.0 // indirect
@@ -275,7 +257,7 @@ require (
github.com/segmentio/asm v1.2.0 // indirect
github.com/segmentio/backo-go v1.0.1 // indirect
github.com/sethvargo/go-retry v0.3.0 // indirect
github.com/shirou/gopsutil/v4 v4.25.11 // indirect
github.com/shirou/gopsutil/v4 v4.25.5 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c // indirect
github.com/shurcooL/vfsgen v0.0.0-20230704071429-0000e147ea92 // indirect
@@ -290,9 +272,9 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/swaggest/openapi-go v0.2.60
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tklauser/go-sysconf v0.3.16 // indirect
github.com/tklauser/numcpus v0.11.0 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tklauser/go-sysconf v0.3.15 // indirect
github.com/tklauser/numcpus v0.10.0 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/trivago/tgo v1.0.7 // indirect
github.com/valyala/fastjson v1.6.4 // indirect
@@ -301,82 +283,83 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.mongodb.org/mongo-driver v1.17.4 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/collector/component v1.48.0 // indirect
go.opentelemetry.io/collector/component/componentstatus v0.142.0 // indirect
go.opentelemetry.io/collector/component/componenttest v0.142.0 // indirect
go.opentelemetry.io/collector/config/configtelemetry v0.142.0 // indirect
go.opentelemetry.io/collector/confmap/provider/envprovider v1.48.0 // indirect
go.opentelemetry.io/collector/confmap/provider/fileprovider v1.48.0 // indirect
go.opentelemetry.io/collector/confmap/xconfmap v0.142.0 // indirect
go.opentelemetry.io/collector/connector v0.142.0 // indirect
go.opentelemetry.io/collector/connector/connectortest v0.142.0 // indirect
go.opentelemetry.io/collector/connector/xconnector v0.142.0 // indirect
go.opentelemetry.io/collector/consumer v1.48.0 // indirect
go.opentelemetry.io/collector/consumer/consumererror v0.142.0 // indirect
go.opentelemetry.io/collector/consumer/consumertest v0.142.0 // indirect
go.opentelemetry.io/collector/consumer/xconsumer v0.142.0 // indirect
go.opentelemetry.io/collector/exporter v1.48.0 // indirect
go.opentelemetry.io/collector/exporter/exportertest v0.142.0 // indirect
go.opentelemetry.io/collector/exporter/xexporter v0.142.0 // indirect
go.opentelemetry.io/collector/extension v1.48.0 // indirect
go.opentelemetry.io/collector/extension/extensioncapabilities v0.142.0 // indirect
go.opentelemetry.io/collector/extension/extensiontest v0.142.0 // indirect
go.opentelemetry.io/collector/extension/xextension v0.142.0 // indirect
go.opentelemetry.io/collector/featuregate v1.48.0 // indirect
go.opentelemetry.io/collector/internal/fanoutconsumer v0.142.0 // indirect
go.opentelemetry.io/collector/internal/telemetry v0.142.0 // indirect
go.opentelemetry.io/collector/pdata/pprofile v0.142.0 // indirect
go.opentelemetry.io/collector/pdata/testdata v0.142.0 // indirect
go.opentelemetry.io/collector/pipeline v1.48.0 // indirect
go.opentelemetry.io/collector/pipeline/xpipeline v0.142.0 // indirect
go.opentelemetry.io/collector/processor v1.48.0 // indirect
go.opentelemetry.io/collector/processor/processorhelper v0.142.0 // indirect
go.opentelemetry.io/collector/processor/processortest v0.142.0 // indirect
go.opentelemetry.io/collector/processor/xprocessor v0.142.0 // indirect
go.opentelemetry.io/collector/receiver v1.48.0 // indirect
go.opentelemetry.io/collector/receiver/receiverhelper v0.142.0 // indirect
go.opentelemetry.io/collector/receiver/receivertest v0.142.0 // indirect
go.opentelemetry.io/collector/receiver/xreceiver v0.142.0 // indirect
go.opentelemetry.io/collector/semconv v0.128.1-0.20250610090210-188191247685
go.opentelemetry.io/collector/service v0.142.0 // indirect
go.opentelemetry.io/collector/service/hostcapabilities v0.142.0 // indirect
go.opentelemetry.io/contrib/bridges/otelzap v0.13.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.63.0 // indirect
go.opentelemetry.io/contrib/otelconf v0.18.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.60.0
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.14.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.39.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.39.0 // indirect
go.opentelemetry.io/otel/log v0.15.0 // indirect
go.opentelemetry.io/otel/sdk/log v0.14.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.39.0
go.opentelemetry.io/proto/otlp v1.9.0 // indirect
go.mongodb.org/mongo-driver v1.17.1 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/collector/component v1.34.0 // indirect
go.opentelemetry.io/collector/component/componentstatus v0.128.0 // indirect
go.opentelemetry.io/collector/component/componenttest v0.128.0 // indirect
go.opentelemetry.io/collector/config/configtelemetry v0.128.0 // indirect
go.opentelemetry.io/collector/confmap/provider/envprovider v1.34.0 // indirect
go.opentelemetry.io/collector/confmap/provider/fileprovider v1.34.0 // indirect
go.opentelemetry.io/collector/confmap/xconfmap v0.128.0 // indirect
go.opentelemetry.io/collector/connector v0.128.0 // indirect
go.opentelemetry.io/collector/connector/connectortest v0.128.0 // indirect
go.opentelemetry.io/collector/connector/xconnector v0.128.0 // indirect
go.opentelemetry.io/collector/consumer v1.34.0 // indirect
go.opentelemetry.io/collector/consumer/consumererror v0.128.0 // indirect
go.opentelemetry.io/collector/consumer/consumertest v0.128.0 // indirect
go.opentelemetry.io/collector/consumer/xconsumer v0.128.0 // indirect
go.opentelemetry.io/collector/exporter v0.128.0 // indirect
go.opentelemetry.io/collector/exporter/exportertest v0.128.0 // indirect
go.opentelemetry.io/collector/exporter/xexporter v0.128.0 // indirect
go.opentelemetry.io/collector/extension v1.34.0 // indirect
go.opentelemetry.io/collector/extension/extensioncapabilities v0.128.0 // indirect
go.opentelemetry.io/collector/extension/extensiontest v0.128.0 // indirect
go.opentelemetry.io/collector/extension/xextension v0.128.0 // indirect
go.opentelemetry.io/collector/featuregate v1.34.0 // indirect
go.opentelemetry.io/collector/internal/fanoutconsumer v0.128.0 // indirect
go.opentelemetry.io/collector/internal/telemetry v0.128.0 // indirect
go.opentelemetry.io/collector/pdata/pprofile v0.128.0 // indirect
go.opentelemetry.io/collector/pdata/testdata v0.128.0 // indirect
go.opentelemetry.io/collector/pipeline v0.128.0 // indirect
go.opentelemetry.io/collector/pipeline/xpipeline v0.128.0 // indirect
go.opentelemetry.io/collector/processor v1.34.0 // indirect
go.opentelemetry.io/collector/processor/processorhelper v0.128.0 // indirect
go.opentelemetry.io/collector/processor/processortest v0.128.0 // indirect
go.opentelemetry.io/collector/processor/xprocessor v0.128.0 // indirect
go.opentelemetry.io/collector/receiver v1.34.0 // indirect
go.opentelemetry.io/collector/receiver/receiverhelper v0.128.0 // indirect
go.opentelemetry.io/collector/receiver/receivertest v0.128.0 // indirect
go.opentelemetry.io/collector/receiver/xreceiver v0.128.0 // indirect
go.opentelemetry.io/collector/semconv v0.128.0
go.opentelemetry.io/collector/service v0.128.0 // indirect
go.opentelemetry.io/collector/service/hostcapabilities v0.128.0 // indirect
go.opentelemetry.io/contrib/bridges/otelzap v0.11.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.60.0 // indirect
go.opentelemetry.io/contrib/otelconf v0.16.0 // indirect
go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.12.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.12.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.36.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.36.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.36.0 // indirect
go.opentelemetry.io/otel/exporters/prometheus v0.58.0
go.opentelemetry.io/otel/exporters/stdout/stdoutlog v0.12.2 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 // indirect
go.opentelemetry.io/otel/log v0.12.2 // indirect
go.opentelemetry.io/otel/sdk/log v0.12.2 // indirect
go.opentelemetry.io/otel/sdk/metric v1.38.0
go.opentelemetry.io/proto/otlp v1.8.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/mock v0.6.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.30.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/time v0.11.0 // indirect
golang.org/x/tools v0.39.0 // indirect
gonum.org/v1/gonum v0.16.0 // indirect
google.golang.org/api v0.257.0
google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
google.golang.org/grpc v1.77.0 // indirect
google.golang.org/api v0.236.0
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
google.golang.org/grpc v1.75.1 // indirect
gopkg.in/telebot.v3 v3.3.8 // indirect
k8s.io/client-go v0.34.2 // indirect
k8s.io/client-go v0.34.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)
replace github.com/expr-lang/expr => github.com/SigNoz/expr v1.17.7-beta

731
go.sum

File diff suppressed because it is too large Load Diff

View File

@@ -76,7 +76,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
// add the paths that are not promoted but have indexes
for path, indexes := range aggr {
path := strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
path := strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix)
path = telemetrytypes.BodyJSONStringSearchPrefix + path
response = append(response, promotetypes.PromotePath{
Path: path,
@@ -156,7 +156,7 @@ func (m *module) PromoteAndIndexPaths(
}
}
if len(it.Indexes) > 0 {
parentColumn := telemetrylogs.LogsV2BodyV2Column
parentColumn := telemetrylogs.LogsV2BodyJSONColumn
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
parentColumn = telemetrylogs.LogsV2BodyPromotedColumn

View File

@@ -87,24 +87,6 @@ func (client *client) Read(ctx context.Context, query *prompb.Query, sortSeries
return remote.FromQueryResult(sortSeries, res), nil
}
func (c *client) ReadMultiple(ctx context.Context, queries []*prompb.Query, sortSeries bool) (storage.SeriesSet, error) {
if len(queries) == 0 {
return storage.EmptySeriesSet(), nil
}
if len(queries) == 1 {
return c.Read(ctx, queries[0], sortSeries)
}
sets := make([]storage.SeriesSet, 0, len(queries))
for _, q := range queries {
ss, err := c.Read(ctx, q, sortSeries)
if err != nil {
return nil, err
}
sets = append(sets, ss)
}
return storage.NewMergeSeriesSet(sets, 0, storage.ChainedSeriesMerge), nil
}
func (client *client) queryToClickhouseQuery(_ context.Context, query *prompb.Query, metricName string, subQuery bool) (string, []any, error) {
var clickHouseQuery string
var conditions []string

View File

@@ -2,7 +2,6 @@ package prometheus
import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
)
@@ -16,24 +15,30 @@ func RemoveExtraLabels(res *promql.Result, labelsToRemove ...string) error {
toRemove[l] = struct{}{}
}
dropLabels := func(metric labels.Labels) labels.Labels {
b := labels.NewBuilder(metric)
for name := range toRemove {
b.Del(name)
}
return b.Labels()
}
switch res.Value.(type) {
case promql.Vector:
value := res.Value.(promql.Vector)
for i := range value {
(value)[i].Metric = dropLabels((value)[i].Metric)
series := &(value)[i]
dst := series.Metric[:0]
for _, lbl := range series.Metric {
if _, drop := toRemove[lbl.Name]; !drop {
dst = append(dst, lbl)
}
}
series.Metric = dst
}
case promql.Matrix:
value := res.Value.(promql.Matrix)
for i := range value {
(value)[i].Metric = dropLabels((value)[i].Metric)
series := &(value)[i]
dst := series.Metric[:0]
for _, lbl := range series.Metric {
if _, drop := toRemove[lbl.Name]; !drop {
dst = append(dst, lbl)
}
}
series.Metric = dst
}
case promql.Scalar:
return nil

View File

@@ -10,9 +10,11 @@ import (
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrystore"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/bytedance/sonic"
)
type builderQuery[T any] struct {
@@ -248,6 +250,40 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string,
return nil, err
}
// merge body_json and promoted into body
if q.spec.Signal == telemetrytypes.SignalLogs {
switch typedPayload := payload.(type) {
case *qbtypes.RawData:
for _, rr := range typedPayload.Rows {
seeder := func() error {
body, ok := rr.Data[telemetrylogs.LogsV2BodyJSONColumn].(map[string]any)
if !ok {
return nil
}
promoted, ok := rr.Data[telemetrylogs.LogsV2BodyPromotedColumn].(map[string]any)
if !ok {
return nil
}
seed(promoted, body)
str, err := sonic.MarshalString(body)
if err != nil {
return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to marshal body")
}
rr.Data["body"] = str
return nil
}
err := seeder()
if err != nil {
return nil, err
}
delete(rr.Data, telemetrylogs.LogsV2BodyJSONColumn)
delete(rr.Data, telemetrylogs.LogsV2BodyPromotedColumn)
}
payload = typedPayload
}
}
return &qbtypes.Result{
Type: q.kind,
Value: payload,
@@ -375,3 +411,18 @@ func decodeCursor(cur string) (int64, error) {
}
return strconv.ParseInt(string(b), 10, 64)
}
func seed(promoted map[string]any, body map[string]any) {
for key, fromValue := range promoted {
if toValue, ok := body[key]; !ok {
body[key] = fromValue
} else {
if fromValue, ok := fromValue.(map[string]any); ok {
if toValue, ok := toValue.(map[string]any); ok {
seed(fromValue, toValue)
body[key] = toValue
}
}
}
}
}

View File

@@ -14,6 +14,7 @@ import (
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/bytedance/sonic"
)
var (
@@ -393,11 +394,17 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
// de-reference the typed pointer to any
val := reflect.ValueOf(cellPtr).Elem().Interface()
// Post-process JSON columns: normalize into String value
// Post-process JSON columns: normalize into structured values
if strings.HasPrefix(strings.ToUpper(colTypes[i].DatabaseTypeName()), "JSON") {
switch x := val.(type) {
case []byte:
val = string(x)
if len(x) > 0 {
var v any
if err := sonic.Unmarshal(x, &v); err == nil {
val = v
}
}
default:
// already a structured type (map[string]any, []any, etc.)
}

View File

@@ -16,7 +16,6 @@ import (
"github.com/SigNoz/signoz/pkg/querybuilder"
qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
)
@@ -241,13 +240,13 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
var series []*qbv5.TimeSeries
for _, v := range matrix {
var s qbv5.TimeSeries
lbls := make([]*qbv5.Label, 0, v.Metric.Len())
v.Metric.Range(func(l labels.Label) {
lbls := make([]*qbv5.Label, 0, len(v.Metric))
for name, value := range v.Metric.Copy().Map() {
lbls = append(lbls, &qbv5.Label{
Key: telemetrytypes.TelemetryFieldKey{Name: l.Name},
Value: l.Value,
Key: telemetrytypes.TelemetryFieldKey{Name: name},
Value: value,
})
})
}
s.Labels = lbls

View File

@@ -19,7 +19,6 @@ import (
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/ruletypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/prometheus/prometheus/model/labels"
"github.com/prometheus/prometheus/promql"
)
@@ -462,12 +461,12 @@ func toCommonSeries(series promql.Series) v3.Series {
Points: make([]v3.Point, 0),
}
series.Metric.Range(func(l labels.Label) {
commonSeries.Labels[l.Name] = l.Value
for _, lbl := range series.Metric {
commonSeries.Labels[lbl.Name] = lbl.Value
commonSeries.LabelsArray = append(commonSeries.LabelsArray, map[string]string{
l.Name: l.Value,
lbl.Name: lbl.Value,
})
})
}
for _, f := range series.Floats {
commonSeries.Points = append(commonSeries.Points, v3.Point{

View File

@@ -204,10 +204,7 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
// While we expect user not to send the mixed data types, it inevitably happens
// So we handle the data type collisions here
switch key.FieldDataType {
case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString, telemetrytypes.FieldDataTypeJSON:
if key.FieldDataType == telemetrytypes.FieldDataTypeJSON {
tblFieldName = fmt.Sprintf("toString(%s)", tblFieldName)
}
case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString:
switch v := value.(type) {
case float64:
// try to convert the string value to to number
@@ -222,6 +219,7 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
// we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string
value = fmt.Sprintf("%t", v)
}
case telemetrytypes.FieldDataTypeInt64,
telemetrytypes.FieldDataTypeArrayInt64,
telemetrytypes.FieldDataTypeNumber,

View File

@@ -313,31 +313,37 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
return ""
}
child := ctx.GetChild(0)
var searchText string
if keyCtx, ok := child.(*grammar.KeyContext); ok {
// create a full text search condition on the body field
searchText = keyCtx.GetText()
keyText := keyCtx.GetText()
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(keyText), v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
return cond
} else if valCtx, ok := child.(*grammar.ValueContext); ok {
var text string
if valCtx.QUOTED_TEXT() != nil {
searchText = trimQuotes(valCtx.QUOTED_TEXT().GetText())
text = trimQuotes(valCtx.QUOTED_TEXT().GetText())
} else if valCtx.NUMBER() != nil {
searchText = valCtx.NUMBER().GetText()
text = valCtx.NUMBER().GetText()
} else if valCtx.BOOL() != nil {
searchText = valCtx.BOOL().GetText()
text = valCtx.BOOL().GetText()
} else if valCtx.KEY() != nil {
searchText = valCtx.KEY().GetText()
text = valCtx.KEY().GetText()
} else {
v.errors = append(v.errors, fmt.Sprintf("unsupported value type: %s", valCtx.GetText()))
return ""
}
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
return cond
}
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
v.warnings = append(v.warnings, v.fullTextColumn.Warnings...)
return cond
}
return "" // Should not happen with valid input
@@ -377,11 +383,9 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
for _, key := range keys {
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, nil, v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error()))
return ""
}
conds = append(conds, condition)
v.warnings = append(v.warnings, key.Warnings...)
}
// if there is only one condition, return it directly, one less `()` wrapper
if len(conds) == 1 {
@@ -454,7 +458,6 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
return ""
}
conds = append(conds, condition)
v.warnings = append(v.warnings, key.Warnings...)
}
if len(conds) == 1 {
return conds[0]
@@ -503,7 +506,6 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
return ""
}
conds = append(conds, condition)
v.warnings = append(v.warnings, key.Warnings...)
}
if len(conds) == 1 {
return conds[0]
@@ -590,7 +592,6 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
return ""
}
conds = append(conds, condition)
v.warnings = append(v.warnings, key.Warnings...)
}
if len(conds) == 1 {
return conds[0]
@@ -647,6 +648,7 @@ func (v *filterExpressionVisitor) VisitValueList(ctx *grammar.ValueListContext)
// VisitFullText handles standalone quoted strings for full-text search
func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) any {
if v.skipFullTextFilter {
return ""
}
@@ -668,8 +670,6 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
v.warnings = append(v.warnings, v.fullTextColumn.Warnings...)
return cond
}

View File

@@ -35,19 +35,13 @@ func (c *conditionBuilder) conditionFor(
return "", err
}
if column.Type.GetType() == schema.ColumnTypeEnumJSON {
// If field data is Not JSON Column Itself, then we need to build a JSON condition
if querybuilder.BodyJSONQueryEnabled && key.MustBuildJSONCondition() {
valueType, value := InferDataType(value, operator, key)
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
if err != nil {
return "", err
}
return cond, nil
} else if key.FieldDataType == telemetrytypes.FieldDataTypeJSON && !operator.IsOpValidForJSON() {
// throw error for invalid operators on JSON columns
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid operator")
if column.IsJSONColumn() && querybuilder.BodyJSONQueryEnabled {
valueType, value := InferDataType(value, operator, key)
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
if err != nil {
return "", err
}
return cond, nil
}
if operator.IsStringSearchOperator() {
@@ -114,6 +108,7 @@ func (c *conditionBuilder) conditionFor(
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
case qbtypes.FilterOperatorNotContains:
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
case qbtypes.FilterOperatorRegexp:
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
@@ -181,16 +176,9 @@ func (c *conditionBuilder) conditionFor(
var value any
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
switch key.FieldDataType {
case telemetrytypes.FieldDataTypeJSON:
if operator == qbtypes.FilterOperatorExists {
return sb.EQ(fmt.Sprintf("empty(%s)", tblFieldName), false), nil
}
return sb.EQ(fmt.Sprintf("empty(%s)", tblFieldName), true), nil
default:
if operator == qbtypes.FilterOperatorExists {
return sb.IsNotNull(tblFieldName), nil
}
if operator == qbtypes.FilterOperatorExists {
return sb.IsNotNull(tblFieldName), nil
} else {
return sb.IsNull(tblFieldName), nil
}
case schema.ColumnTypeEnumLowCardinality:

View File

@@ -1,10 +1,7 @@
package telemetrylogs
import (
"fmt"
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
@@ -20,7 +17,7 @@ const (
LogsV2TimestampColumn = "timestamp"
LogsV2ObservedTimestampColumn = "observed_timestamp"
LogsV2BodyColumn = "body"
LogsV2BodyV2Column = constants.BodyV2Column
LogsV2BodyJSONColumn = constants.BodyJSONColumn
LogsV2BodyPromotedColumn = constants.BodyPromotedColumn
LogsV2TraceIDColumn = "trace_id"
LogsV2SpanIDColumn = "span_id"
@@ -37,25 +34,11 @@ const (
LogsV2ResourcesStringColumn = "resources_string"
LogsV2ScopeStringColumn = "scope_string"
BodyV2ColumnPrefix = constants.BodyV2ColumnPrefix
BodyJSONColumnPrefix = constants.BodyJSONColumnPrefix
BodyPromotedColumnPrefix = constants.BodyPromotedColumnPrefix
MessageSubColumn = "message"
ftsBodyFieldKey = "object(body)"
bodySearchDefaultWarning = "When you search on `body` (full text or by field), Query Builder uses body.message:string by default. Check that this matches what you want to search."
ftsBodyFieldWarning = "Avoid using `object(body)` in queries. It performs a full scan and can severely slow queries. Use explicit fields instead (for example, `body.response.status_code` or `response.status_code`). Only use `object(body)` for temporary schema exploration. We do not recommend its usage elsewhere."
)
var (
// mapping for body logical field to message sub column
// TODO(Piyush): Add description for detailed explanation of remapping of body to message
BodyLogicalFieldJSONMapping = &telemetrytypes.TelemetryFieldKey{
Name: MessageSubColumn,
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: telemetrytypes.FieldDataTypeString,
JSONDataType: &telemetrytypes.String,
Warnings: []string{bodySearchDefaultWarning},
}
DefaultFullTextColumn = &telemetrytypes.TelemetryFieldKey{
Name: "body",
Signal: telemetrytypes.SignalLogs,
@@ -135,44 +118,3 @@ var (
},
}
)
func bodyAliasExpression() string {
if !querybuilder.BodyJSONQueryEnabled {
return LogsV2BodyColumn
}
return fmt.Sprintf("%s as body", LogsV2BodyV2Column)
}
func enrichIntrinsicFields() error {
// body logical field is mapped to message field in the body context that too only with String data type
err := BodyLogicalFieldJSONMapping.SetJSONAccessPlan(telemetrytypes.JSONColumnMetadata{
BaseColumn: LogsV2BodyV2Column,
PromotedColumn: LogsV2BodyPromotedColumn,
}, map[string][]telemetrytypes.JSONDataType{
MessageSubColumn: {telemetrytypes.String},
})
if err != nil {
return err
}
if querybuilder.BodyJSONQueryEnabled {
DefaultFullTextColumn = BodyLogicalFieldJSONMapping
IntrinsicFields["body"] = *BodyLogicalFieldJSONMapping
IntrinsicFields[ftsBodyFieldKey] = telemetrytypes.TelemetryFieldKey{
Name: "body_v2",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeJSON,
Warnings: []string{ftsBodyFieldWarning},
}
}
return nil
}
func init() {
err := enrichIntrinsicFields()
if err != nil {
panic(err)
}
}

View File

@@ -30,7 +30,7 @@ var (
"severity_text": {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}},
"severity_number": {Name: "severity_number", Type: schema.ColumnTypeUInt8},
"body": {Name: "body", Type: schema.ColumnTypeString},
LogsV2BodyV2Column: {Name: LogsV2BodyV2Column, Type: schema.JSONColumnType{
LogsV2BodyJSONColumn: {Name: LogsV2BodyJSONColumn, Type: schema.JSONColumnType{
MaxDynamicTypes: utils.ToPointer(uint(32)),
MaxDynamicPaths: utils.ToPointer(uint(0)),
}},
@@ -89,9 +89,9 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry
}
case telemetrytypes.FieldContextBody:
// Body context is for JSON body fields
// Use body_v2 if feature flag is enabled
// Use body_json if feature flag is enabled
if querybuilder.BodyJSONQueryEnabled {
return logsV2Columns[LogsV2BodyV2Column], nil
return logsV2Columns[LogsV2BodyJSONColumn], nil
}
// Fall back to legacy body column
return logsV2Columns["body"], nil
@@ -100,9 +100,9 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry
if !ok {
// check if the key has body JSON search
if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
// Use body_v2 if feature flag is enabled and we have a body condition builder
// Use body_json if feature flag is enabled and we have a body condition builder
if querybuilder.BodyJSONQueryEnabled {
return logsV2Columns[LogsV2BodyV2Column], nil
return logsV2Columns[LogsV2BodyJSONColumn], nil
}
// Fall back to legacy body column
return logsV2Columns["body"], nil
@@ -147,9 +147,6 @@ func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.Telemetr
}
return m.buildFieldForJSON(key)
case telemetrytypes.FieldContextLog:
// return the column name as is for log context fields
return column.Name, nil
default:
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String)
}

View File

@@ -30,7 +30,7 @@ func NewJSONConditionBuilder(key *telemetrytypes.TelemetryFieldKey, valueType te
return &jsonConditionBuilder{key: key, valueType: telemetrytypes.MappingFieldDataTypeToJSONDataType[valueType]}
}
// BuildCondition builds the full WHERE condition for body_v2 JSON paths
// BuildCondition builds the full WHERE condition for body_json JSON paths
func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
conditions := []string{}
for _, node := range c.key.JSONPlan {
@@ -40,7 +40,6 @@ func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperato
}
conditions = append(conditions, condition)
}
return sb.Or(conditions...), nil
}
@@ -289,9 +288,9 @@ func (c *jsonConditionBuilder) applyOperator(sb *sqlbuilder.SelectBuilder, field
}
return sb.NotIn(fieldExpr, values...), nil
case qbtypes.FilterOperatorExists:
return sb.IsNotNull(fieldExpr), nil
return fmt.Sprintf("%s IS NOT NULL", fieldExpr), nil
case qbtypes.FilterOperatorNotExists:
return sb.IsNull(fieldExpr), nil
return fmt.Sprintf("%s IS NULL", fieldExpr), nil
// between and not between
case qbtypes.FilterOperatorBetween, qbtypes.FilterOperatorNotBetween:
values, ok := value.([]any)

File diff suppressed because one or more lines are too long

View File

@@ -203,6 +203,7 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri
}
func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
// First check if it matches with any intrinsic fields
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
if _, ok := IntrinsicFields[key.Name]; ok {
@@ -211,6 +212,7 @@ func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldK
}
return querybuilder.AdjustKey(key, keys, nil)
}
// buildListQuery builds a query for list panel type
@@ -247,7 +249,11 @@ func (b *logQueryStatementBuilder) buildListQuery(
sb.SelectMore(LogsV2SeverityNumberColumn)
sb.SelectMore(LogsV2ScopeNameColumn)
sb.SelectMore(LogsV2ScopeVersionColumn)
sb.SelectMore(bodyAliasExpression())
sb.SelectMore(LogsV2BodyColumn)
if querybuilder.BodyJSONQueryEnabled {
sb.SelectMore(LogsV2BodyJSONColumn)
sb.SelectMore(LogsV2BodyPromotedColumn)
}
sb.SelectMore(LogsV2AttributesStringColumn)
sb.SelectMore(LogsV2AttributesNumberColumn)
sb.SelectMore(LogsV2AttributesBoolColumn)

View File

@@ -5,7 +5,6 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter"
@@ -887,154 +886,3 @@ func TestAdjustKey(t *testing.T) {
})
}
}
func TestStmtBuilderBodyField(t *testing.T) {
cases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
enableBodyJSONQuery bool
expected qbtypes.Statement
expectedErr error
}{
{
name: "body_exists",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body Exists"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`message`, 'String') IS NOT NULL) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,
},
{
name: "body_exists_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body Exists"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
{
name: "body_empty",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body == ''"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (dynamicElement(body_v2.`message`, 'String') = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,
},
{
name: "body_empty_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body == ''"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
{
name: "body_contains",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(dynamicElement(body_v2.`message`, 'String')) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,
},
{
name: "body_contains_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
enable, disable := jsonQueryTestUtil(t)
defer disable()
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if c.enableBodyJSONQuery {
enable()
} else {
disable()
}
// build the key map after enabling/disabling body JSON query
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
resourceFilterStmtBuilder := resourceFilterStmtBuilder()
statementBuilder := NewLogQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
resourceFilterStmtBuilder,
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
)
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
if c.expectedErr != nil {
require.Error(t, err)
require.Contains(t, err.Error(), c.expectedErr.Error())
} else {
if err != nil {
_, _, _, _, _, add := errors.Unwrapb(err)
t.Logf("error additionals: %v", add)
}
require.NoError(t, err)
require.Equal(t, c.expected.Query, q.Query)
require.Equal(t, c.expected.Args, q.Args)
require.Equal(t, c.expected.Warnings, q.Warnings)
}
})
}
}

View File

@@ -27,6 +27,13 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"body": {
{
Name: "body",
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"http.status_code": {
{
Name: "http.status_code",
@@ -938,11 +945,6 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
key.Signal = telemetrytypes.SignalLogs
}
}
// add intrinsic fields to the map
for fieldName, key := range IntrinsicFields {
keysMap[fieldName] = append(keysMap[fieldName], &key)
}
return keysMap
}

View File

@@ -254,7 +254,7 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
sb.Where(sb.Equal("database", telemetrylogs.DBName))
sb.Where(sb.Equal("table", telemetrylogs.LogsV2LocalTableName))
sb.Where(sb.Or(
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix))),
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix))),
sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix))),
))
@@ -338,7 +338,7 @@ func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, li
if promoted {
path = telemetrylogs.BodyPromotedColumnPrefix + path
} else {
path = telemetrylogs.BodyV2ColumnPrefix + path
path = telemetrylogs.BodyJSONColumnPrefix + path
}
from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
@@ -529,7 +529,7 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
// TODO(Piyush): Remove this function
func CleanPathPrefixes(path string) string {
path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix)
return path
}

View File

@@ -117,7 +117,7 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
expectedArgs: []any{
telemetrylogs.DBName,
telemetrylogs.LogsV2LocalTableName,
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
},
},
@@ -130,7 +130,7 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
expectedArgs: []any{
telemetrylogs.DBName,
telemetrylogs.LogsV2LocalTableName,
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyV2ColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix)),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("foo")),
fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains("bar")),

View File

@@ -100,7 +100,7 @@ func NewTelemetryMetaStore(
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
telemetrytypes.SignalLogs: {
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
BaseColumn: telemetrylogs.LogsV2BodyV2Column,
BaseColumn: telemetrylogs.LogsV2BodyJSONColumn,
PromotedColumn: telemetrylogs.LogsV2BodyPromotedColumn,
},
},
@@ -333,7 +333,7 @@ func (t *telemetryMetaStore) logsTblStatementToFieldKeys(ctx context.Context) ([
}
// getLogsKeys returns the keys from the spans that match the field selection criteria
func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) (telemetrytypes.TelemetryFieldKeys, bool, error) {
func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
if len(fieldKeySelectors) == 0 {
return nil, true, nil
}
@@ -343,10 +343,9 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
if err != nil {
return nil, false, err
}
// setOfKeys to reuse the same key object for qualified names
setOfKeys := make(map[string]*telemetrytypes.TelemetryFieldKey)
mapOfKeys := make(map[string]*telemetrytypes.TelemetryFieldKey)
for _, key := range matKeys {
setOfKeys[key.QualifiedName()] = key
mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
}
// queries for both attribute and resource keys tables
@@ -447,7 +446,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
if len(queries) == 0 {
// No matching contexts, return empty result
return nil, true, nil
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
}
// Combine queries with UNION ALL
@@ -475,7 +474,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
}
defer rows.Close()
mapOfKeys := make(telemetrytypes.TelemetryFieldKeys)
keys := []*telemetrytypes.TelemetryFieldKey{}
rowCount := 0
searchTexts := []string{}
dataTypes := []telemetrytypes.FieldDataType{}
@@ -503,7 +502,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
key, ok := setOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
// if there is no materialised column, create a key with the field context and data type
if !ok {
@@ -515,8 +514,8 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
}
}
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
setOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
if rows.Err() != nil {
@@ -542,13 +541,17 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
if found {
if field, exists := telemetrylogs.IntrinsicFields[key]; exists {
if _, added := setOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
mapOfKeys[field.Name] = append(mapOfKeys[field.Name], &field)
// intrinsic field can be a logical field mapped to a different physical location
mapOfKeys[key] = append(mapOfKeys[key], &field)
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: key,
FieldContext: telemetrytypes.FieldContextLog,
Signal: telemetrytypes.SignalLogs,
})
}
}
@@ -557,13 +560,10 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
if err != nil {
t.logger.ErrorContext(ctx, "failed to extract body JSON paths", "error", err)
}
for _, key := range bodyJSONPaths {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
keys = append(keys, bodyJSONPaths...)
complete = complete && finished
}
return mapOfKeys, complete, nil
return keys, complete, nil
}
func getPriorityForContext(ctx telemetrytypes.FieldContext) int {
@@ -847,20 +847,12 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele
if fieldKeySelector != nil {
selectors = []*telemetrytypes.FieldKeySelector{fieldKeySelector}
}
mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
switch fieldKeySelector.Signal {
case telemetrytypes.SignalTraces:
keys, complete, err = t.getTracesKeys(ctx, selectors)
case telemetrytypes.SignalLogs:
mapOfLogKeys, logsComplete, err := t.getLogsKeys(ctx, selectors)
if err != nil {
return nil, false, err
}
for keyName, keys := range mapOfLogKeys {
mapOfKeys[keyName] = append(mapOfKeys[keyName], keys...)
}
complete = complete && logsComplete
keys, complete, err = t.getLogsKeys(ctx, selectors)
case telemetrytypes.SignalMetrics:
if fieldKeySelector.Source == telemetrytypes.SourceMeter {
keys, complete, err = t.getMeterSourceMetricKeys(ctx, selectors)
@@ -876,14 +868,11 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele
keys = append(keys, tracesKeys...)
// get logs keys
mapOfLogKeys, logsComplete, err := t.getLogsKeys(ctx, selectors)
logsKeys, logsComplete, err := t.getLogsKeys(ctx, selectors)
if err != nil {
return nil, false, err
}
for keyName, keys := range mapOfLogKeys {
mapOfKeys[keyName] = append(mapOfKeys[keyName], keys...)
}
complete = complete && logsComplete
keys = append(keys, logsKeys...)
// get metrics keys
metricsKeys, metricsComplete, err := t.getMetricsKeys(ctx, selectors)
@@ -898,6 +887,7 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele
return nil, false, err
}
mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
for _, key := range keys {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
@@ -934,7 +924,7 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
}
}
mapOfLogKeys, logsComplete, err := t.getLogsKeys(ctx, logsSelectors)
logsKeys, logsComplete, err := t.getLogsKeys(ctx, logsSelectors)
if err != nil {
return nil, false, err
}
@@ -955,8 +945,8 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
complete := logsComplete && tracesComplete && metricsComplete
mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
for keyName, keys := range mapOfLogKeys {
mapOfKeys[keyName] = append(mapOfKeys[keyName], keys...)
for _, key := range logsKeys {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
for _, key := range tracesKeys {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)

View File

@@ -7,6 +7,6 @@ const (
AttributesMetadataTableName = "distributed_attributes_metadata"
AttributesMetadataLocalTableName = "attributes_metadata"
PathTypesTableName = otelcollectorconst.DistributedPathTypesTable
PromotedPathsTableName = "distributed_json_promoted_paths"
PromotedPathsTableName = otelcollectorconst.DistributedPromotedPathsTable
SkipIndexTableName = "system.data_skipping_indices"
)

View File

@@ -36,8 +36,8 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "array paths can not be promoted or indexed")
}
if strings.HasPrefix(i.Path, constants.BodyV2ColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyV2ColumnPrefix, constants.BodyPromotedColumnPrefix)
if strings.HasPrefix(i.Path, constants.BodyJSONColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyJSONColumnPrefix, constants.BodyPromotedColumnPrefix)
}
if !strings.HasPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix) {

View File

@@ -197,17 +197,6 @@ func (f FilterOperator) IsArrayOperator() bool {
}
}
func (f FilterOperator) IsOpValidForJSON() bool {
switch f {
case FilterOperatorExists, FilterOperatorNotExists,
FilterOperatorContains, FilterOperatorNotContains,
FilterOperatorRegexp, FilterOperatorNotRegexp:
return true
default:
return false
}
}
type OrderDirection struct {
valuer.String
}

View File

@@ -29,8 +29,6 @@ const (
ArrayAnyIndexSuffix = "[*]"
)
type TelemetryFieldKeys map[string][]*TelemetryFieldKey
type TelemetryFieldKey struct {
Name string `json:"name" required:"true"`
Description string `json:"description,omitempty"`
@@ -43,7 +41,6 @@ type TelemetryFieldKey struct {
JSONPlan JSONAccessPlan `json:"-"`
Indexes []JSONDataTypeIndex `json:"-"`
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
Warnings []string `json:"-"` // warnings attached to using a particular field key
}
func (f *TelemetryFieldKey) KeyNameContainsArray() bool {
@@ -65,10 +62,6 @@ func (f *TelemetryFieldKey) ArrayParentPaths() []string {
return paths
}
func (f *TelemetryFieldKey) MustBuildJSONCondition() bool {
return f.FieldDataType != FieldDataTypeJSON && f.JSONDataType != nil
}
func (f *TelemetryFieldKey) ArrayParentSelectors() []*FieldKeySelector {
paths := f.ArrayParentPaths()
selectors := make([]*FieldKeySelector, 0, len(paths))
@@ -176,10 +169,6 @@ func (f *TelemetryFieldKey) Normalize() {
}
func (f *TelemetryFieldKey) QualifiedName() string {
return f.Name + ";" + f.FieldContext.StringValue() + ";" + f.FieldDataType.StringValue()
}
// GetFieldKeyFromKeyText returns a TelemetryFieldKey from a key text.
// The key text is expected to be in the format of `fieldContext.fieldName:fieldDataType` in the search query.
// Both fieldContext and :fieldDataType are optional.

View File

@@ -21,7 +21,6 @@ var (
// int64 and number are synonyms for float64
FieldDataTypeInt64 = FieldDataType{valuer.NewString("int64")}
FieldDataTypeNumber = FieldDataType{valuer.NewString("number")}
FieldDataTypeJSON = FieldDataType{valuer.NewString("json")}
FieldDataTypeUnspecified = FieldDataType{valuer.NewString("")}
FieldDataTypeArrayString = FieldDataType{valuer.NewString("[]string")}

View File

@@ -40,7 +40,7 @@ type JSONAccessNode struct {
// Node information
Name string
IsTerminal bool
isRoot bool // marked true for only body_v2 and body_promoted
isRoot bool // marked true for only body_json and body_json_promoted
// Precomputed type information (single source of truth)
AvailableTypes []JSONDataType

View File

@@ -1,19 +1,12 @@
package telemetrytypes
import (
"fmt"
"testing"
otelconstants "github.com/SigNoz/signoz-otel-collector/constants"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
const (
bodyV2Column = otelconstants.BodyV2Column
bodyPromotedColumn = otelconstants.BodyPromotedColumn
)
// ============================================================================
// Helper Functions for Test Data Creation
// ============================================================================
@@ -116,8 +109,8 @@ func TestNode_Alias(t *testing.T) {
}{
{
name: "Root node returns name as-is",
node: NewRootJSONAccessNode(bodyV2Column, 32, 0),
expected: bodyV2Column,
node: NewRootJSONAccessNode("body_json", 32, 0),
expected: "body_json",
},
{
name: "Node without parent returns backticked name",
@@ -131,9 +124,9 @@ func TestNode_Alias(t *testing.T) {
name: "Node with root parent uses dot separator",
node: &JSONAccessNode{
Name: "age",
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
expected: "`" + bodyV2Column + ".age`",
expected: "`" + "body_json" + ".age`",
},
{
name: "Node with non-root parent uses array separator",
@@ -141,10 +134,10 @@ func TestNode_Alias(t *testing.T) {
Name: "name",
Parent: &JSONAccessNode{
Name: "education",
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
},
expected: "`" + bodyV2Column + ".education[].name`",
expected: "`" + "body_json" + ".education[].name`",
},
{
name: "Nested array path with multiple levels",
@@ -154,11 +147,11 @@ func TestNode_Alias(t *testing.T) {
Name: "awards",
Parent: &JSONAccessNode{
Name: "education",
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
},
},
expected: "`" + bodyV2Column + ".education[].awards[].type`",
expected: "`" + "body_json" + ".education[].awards[].type`",
},
}
@@ -180,18 +173,18 @@ func TestNode_FieldPath(t *testing.T) {
name: "Simple field path from root",
node: &JSONAccessNode{
Name: "user",
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
// FieldPath() always wraps the field name in backticks
expected: bodyV2Column + ".`user`",
expected: "body_json" + ".`user`",
},
{
name: "Field path with backtick-required key",
node: &JSONAccessNode{
Name: "user-name", // requires backtick
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
expected: bodyV2Column + ".`user-name`",
expected: "body_json" + ".`user-name`",
},
{
name: "Nested field path",
@@ -199,11 +192,11 @@ func TestNode_FieldPath(t *testing.T) {
Name: "age",
Parent: &JSONAccessNode{
Name: "user",
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
},
// FieldPath() always wraps the field name in backticks
expected: "`" + bodyV2Column + ".user`.`age`",
expected: "`" + "body_json" + ".user`.`age`",
},
{
name: "Array element field path",
@@ -211,11 +204,11 @@ func TestNode_FieldPath(t *testing.T) {
Name: "name",
Parent: &JSONAccessNode{
Name: "education",
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
Parent: NewRootJSONAccessNode("body_json", 32, 0),
},
},
// FieldPath() always wraps the field name in backticks
expected: "`" + bodyV2Column + ".education`.`name`",
expected: "`" + "body_json" + ".education`.`name`",
},
}
@@ -243,36 +236,36 @@ func TestPlanJSON_BasicStructure(t *testing.T) {
{
name: "Simple path not promoted",
key: makeKey("user.name", String, false),
expectedYAML: fmt.Sprintf(`
expectedYAML: `
- name: user.name
column: %s
column: body_json
availableTypes:
- String
maxDynamicTypes: 16
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
{
name: "Simple path promoted",
key: makeKey("user.name", String, true),
expectedYAML: fmt.Sprintf(`
expectedYAML: `
- name: user.name
column: %s
column: body_json
availableTypes:
- String
maxDynamicTypes: 16
isTerminal: true
elemType: String
- name: user.name
column: %s
column: body_json_promoted
availableTypes:
- String
maxDynamicTypes: 16
maxDynamicPaths: 256
isTerminal: true
elemType: String
`, bodyV2Column, bodyPromotedColumn),
`,
},
{
name: "Empty path returns error",
@@ -285,8 +278,8 @@ func TestPlanJSON_BasicStructure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
}, types)
if tt.expectErr {
require.Error(t, err)
@@ -311,9 +304,9 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
{
name: "Single array level - JSON branch only",
path: "education[].name",
expectedYAML: fmt.Sprintf(`
expectedYAML: `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -325,14 +318,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
maxDynamicTypes: 8
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
{
name: "Single array level - both JSON and Dynamic branches",
path: "education[].awards[].type",
expectedYAML: fmt.Sprintf(`
expectedYAML: `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -359,14 +352,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
maxDynamicPaths: 256
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
{
name: "Deeply nested array path",
path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name",
expectedYAML: fmt.Sprintf(`
expectedYAML: `
- name: interests
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -406,14 +399,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
- String
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
{
name: "ArrayAnyIndex replacement [*] to []",
path: "education[*].name",
expectedYAML: fmt.Sprintf(`
expectedYAML: `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -425,7 +418,7 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
maxDynamicTypes: 8
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
}
@@ -433,8 +426,8 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
key := makeKey(tt.path, String, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
}, types)
require.NoError(t, err)
require.NotNil(t, key.JSONPlan)
@@ -452,15 +445,15 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
t.Run("Non-promoted plan", func(t *testing.T) {
key := makeKey(path, String, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
}, types)
require.NoError(t, err)
require.Len(t, key.JSONPlan, 1)
expectedYAML := fmt.Sprintf(`
expectedYAML := `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -487,7 +480,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
maxDynamicPaths: 256
isTerminal: true
elemType: String
`, bodyV2Column)
`
got := plansToYAML(t, key.JSONPlan)
require.YAMLEq(t, expectedYAML, got)
})
@@ -495,15 +488,15 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
t.Run("Promoted plan", func(t *testing.T) {
key := makeKey(path, String, true)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
}, types)
require.NoError(t, err)
require.Len(t, key.JSONPlan, 2)
expectedYAML := fmt.Sprintf(`
expectedYAML := `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -531,7 +524,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
isTerminal: true
elemType: String
- name: education
column: %s
column: body_json_promoted
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -561,7 +554,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
maxDynamicPaths: 256
isTerminal: true
elemType: String
`, bodyV2Column, bodyPromotedColumn)
`
got := plansToYAML(t, key.JSONPlan)
require.YAMLEq(t, expectedYAML, got)
})
@@ -582,11 +575,11 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
expectErr: true,
},
{
name: "Very deep nesting - validates progression doesn't go negative",
path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name",
expectedYAML: fmt.Sprintf(`
name: "Very deep nesting - validates progression doesn't go negative",
path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name",
expectedYAML: `
- name: interests
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -626,14 +619,14 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
- String
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
{
name: "Path with mixed scalar and array types",
path: "education[].type",
expectedYAML: fmt.Sprintf(`
name: "Path with mixed scalar and array types",
path: "education[].type",
expectedYAML: `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -646,20 +639,20 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
maxDynamicTypes: 8
isTerminal: true
elemType: String
`, bodyV2Column),
`,
},
{
name: "Exists with only array types available",
path: "education",
expectedYAML: fmt.Sprintf(`
name: "Exists with only array types available",
path: "education",
expectedYAML: `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
isTerminal: true
elemType: Array(JSON)
`, bodyV2Column),
`,
},
}
@@ -675,8 +668,8 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
}
key := makeKey(tt.path, keyType, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
}, types)
if tt.expectErr {
require.Error(t, err)
@@ -694,15 +687,15 @@ func TestPlanJSON_TreeStructure(t *testing.T) {
path := "education[].awards[].participated[].team[].branch"
key := makeKey(path, String, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
}, types)
require.NoError(t, err)
require.Len(t, key.JSONPlan, 1)
expectedYAML := fmt.Sprintf(`
expectedYAML := `
- name: education
column: %s
column: body_json
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -787,7 +780,7 @@ func TestPlanJSON_TreeStructure(t *testing.T) {
maxDynamicPaths: 64
isTerminal: true
elemType: String
`, bodyV2Column)
`
got := plansToYAML(t, key.JSONPlan)
require.YAMLEq(t, expectedYAML, got)

View File

@@ -2,7 +2,6 @@ package telemetrytypestest
import (
"context"
"slices"
"strings"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
@@ -177,9 +176,8 @@ func matchesKey(selector *telemetrytypes.FieldKeySelector, key *telemetrytypes.T
return true
}
matchNameExceptions := []string{"body"}
// Check name (already checked in matchesName, but double-check here)
if selector.Name != "" && !matchesName(selector, key.Name) && slices.Contains(matchNameExceptions, key.Name) {
if selector.Name != "" && !matchesName(selector, key.Name) {
return false
}