Compare commits

..

7 Commits

Author SHA1 Message Date
Naman Verma
8101a79371 Merge branch 'main' into nv/exec-stats 2026-11-16 10:43:03 +05:30
Naman Verma
62d477e920 Merge branch 'main' into nv/exec-stats 2026-11-09 12:20:27 +05:30
Naman Verma
fe45b9dcae Merge branch 'main' into nv/exec-stats 2026-11-06 00:51:46 +05:30
Naman Verma
63ee60a40c fix(querier): stop cache refreshes from inflating exec stats 2026-11-06 00:51:03 +05:30
Naman Verma
29a393e47e Merge branch 'main' into nv/exec-stats 2026-11-05 10:51:11 +05:30
Naman Verma
c58f76f998 Merge branch 'main' into nv/exec-stats 2026-11-03 11:34:10 +05:30
Naman Verma
e39ad9e221 test: check how exec stats fail 2026-11-02 22:24:22 +05:30
45 changed files with 567 additions and 1243 deletions

View File

@@ -2,50 +2,44 @@ name: e2eci
on:
pull_request:
types:
- opened
- synchronize
- reopened
- labeled
merge_group:
types:
- checks_requested
# Labels other than safe-to-e2e get a group of their own, so they never cancel a running e2e.
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'safe-to-e2e' && format('-{0}', github.run_id) || '' }}
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
changes:
if: github.event.action != 'labeled' || github.event.label.name == 'safe-to-e2e'
runs-on: ubuntu-latest
outputs:
e2e: ${{ steps.changes.outputs.e2e }}
steps:
- name: merge-queue-pass
if: github.event_name == 'merge_group'
run: echo "::notice::Merge queue run. E2E already gated the PR, so the required e2e check auto-passes."
- name: changes
id: changes
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
FORCE: ${{ github.event_name == 'merge_group' || contains(github.event.pull_request.labels.*.name, 'safe-to-e2e') }}
FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'safe-to-e2e') }}
run: |
if [ "$FORCE" = "true" ]; then
echo "e2e=true" >> "$GITHUB_OUTPUT"
exit 0
fi
CHANGED_FILES=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename | select((startswith("tests/integration/") or startswith("frontend/src/api/generated/")) | not)')
PATTERN='^(frontend/|tests/|\.github/workflows/e2eci\.yaml$|cmd/enterprise/Dockerfile\.with-web\.integration$)'
if printf '%s\n' "$CHANGED_FILES" | grep -qE "$PATTERN"; then
if [ "$FORCE" = "true" ]; then
echo "e2e=true" >> "$GITHUB_OUTPUT"
elif printf '%s\n' "$CHANGED_FILES" | grep -qE "$PATTERN"; then
echo "e2e=true" >> "$GITHUB_OUTPUT"
else
echo "e2e=false" >> "$GITHUB_OUTPUT"
fi
fmtlint:
needs: changes
if: github.event_name != 'merge_group' && needs.changes.outputs.e2e == 'true'
if: needs.changes.outputs.e2e == 'true'
runs-on: ubuntu-latest
steps:
- name: checkout
@@ -186,8 +180,6 @@ jobs:
retention-days: 5
# Required status check. Fails when e2e was needed but did not pass.
e2e:
# Runs for other labels must not report as e2e, or their pass would hide the real result.
name: ${{ github.event.action == 'labeled' && github.event.label.name != 'safe-to-e2e' && 'ignored-label' || 'e2e' }}
if: always()
needs: [changes, fmtlint, test]
runs-on: ubuntu-latest

View File

@@ -45,7 +45,6 @@ jobs:
- callbackauthn
- cloudintegrations
- dashboard
- fileauditor
- ingestionkeys
- inframonitoring
- llmpricingrules

View File

@@ -13,7 +13,6 @@ import { useAppContext } from 'providers/App/App';
import { LicensePlatform, LicenseState } from 'types/api/licensesV3/getActive';
import { OrgPreference } from 'types/api/preferences/preference';
import { USER_ROLES } from 'types/roles';
import { isSignozAdminEmail } from 'utils/app';
import {
routePermission,
routeWithInitialAuthZSupport,
@@ -76,7 +75,7 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
const users = usersData?.data || [];
const remainingUsers = (Array.isArray(users) ? users : []).filter(
(u) => !isSignozAdminEmail(u.email),
(user) => user.email !== 'admin@signoz.cloud',
);
return remainingUsers.length === 1;
@@ -204,7 +203,6 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
const isFirstUser = checkFirstTimeUser();
if (
isFirstUser &&
!isSignozAdminEmail(user.email) &&
!isOnboardingComplete &&
!ROUTES_NOT_TO_BE_OVERRIDEN.includes(pathname) &&
pathname !== ROUTES.ONBOARDING

View File

@@ -2,7 +2,6 @@ import { ReactElement } from 'react';
import { QueryClient, QueryClientProvider } from 'react-query';
import { MemoryRouter, Route, Switch, useLocation } from 'react-router-dom';
import { act, render, screen, waitFor } from '@testing-library/react';
import { SIGNOZ_ADMIN_EMAIL } from 'constants/app';
import { LOCALSTORAGE } from 'constants/localStorage';
import { ORG_PREFERENCES } from 'constants/orgPreferences';
import ROUTES from 'constants/routes';
@@ -1090,33 +1089,6 @@ describe('PrivateRoute', () => {
await assertRedirectsTo(ROUTES.ONBOARDING);
});
it('should not redirect the signoz admin to onboarding while the first user has not completed it', async () => {
mockUsersData = [
{ email: 'test@example.com' },
{ email: SIGNOZ_ADMIN_EMAIL },
];
renderPrivateRoute({
initialRoute: ROUTES.HOME,
appContext: {
isLoggedIn: true,
isFetchingOrgPreferences: false,
orgPreferences: createMockOrgPreferences(false),
user: createMockUser({
role: USER_ROLES.ADMIN as ROLES,
email: SIGNOZ_ADMIN_EMAIL,
}),
},
isCloudUser: true,
});
await act(async () => {
await Promise.resolve();
});
assertStaysOnRoute(ROUTES.HOME);
});
it('should not redirect to onboarding when org preferences are still fetching', () => {
renderPrivateRoute({
initialRoute: ROUTES.HOME,

View File

@@ -1,6 +1,5 @@
import { useCallback, useMemo, useState } from 'react';
import { Row, Select, Spin } from 'antd';
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import { navigate } from 'lib/router/navigation';
import { useAppLocation } from 'lib/router/useAppLocation';
import {
@@ -18,7 +17,6 @@ export interface SelectOptionConfig {
placeholder: string;
queryParam: QueryParams;
filterType: string | string[];
fieldContext?: TelemetrytypesFieldContextDTO;
shouldSetQueryParams?: boolean;
onChange?: (value: string | string[]) => void;
values?: string | string[];
@@ -29,16 +27,13 @@ export function FilterSelect({
placeholder,
queryParam,
filterType,
fieldContext,
values,
shouldSetQueryParams,
onChange,
isMultiple,
}: SelectOptionConfig): JSX.Element {
const { handleSearch, isFetching, options } = useCeleryFilterOptions(
filterType,
fieldContext,
);
const { handleSearch, isFetching, options } =
useCeleryFilterOptions(filterType);
const urlQuery = useUrlQuery();
const location = useAppLocation();
@@ -123,7 +118,6 @@ export function FilterSelect({
}
FilterSelect.defaultProps = {
fieldContext: undefined,
shouldSetQueryParams: true,
onChange: (): void => {},
values: [],
@@ -135,14 +129,12 @@ function CeleryOverviewConfigOptions(): JSX.Element {
{
placeholder: 'Service Name',
queryParam: QueryParams.service,
filterType: 'service.name',
fieldContext: TelemetrytypesFieldContextDTO.resource,
filterType: 'serviceName',
},
{
placeholder: 'Span Name',
queryParam: QueryParams.spanName,
filterType: 'name',
fieldContext: TelemetrytypesFieldContextDTO.span,
},
{
placeholder: 'Msg System',
@@ -170,7 +162,6 @@ function CeleryOverviewConfigOptions(): JSX.Element {
placeholder={config.placeholder}
queryParam={config.queryParam}
filterType={config.filterType}
fieldContext={config.fieldContext}
/>
))}
</Row>

View File

@@ -1,37 +0,0 @@
import type { GetFieldsValues200 } from 'api/generated/services/sigNoz.schemas';
import { rest, server } from 'mocks-server/server';
import { render, waitFor } from 'tests/test-utils';
import CeleryOverviewConfigOptions from '../CeleryOverviewConfigOptions';
const emptyValues: GetFieldsValues200 = {
status: 'success',
data: { complete: true, values: { stringValues: [] } },
};
describe('CeleryOverviewConfigOptions', () => {
it('scopes the service and span name lookups to their field context', async () => {
const contextByKey = new Map<string, string | null>();
server.use(
rest.get('*/api/v1/fields/values', (req, res, ctx) => {
contextByKey.set(
req.url.searchParams.get('name') ?? '',
req.url.searchParams.get('fieldContext'),
);
return res(ctx.status(200), ctx.json(emptyValues));
}),
);
render(<CeleryOverviewConfigOptions />);
await waitFor(() => expect(contextByKey.size).toBe(6));
expect(Object.fromEntries(contextByKey)).toStrictEqual({
'service.name': 'resource',
name: 'span',
'messaging.system': null,
'messaging.destination.name': null,
'messaging.destination': null,
kind_string: null,
});
});
});

View File

@@ -1,21 +1,23 @@
import { useQuery } from 'react-query';
// eslint-disable-next-line no-restricted-imports
import { useSelector } from 'react-redux';
import type { DefaultOptionType } from 'antd/es/select';
import type { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import { getFieldValueSuggestions } from 'api/querySuggestions/getFieldValueSuggestions';
import { FIELD_API_CACHE_TIME } from 'constants/queryCacheTime';
import { getAttributesValues } from 'api/queryBuilder/getAttributesValues';
import { DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY } from 'constants/queryBuilder';
import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
import { toFieldValues } from 'hooks/querySuggestions/useFieldValuesSuggestion';
import { AppState } from 'store/reducers';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { DATA_SOURCE_TO_SIGNAL, DataSource } from 'types/common/queryBuilder';
import { getValuesForDataType } from 'utils/fieldValues';
import { DataSource } from 'types/common/queryBuilder';
import { GlobalReducer } from 'types/reducer/globalTime';
export interface Filters {
searchText: string;
attributeKey: string | string[];
aggregateOperator?: string;
dataSource?: DataSource;
metricName?: string;
aggregateAttribute?: string;
filterAttributeKeyDataType?: DataTypes;
fieldContext?: TelemetrytypesFieldContextDTO;
tagType?: string;
}
export interface GetAllFiltersResponse {
@@ -27,41 +29,54 @@ export function useGetAllFilters(props: Filters): GetAllFiltersResponse {
const {
searchText,
attributeKey,
aggregateOperator,
dataSource,
metricName,
aggregateAttribute,
filterAttributeKeyDataType,
fieldContext,
tagType,
} = props;
const { minTime, maxTime } = useSelector<AppState, GlobalReducer>(
(state) => state.globalTime,
);
const { data, isLoading } = useQuery(
[
REACT_QUERY_KEY.GET_ATTRIBUTE_VALUES,
dataSource,
metricName,
attributeKey,
fieldContext,
searchText,
minTime,
maxTime,
],
async () => {
const keys = Array.isArray(attributeKey) ? attributeKey : [attributeKey];
const responses = await Promise.all(
keys.map((key) =>
getFieldValueSuggestions({
signal: DATA_SOURCE_TO_SIGNAL[dataSource || DataSource.TRACES],
name: key,
fieldContext,
getAttributesValues({
aggregateOperator: aggregateOperator || 'noop',
dataSource: dataSource || DataSource.TRACES,
aggregateAttribute: aggregateAttribute || '',
attributeKey: key,
searchText: searchText ?? '',
metricName,
filterAttributeKeyDataType:
filterAttributeKeyDataType || DataTypes.String,
tagType: tagType || 'tag',
}),
),
);
const uniqueValues = [
...new Set(
responses.flatMap((response) =>
getValuesForDataType(toFieldValues(response), filterAttributeKeyDataType),
),
responses.flatMap(({ payload }) => {
if (!payload) {
return [];
}
const dataType = filterAttributeKeyDataType || DataTypes.String;
const key = DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY[dataType];
return key ? payload[key] || [] : [];
}),
),
];
@@ -70,7 +85,6 @@ export function useGetAllFilters(props: Filters): GetAllFiltersResponse {
value: val,
}));
},
{ staleTime: FIELD_API_CACHE_TIME },
);
return { options: data ?? [], isFetching: isLoading };

View File

@@ -8,9 +8,12 @@ import { CardContainer } from 'container/WidgetCard/styles';
import { useIsDarkMode } from 'hooks/useDarkMode';
import { ChevronDown, ChevronUp } from '@signozhq/icons';
import { AppState } from 'store/reducers';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { DataSource } from 'types/common/queryBuilder';
import { GlobalReducer } from 'types/reducer/globalTime';
import { CaptureDataProps } from '../CeleryTaskDetail/CeleryTaskDetail';
import { useCeleryFilterOptions } from '../useCeleryFilterOptions';
import CeleryTaskBar from './CeleryTaskBar';
import CeleryTaskGraph from './CeleryTaskGraph';
import {
@@ -20,7 +23,6 @@ import {
celeryTasksByWorkerWidgetData,
} from './CeleryTaskGraphUtils';
import CeleryTaskLatencyGraph from './CeleryTaskLatencyGraph';
import { useCeleryWorkerCount } from './useCeleryWorkerCount';
import './CeleryTaskGraph.style.scss';
@@ -69,7 +71,14 @@ export default function CeleryTaskGraphGrid({
'Latency by worker',
];
const { data: workerCount } = useCeleryWorkerCount();
const { options } = useCeleryFilterOptions(
'worker',
'rate',
DataSource.METRICS,
'flower_task_runtime_seconds_sum',
DataTypes.String,
'tag',
);
const isDarkMode = useIsDarkMode();
const [collapsedSections, setCollapsedSections] = useState<{
@@ -142,11 +151,8 @@ export default function CeleryTaskGraphGrid({
</Typography.Text>
</div>
<div className="worker-count-text-container">
<Typography.Text
className="celery-task-graph-worker-count-text"
data-testid="celery-worker-count"
>
{workerCount ?? '-'}
<Typography.Text className="celery-task-graph-worker-count-text">
{options.filter((option) => option.value).length}
</Typography.Text>
</div>
</Card>

View File

@@ -1,12 +1,3 @@
import {
MetrictypesSpaceAggregationDTO,
MetrictypesTimeAggregationDTO,
Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5MetricAggregationDTOSignal as MetricSignal,
Querybuildertypesv5QueryEnvelopeBuilderDTOType,
type Querybuildertypesv5QueryRangeRequestDTO,
Querybuildertypesv5ReduceToDTO,
Querybuildertypesv5RequestTypeDTO,
} from 'api/generated/services/sigNoz.schemas';
import { PANEL_TYPES } from 'constants/queryBuilder';
import { getWidgetQueryBuilder } from 'container/MetricsApplication/MetricsApplication.factory';
import { getWidgetQuery } from 'pages/MessagingQueues/MQDetails/MetricPage/MetricPageUtil';
@@ -1051,34 +1042,3 @@ export const celeryRetryStateCountWidgetData = getWidgetQueryBuilder(
],
}),
);
// One row per worker that reported in the range; the card counts the rows.
export const getCeleryWorkerCountRequest = (
startTime: number,
endTime: number,
): Querybuildertypesv5QueryRangeRequestDTO => ({
schemaVersion: 'v1',
start: Math.floor(startTime / 1e6),
end: Math.floor(endTime / 1e6),
requestType: Querybuildertypesv5RequestTypeDTO.scalar,
compositeQuery: {
queries: [
{
type: Querybuildertypesv5QueryEnvelopeBuilderDTOType.builder_query,
spec: {
name: 'A',
signal: MetricSignal.metrics,
aggregations: [
{
metricName: 'flower_task_runtime_seconds_sum',
timeAggregation: MetrictypesTimeAggregationDTO.count,
spaceAggregation: MetrictypesSpaceAggregationDTO.sum,
reduceTo: Querybuildertypesv5ReduceToDTO.sum,
},
],
groupBy: [{ name: 'worker' }],
},
},
],
},
});

View File

@@ -1,93 +0,0 @@
import {
Querybuildertypesv5ColumnTypeDTO,
type Querybuildertypesv5QueryRangeRequestDTO,
Querybuildertypesv5RequestTypeDTO,
type QueryRangeV5200,
} from 'api/generated/services/sigNoz.schemas';
import { rest, server } from 'mocks-server/server';
import store from 'store';
import { render, screen, waitFor } from 'tests/test-utils';
import CeleryTaskGraphGrid from '../CeleryTaskGraphGrid';
const workerCountResponse = (workers: string[]): QueryRangeV5200 => ({
status: 'success',
data: {
type: Querybuildertypesv5RequestTypeDTO.scalar,
data: {
results: [
{
columns: [
{
name: 'worker',
queryName: 'A',
columnType: Querybuildertypesv5ColumnTypeDTO.group,
},
{
name: '__result_0',
queryName: 'A',
aggregationIndex: 0,
columnType: Querybuildertypesv5ColumnTypeDTO.aggregation,
},
],
data: workers.map((worker) => [worker, 4]),
},
],
},
},
});
describe('CeleryTaskGraphGrid worker count', () => {
let requests: Querybuildertypesv5QueryRangeRequestDTO[];
beforeEach(() => {
requests = [];
server.use(
rest.post('*/api/v5/query_range', async (req, res, ctx) => {
requests.push(await req.json());
return res(
ctx.status(200),
ctx.json(workerCountResponse(['worker-a', 'worker-b', ''])),
);
}),
);
});
it('counts the workers that reported in the selected range', async () => {
render(<CeleryTaskGraphGrid onClick={jest.fn()} queryEnabled={false} />);
await waitFor(() =>
expect(screen.getByTestId('celery-worker-count')).toHaveTextContent('2'),
);
const { minTime, maxTime } = store.getState().globalTime;
expect(requests).toStrictEqual([
{
schemaVersion: 'v1',
start: Math.floor(minTime / 1e6),
end: Math.floor(maxTime / 1e6),
requestType: 'scalar',
compositeQuery: {
queries: [
{
type: 'builder_query',
spec: {
name: 'A',
signal: 'metrics',
aggregations: [
{
metricName: 'flower_task_runtime_seconds_sum',
timeAggregation: 'count',
spaceAggregation: 'sum',
reduceTo: 'sum',
},
],
groupBy: [{ name: 'worker' }],
},
},
],
},
},
]);
});
});

View File

@@ -1,31 +0,0 @@
import { useMemo } from 'react';
import { useQuery, UseQueryResult } from 'react-query';
// eslint-disable-next-line no-restricted-imports
import { useSelector } from 'react-redux';
import { queryRangeV5 } from 'api/generated/services/querier';
import type { QueryRangeV5200 } from 'api/generated/services/sigNoz.schemas';
import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
import { getScalarResults } from 'pages/DashboardPage/DashboardContainer/queryV5/v5ResponseData';
import { AppState } from 'store/reducers';
import { GlobalReducer } from 'types/reducer/globalTime';
import { getCeleryWorkerCountRequest } from './CeleryTaskGraphUtils';
export function useCeleryWorkerCount(): UseQueryResult<number, Error> {
const { minTime, maxTime } = useSelector<AppState, GlobalReducer>(
(state) => state.globalTime,
);
const request = useMemo(
() => getCeleryWorkerCountRequest(minTime, maxTime),
[minTime, maxTime],
);
return useQuery<QueryRangeV5200, Error, number>({
queryKey: [REACT_QUERY_KEY.GET_CELERY_WORKER_COUNT, request],
queryFn: ({ signal }) => queryRangeV5(request, signal),
select: (response) =>
(getScalarResults(response)[0]?.data ?? []).filter(([worker]) => worker)
.length,
});
}

View File

@@ -1,13 +1,18 @@
import { useState } from 'react';
import type { DefaultOptionType } from 'antd/es/select';
import type { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import useDebouncedFn from 'hooks/useDebouncedFunction';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { DataSource } from 'types/common/queryBuilder';
import { useGetAllFilters } from './CeleryTaskConfigOptions/useGetCeleryFilters';
export const useCeleryFilterOptions = (
type: string | string[],
fieldContext?: TelemetrytypesFieldContextDTO,
aggregateOperator?: string,
dataSource?: DataSource,
aggregateAttribute?: string,
filterAttributeKeyDataType?: DataTypes,
tagType?: string,
): {
searchText: string;
handleSearch: (value: string) => void;
@@ -18,7 +23,11 @@ export const useCeleryFilterOptions = (
const { isFetching, options } = useGetAllFilters({
attributeKey: type,
searchText,
fieldContext,
aggregateOperator,
dataSource,
aggregateAttribute,
filterAttributeKeyDataType,
tagType,
});
const handleDebouncedSearch = useDebouncedFn((searchText): void => {
setSearchText(searchText as string);

View File

@@ -1,5 +1,5 @@
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import { TelemetryFieldKey } from 'types/api/v5/queryRange';
import { fieldContextToAttributeType } from 'utils/fieldContext';
import { fieldDataTypeToDataType } from 'utils/fieldDataType';
import { FiltersType, IQuickFiltersConfig } from './types';
@@ -14,6 +14,19 @@ const FILTER_TYPE_MAP: Record<string, FiltersType> = {
duration_nano: FiltersType.DURATION,
};
// The map below exists only for the old v3 attribute-values fetch
// (useCheckboxFilterValues), the sole reader of attributeKey.dataType/type.
// Once the values fetch moves to fields/values, remove this and reduce
// attributeKey to { id, key }.
const FIELD_CONTEXT_TO_ATTRIBUTE_TYPE: Record<string, string> = {
[TelemetrytypesFieldContextDTO.attribute]: 'tag',
[TelemetrytypesFieldContextDTO.resource]: 'resource',
};
const mapFieldContext = (fieldContext?: string): string =>
(fieldContext && FIELD_CONTEXT_TO_ATTRIBUTE_TYPE[fieldContext]) || '';
const getFilterName = (str: string): string => {
if (FILTER_TITLE_MAP[str]) {
return FILTER_TITLE_MAP[str];
@@ -48,15 +61,11 @@ export const getFilterConfig = (
({
type: getFilterType(att),
title: getFilterName(att.name),
// The type mapping below exists only for the old v3 attribute-values fetch
// (useCheckboxFilterValues), the sole reader of attributeKey.dataType/type.
// Once the values fetch moves to fields/values, remove this and reduce
// attributeKey to { id, key }.
attributeKey: {
id: att.name,
key: att.name,
dataType: fieldDataTypeToDataType(att.fieldDataType),
type: fieldContextToAttributeType(att.fieldContext),
type: mapFieldContext(att.fieldContext),
},
defaultOpen: index < 2,
}) as IQuickFiltersConfig,

View File

@@ -15,5 +15,3 @@ export const DASHBOARD_TIME_IN_DURATION = 'refreshInterval';
export const DEFAULT_ENTITY_VERSION = 'v3';
export const ENTITY_VERSION_V4 = 'v4';
export const ENTITY_VERSION_V5 = 'v5';
export const SIGNOZ_ADMIN_EMAIL = 'admin@signoz.cloud';

View File

@@ -109,9 +109,6 @@ export const REACT_QUERY_KEY = {
// Field Values Suggestion Query Keys
FIELD_VALUES_SUGGESTION: 'FIELD_VALUES_SUGGESTION',
// Celery Query Keys
GET_CELERY_WORKER_COUNT: 'GET_CELERY_WORKER_COUNT',
// AI Assistant Query Keys
AI_ASSISTANT_EMPTY_STATE_CHIPS: 'AI_ASSISTANT_EMPTY_STATE_CHIPS',
} as const;

View File

@@ -1,70 +0,0 @@
import { Form } from 'antd';
import QueryBuilderSearchV2 from 'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2';
import { rest, server } from 'mocks-server/server';
import { act, render, waitFor } from 'tests/test-utils';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import type { QueryRangePayload } from 'types/api/metrics/getQueryRange';
import type {
IBuilderQuery,
TagFilter,
} from 'types/api/queryBuilder/queryBuilderData';
import { pipelineFields } from '../../../../config';
import FilterInput from '..';
jest.mock(
'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2',
() => ({ __esModule: true, default: jest.fn(() => null) }),
);
const filterField = pipelineFields.find((field) => field.name === 'filter');
const v5Filter: TagFilter = {
op: 'AND',
items: [
{
id: 'attribute-pill',
key: { key: 'http.method', dataType: DataTypes.String, type: 'attribute' },
op: '=',
value: 'GET',
},
{
id: 'log-pill',
key: { key: 'severity_text', dataType: DataTypes.String, type: 'log' },
op: '=',
value: 'ERROR',
},
],
};
describe('Pipeline FilterInput', () => {
it('previews sample logs with the legacy key types the v3 logs builder reads', async () => {
const previewFilters: TagFilter[] = [];
server.use(
rest.post('*/api/v3/query_range', async (req, res, ctx) => {
const body: QueryRangePayload = await req.json();
const query = body.compositeQuery.builderQueries?.A as IBuilderQuery;
if (query.filters) {
previewFilters.push(query.filters);
}
return res(ctx.status(200), ctx.json({ data: { result: [] } }));
}),
);
render(<Form>{filterField && <FilterInput fieldData={filterField} />}</Form>);
const { onChange } = jest.mocked(QueryBuilderSearchV2).mock.calls[0][0];
act(() => onChange(v5Filter));
await waitFor(() => expect(previewFilters).not.toHaveLength(0));
expect(
previewFilters[previewFilters.length - 1].items.map((item) => [
item.key?.key,
item.key?.type,
]),
).toStrictEqual([
['http.method', 'tag'],
['severity_text', ''],
]);
});
});

View File

@@ -4,7 +4,6 @@ import { initialQueryBuilderFormValuesMap } from 'constants/queryBuilder';
import QueryBuilderSearchV2 from 'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2';
import isEqual from 'lodash-es/isEqual';
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
import { toLegacyLogsFilter } from 'utils/fieldContext';
import { ProcessorFormField } from '../../../AddNewProcessor/config';
import { formValidationRules } from '../../../config';
@@ -23,10 +22,9 @@ function TagFilterInput({
}
const onQueryChange = (newValue: TagFilter): void => {
const legacyValue = toLegacyLogsFilter(newValue);
// Avoid unnecessary onChange calls
if (!isEqual(legacyValue, query.filters)) {
onChange(legacyValue);
if (!isEqual(newValue, query.filters)) {
onChange(newValue);
}
};

View File

@@ -10,6 +10,7 @@ import {
userEvent,
waitFor,
} from 'tests/test-utils';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { pipelineApiResponseMockData } from '../mocks/pipeline';
import PipelineListsView from '../PipelineListsView';
@@ -84,7 +85,7 @@ jest.mock('providers/preferences/sync/usePreferenceSync', () => ({
}));
const BASE_URL = ENVIRONMENT.baseURL;
const fieldKeysURL = `${BASE_URL}/api/v1/fields/keys`;
const attributeKeysURL = `${BASE_URL}/api/v3/filter_suggestions`;
describe('PipelinePage container test', () => {
beforeAll(() => {
@@ -338,41 +339,42 @@ describe('PipelinePage container test', () => {
expect(screen.queryAllByText('source = nginx')).toHaveLength(2);
server.use(
rest.get(fieldKeysURL, (_req, res, ctx) =>
rest.get(attributeKeysURL, (_req, res, ctx) =>
res(
ctx.status(200),
ctx.json({
status: 'success',
data: {
complete: true,
keys: {
otelServiceName: [
{
name: 'otelServiceName',
fieldContext: 'attribute',
fieldDataType: 'string',
},
],
'service.name': [
{
name: 'service.name',
fieldContext: 'resource',
fieldDataType: 'string',
},
{
name: 'service.name',
fieldContext: 'attribute',
fieldDataType: 'string',
},
],
'service.instance.id': [
{
name: 'service.instance.id',
fieldContext: 'resource',
fieldDataType: 'string',
},
],
},
attributes: [
{
key: 'otelServiceName',
dataType: DataTypes.String,
type: 'tag',
isColumn: false,
isJSON: false,
},
{
key: 'service.name',
dataType: DataTypes.String,
type: 'resource',
isColumn: false,
isJSON: false,
},
{
key: 'service.instance.id',
dataType: DataTypes.String,
type: 'resource',
isColumn: false,
isJSON: false,
},
{
key: 'service.name',
dataType: DataTypes.String,
type: 'tag',
isColumn: false,
isJSON: false,
},
],
},
}),
),

View File

@@ -10,9 +10,9 @@ import {
} from 'react';
import { type RefSelectProps, Select, Spin } from 'antd';
import { Pill } from '@signozhq/ui/pill';
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import cx from 'classnames';
import {
DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY,
OperatorConfigKeys,
OPERATORS,
QUERY_BUILDER_OPERATORS_BY_TYPES,
@@ -22,14 +22,23 @@ import { DEBOUNCE_DELAY } from 'constants/queryBuilderFilterConfig';
import { LogsExplorerShortcuts } from 'constants/shortcuts/logsExplorerShortcuts';
import { useDynamicVariableSuggestions } from 'hooks/dashboard/useDynamicVariableSuggestions';
import { useKeyboardHotkeys } from 'hooks/hotkeys/useKeyboardHotkeys';
import { useGetAggregateKeys } from 'hooks/queryBuilder/useGetAggregateKeys';
import { useGetAggregateValues } from 'hooks/queryBuilder/useGetAggregateValues';
import { useGetAttributeSuggestions } from 'hooks/queryBuilder/useGetAttributeSuggestions';
import { validationMapper } from 'hooks/queryBuilder/useIsValidTag';
import { operatorTypeMapper } from 'hooks/queryBuilder/useOperatorType';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { useRunQueryWithFilterItems } from 'hooks/queryBuilder/useRunQueryWithFilterItems';
import { useFieldKeysSuggestion } from 'hooks/querySuggestions/useFieldKeysSuggestion';
import { useFieldValuesSuggestion } from 'hooks/querySuggestions/useFieldValuesSuggestion';
import useDebounceValue from 'hooks/useDebounce';
import { isArray, isEmpty, isEqual, isObject, isUndefined } from 'lodash-es';
import {
cloneDeep,
isArray,
isEmpty,
isEqual,
isObject,
isUndefined,
unset,
} from 'lodash-es';
import { ChevronDown, ChevronUp } from '@signozhq/icons';
import {
BaseAutocompleteData,
@@ -39,13 +48,7 @@ import {
IBuilderQuery,
TagFilter,
} from 'types/api/queryBuilder/queryBuilderData';
import {
DATA_SOURCE_TO_SIGNAL,
DataSource,
QUERY_SOURCE_TO_FIELD_SOURCE,
} from 'types/common/queryBuilder';
import { attributeTypeToFieldContext } from 'utils/fieldContext';
import { getValuesForDataType } from 'utils/fieldValues';
import { DataSource } from 'types/common/queryBuilder';
import { useSelectPopupContainer } from 'utils/selectPopupContainer';
import { v4 as uuid } from 'uuid';
@@ -58,8 +61,7 @@ import {
getTagToken,
isInNInOperator,
} from './utils';
import { toAttributeKey } from '../GroupByFilter/utils';
import { filterByOperatorConfig, getMetricNameForQuery } from '../utils';
import { filterByOperatorConfig } from '../utils';
import QueryBuilderSearchDropdown from './QueryBuilderSearchDropdown';
import SpanScopeSelector from './SpanScopeSelector';
import Suggestions from './Suggestions';
@@ -73,8 +75,6 @@ export interface ITag {
value: (string | number | boolean)[] | string | number | boolean;
}
const SPAN_SEARCH_SCOPE_KEYS = ['isRoot', 'isEntryPoint'];
interface CustomTagProps {
label: React.ReactNode;
value: string;
@@ -186,14 +186,69 @@ function QueryBuilderSearchV2(
[query.dataSource],
);
const debouncedKeySearchText = useDebounceValue(
searchValue?.split(' ')[0] ?? '',
DEBOUNCE_DELAY,
const memoizedSearchParams = useMemo(
() => [
searchValue,
query.dataSource,
query.aggregateOperator,
query.aggregateAttribute?.key,
],
[
searchValue,
query.dataSource,
query.aggregateOperator,
query.aggregateAttribute?.key,
],
);
const valueSearchText = isArray(currentFilterItem?.value)
? String(currentFilterItem?.value?.[currentFilterItem.value.length - 1] ?? '')
: currentFilterItem?.value?.toString() || '';
const queryFiltersWithoutId = useMemo(
() => ({
...query.filters,
items: query.filters?.items.map((item) => {
const filterWithoutId = cloneDeep(item);
unset(filterWithoutId, 'id');
return filterWithoutId;
}),
}),
[query.filters],
);
const memoizedSuggestionsParams = useMemo(
() => [searchValue, query.dataSource, queryFiltersWithoutId],
[query.dataSource, queryFiltersWithoutId, searchValue],
);
const memoizedValueParams = useMemo(
() => [
query.aggregateOperator,
query.dataSource,
query.aggregateAttribute?.key,
currentFilterItem?.key?.key || '',
currentFilterItem?.key?.dataType,
currentFilterItem?.key?.type ?? '',
isArray(currentFilterItem?.value)
? currentFilterItem?.value?.[currentFilterItem.value.length - 1]
: currentFilterItem?.value,
],
[
query.aggregateOperator,
query.dataSource,
query.aggregateAttribute?.key,
currentFilterItem?.key?.key,
currentFilterItem?.key?.dataType,
currentFilterItem?.key?.type,
currentFilterItem?.value,
],
);
const searchParams = useDebounceValue(memoizedSearchParams, DEBOUNCE_DELAY);
const valueParams = useDebounceValue(memoizedValueParams, DEBOUNCE_DELAY);
const suggestionsParams = useDebounceValue(
memoizedSuggestionsParams,
DEBOUNCE_DELAY,
);
const isQueryEnabled = useMemo(() => {
if (currentState === DropdownState.ATTRIBUTE_KEY) {
@@ -207,76 +262,53 @@ function QueryBuilderSearchV2(
const dashboardDynamicVariables = useDynamicVariableSuggestions();
const signal = DATA_SOURCE_TO_SIGNAL[query.dataSource];
const source = QUERY_SOURCE_TO_FIELD_SOURCE[query.source ?? ''];
const metricName = getMetricNameForQuery(query);
const { data: fieldKeys, isFetching } = useFieldKeysSuggestion(
const { data, isFetching } = useGetAggregateKeys(
{
signal,
source,
metricName,
fieldContext: attributeTypeToFieldContext(query.aggregateAttribute?.type),
searchText: debouncedKeySearchText,
searchText: searchValue?.split(' ')[0],
dataSource: query.dataSource,
aggregateOperator: query.aggregateOperator || '',
aggregateAttribute: query.aggregateAttribute?.key || '',
tagType: query.aggregateAttribute?.type ?? null,
},
query.builderQueryType,
{
enabled: isQueryEnabled && !hardcodedAttributeKeys,
keepPreviousData: true,
queryKey: [searchParams],
enabled: isQueryEnabled && !isLogsDataSource && !hardcodedAttributeKeys,
},
);
// The legacy traces builder checks `exists` on the column only for keys marked as columns.
// Body JSON keys come back as bare paths, which the legacy builders read as top-level fields,
// and the span search scope selectors are not columns the legacy filter builder can read.
const attributeKeys = useMemo(
() =>
(fieldKeys ?? [])
.filter(
(fieldKey) =>
fieldKey.fieldContext !== TelemetrytypesFieldContextDTO.body &&
!(
fieldKey.fieldContext === TelemetrytypesFieldContextDTO.span &&
SPAN_SEARCH_SCOPE_KEYS.includes(fieldKey.name)
),
)
.map((fieldKey) => ({
...toAttributeKey(fieldKey),
isColumn:
fieldKey.fieldContext === TelemetrytypesFieldContextDTO.span ||
fieldKey.fieldContext === TelemetrytypesFieldContextDTO.log,
})),
[fieldKeys],
);
const { data: suggestionsData, isFetching: isFetchingSuggestions } =
useGetAttributeSuggestions(
{
searchText: searchValue?.split(' ')[0],
dataSource: query.dataSource,
filters: query.filters || { items: [], op: 'AND' },
},
{
queryKey: [suggestionsParams],
enabled: isQueryEnabled && isLogsDataSource,
},
);
const fieldValuesConfig = useMemo(
() => ({
signal,
source,
metricName,
name: currentFilterItem?.key?.key || '',
fieldContext: attributeTypeToFieldContext(currentFilterItem?.key?.type),
searchText: valueSearchText,
}),
[
signal,
source,
metricName,
currentFilterItem?.key?.key,
currentFilterItem?.key?.type,
valueSearchText,
],
);
const debouncedFieldValuesConfig = useDebounceValue(
fieldValuesConfig,
DEBOUNCE_DELAY,
);
const { data: fieldValues, isFetching: isFetchingAttributeValues } =
useFieldValuesSuggestion(debouncedFieldValuesConfig, query.builderQueryType, {
enabled: currentState === DropdownState.ATTRIBUTE_VALUE,
});
const { data: attributeValues, isFetching: isFetchingAttributeValues } =
useGetAggregateValues(
{
aggregateOperator: query.aggregateOperator || '',
dataSource: query.dataSource,
aggregateAttribute: query.aggregateAttribute?.key || '',
attributeKey: currentFilterItem?.key?.key || '',
filterAttributeKeyDataType:
currentFilterItem?.key?.dataType ?? DataTypes.EMPTY,
tagType: currentFilterItem?.key?.type ?? '',
searchText: isArray(currentFilterItem?.value)
? String(currentFilterItem?.value?.[currentFilterItem.value.length - 1]) ||
''
: currentFilterItem?.value?.toString() || '',
},
{
enabled: currentState === DropdownState.ATTRIBUTE_VALUE,
queryKey: [valueParams],
},
);
const handleDropdownSelect = useCallback(
(value: string) => {
@@ -501,14 +533,16 @@ function QueryBuilderSearchV2(
if (tagKey && isUndefined(currentFilterItem?.key)) {
let currentRunningAttributeKey;
const isSuggestedKeyInAutocomplete =
isLogsDataSource &&
attributeKeys.some((value) => value.key === tagKey.split(' ')[0]);
if (isSuggestedKeyInAutocomplete) {
const allAttributesMatchingTheKey = attributeKeys.filter(
suggestionsData?.payload?.attributes?.some(
(value) => value.key === tagKey.split(' ')[0],
);
if (isSuggestedKeyInAutocomplete) {
const allAttributesMatchingTheKey =
suggestionsData?.payload?.attributes?.filter(
(value) => value.key === tagKey.split(' ')[0],
) || [];
if (allAttributesMatchingTheKey?.length === 1) {
[currentRunningAttributeKey] = allAttributesMatchingTheKey;
}
@@ -614,10 +648,9 @@ function QueryBuilderSearchV2(
currentFilterItem,
currentFilterItem?.key,
currentFilterItem?.op,
attributeKeys,
isLogsDataSource,
suggestionsData?.payload?.attributes,
searchValue,
isFetching,
isFetchingSuggestions,
currentState,
]);
@@ -629,7 +662,9 @@ function QueryBuilderSearchV2(
// add the user typed option in the dropdown to select that and move ahead irrespective of the matches and all
setDropdownOptions([
...(!isEmpty(tagKey) &&
!attributeKeys.some((val) => isEqual(val.key, tagKey))
!suggestionsData?.payload?.attributes?.some((val) =>
isEqual(val.key, tagKey),
)
? [
{
label: tagKey,
@@ -641,10 +676,10 @@ function QueryBuilderSearchV2(
},
]
: []),
...attributeKeys.map((key) => ({
...(suggestionsData?.payload?.attributes?.map((key) => ({
label: key.key,
value: key,
})),
})) || []),
]);
} else if (hardcodedAttributeKeys) {
const filteredKeys = hardcodedAttributeKeys.filter((key) =>
@@ -662,7 +697,7 @@ function QueryBuilderSearchV2(
setDropdownOptions([
// Add user typed option if it doesn't exist in the payload
...(tagKey.trim().length > 0 &&
!attributeKeys.some((val) => val.key === tagKey)
!data?.payload?.attributeKeys?.some((val) => val.key === tagKey)
? [
{
label: tagKey,
@@ -675,10 +710,10 @@ function QueryBuilderSearchV2(
]
: []),
// Map existing attribute keys from payload
...attributeKeys.map((key) => ({
...(data?.payload?.attributeKeys?.map((key) => ({
label: key.key,
value: key,
})),
})) || []),
]);
}
}
@@ -743,10 +778,10 @@ function QueryBuilderSearchV2(
values.push(tagValue);
}
if (fieldValues) {
values.push(
...getValuesForDataType(fieldValues, currentFilterItem?.key?.dataType),
);
if (attributeValues?.payload) {
const dataType = currentFilterItem?.key?.dataType || DataTypes.String;
const key = DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY[dataType];
values.push(...(attributeValues?.payload?.[key] || []));
// here we want to suggest the variable name matching with the key here, we will go over the dynamic variables for the keys
const variableName = dashboardDynamicVariables.find(
@@ -770,12 +805,13 @@ function QueryBuilderSearchV2(
}
}, [
hardcodedAttributeKeys,
fieldValues,
attributeValues?.payload,
currentFilterItem?.key?.dataType,
currentState,
attributeKeys,
data?.payload?.attributeKeys,
isLogsDataSource,
searchValue,
suggestionsData?.payload?.attributes,
operatorConfigKey,
currentFilterItem?.key?.key,
dashboardDynamicVariables,
@@ -844,8 +880,8 @@ function QueryBuilderSearchV2(
}, [deregisterShortcut, isLastQuery, registerShortcut]);
const loading = useMemo(
() => isFetching || isFetchingAttributeValues,
[isFetching, isFetchingAttributeValues],
() => isFetching || isFetchingAttributeValues || isFetchingSuggestions,
[isFetching, isFetchingAttributeValues, isFetchingSuggestions],
);
const isMetricsDataSource = useMemo(
@@ -955,7 +991,7 @@ function QueryBuilderSearchV2(
setTags((prev) => [...prev, ...(val.items as ITag[])]);
}}
searchValue={searchValue}
exampleQueries={[]}
exampleQueries={suggestionsData?.payload?.example_queries || []}
tags={tags}
currentFilterItem={currentFilterItem}
isLogsDataSource={isLogsDataSource}

View File

@@ -1,23 +1,13 @@
import { QueryClient, QueryClientProvider } from 'react-query';
import {
render,
RenderResult,
screen,
waitFor,
within,
} from '@testing-library/react';
import { render, RenderResult, screen, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import {
initialQueriesMap,
initialQueryBuilderFormValues,
} from 'constants/queryBuilder';
import { TelemetrytypesTelemetryFieldValuesDTO } from 'api/generated/services/sigNoz.schemas';
import { DynamicVariableSuggestion } from 'providers/Dashboard/store/dynamicVariableSuggestions';
import { useFieldKeysSuggestion } from 'hooks/querySuggestions/useFieldKeysSuggestion';
import { useFieldValuesSuggestion } from 'hooks/querySuggestions/useFieldValuesSuggestion';
import { QueryBuilderContext } from 'providers/QueryBuilder';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { FieldDataType, TelemetryFieldKey } from 'types/api/v5/queryRange';
import { DataSource } from 'types/common/queryBuilder';
import QueryBuilderSearchV2 from '../QueryBuilderSearchV2';
@@ -98,34 +88,57 @@ const renderWithContext = (props = {}): RenderResult => {
const getSearchCombobox = (): HTMLElement =>
within(screen.getByTestId('qb-search-select')).getByRole('combobox');
const mockFieldKeys: TelemetryFieldKey[] = [
{ name: 'http.status', fieldContext: 'attribute', fieldDataType: 'string' },
{ name: 'service.name', fieldContext: 'attribute', fieldDataType: 'string' },
{ name: 'name', fieldContext: 'span', fieldDataType: 'string' },
{ name: 'user.id', fieldContext: 'body', fieldDataType: 'string' },
{ name: 'isRoot', fieldContext: 'span', fieldDataType: 'bool' },
{
name: 'unmapped.attribute',
fieldContext: 'attribute',
fieldDataType: 'not-a-real-data-type' as FieldDataType,
},
];
// Constants to fix linter errors
const TYPE_TAG = 'tag';
const IS_COLUMN_FALSE = false;
const IS_JSON_FALSE = false;
jest.mock('hooks/querySuggestions/useFieldKeysSuggestion', () => ({
useFieldKeysSuggestion: jest.fn(() => ({
data: mockFieldKeys,
const mockAggregateKeysData = {
payload: {
attributeKeys: [
{
key: 'http.status',
dataType: DataTypes.String,
type: TYPE_TAG,
id: 'http.status--string--tag--false',
},
{
key: 'service.name',
dataType: DataTypes.String,
type: TYPE_TAG,
isColumn: IS_COLUMN_FALSE,
isJSON: IS_JSON_FALSE,
id: 'service.name--string--tag--false',
},
{
key: 'unmapped.attribute',
dataType: 'not-a-real-data-type' as unknown as DataTypes,
type: TYPE_TAG,
isColumn: IS_COLUMN_FALSE,
isJSON: IS_JSON_FALSE,
id: 'unmapped.attribute--String--tag--false',
},
],
},
};
jest.mock('hooks/queryBuilder/useGetAggregateKeys', () => ({
useGetAggregateKeys: jest.fn(() => ({
data: mockAggregateKeysData,
isFetching: false,
})),
}));
const mockFieldValues: TelemetrytypesTelemetryFieldValuesDTO = {
stringValues: ['200', '404', '500'],
numberValues: [200, 404, 500],
const mockAggregateValuesData = {
payload: {
stringAttributeValues: ['200', '404', '500'],
numberAttributeValues: [200, 404, 500],
},
};
jest.mock('hooks/querySuggestions/useFieldValuesSuggestion', () => ({
useFieldValuesSuggestion: jest.fn(() => ({
data: mockFieldValues,
jest.mock('hooks/queryBuilder/useGetAggregateValues', () => ({
useGetAggregateValues: jest.fn(() => ({
data: mockAggregateValuesData,
isFetching: false,
})),
}));
@@ -183,11 +196,7 @@ describe('Suggestion Key -> Operator -> Value Flow', () => {
expect.objectContaining({
items: expect.arrayContaining([
expect.objectContaining({
key: expect.objectContaining({
key: 'http.status',
type: 'attribute',
dataType: 'string',
}),
key: expect.objectContaining({ key: 'http.status' }),
op: '=',
value: '200',
}),
@@ -283,7 +292,7 @@ describe('Filter chips', () => {
key: {
key: 'http.status',
dataType: DataTypes.String,
type: 'tag',
type: TYPE_TAG,
id: 'http.status--string--tag--false',
},
op: '=',
@@ -323,87 +332,3 @@ describe('Filter chips', () => {
expect(getSearchCombobox()).toHaveDisplayValue(CHIP_LABEL);
});
});
describe('Key fetch context', () => {
it('limits keys to the aggregate attribute context, as the resource filter needs', () => {
renderWithContext({
query: {
...defaultProps.query,
aggregateAttribute: {
...defaultProps.query.aggregateAttribute,
type: 'resource',
},
},
});
expect(useFieldKeysSuggestion).toHaveBeenLastCalledWith(
expect.objectContaining({ fieldContext: 'resource' }),
undefined,
expect.anything(),
);
});
it('fetches keys from every context when the aggregate attribute has no type', () => {
renderWithContext();
expect(useFieldKeysSuggestion).toHaveBeenLastCalledWith(
expect.objectContaining({ fieldContext: undefined }),
undefined,
expect.anything(),
);
});
});
describe('Keys the legacy filter builders cannot read', () => {
it('does not suggest body JSON keys or span search scope selectors', async () => {
const user = userEvent.setup({ delay: null });
renderWithContext();
const combobox = getSearchCombobox();
await user.click(combobox);
await user.type(combobox, 'user');
await screen.findByRole('listbox');
await expect(screen.findByText('http.status')).resolves.toBeInTheDocument();
expect(screen.queryByText('user.id')).not.toBeInTheDocument();
expect(screen.queryByText('isRoot')).not.toBeInTheDocument();
});
});
describe('Span field keys', () => {
it('marks span fields as columns and looks up their values in the span context', async () => {
const user = userEvent.setup({ delay: null });
renderWithContext();
const combobox = getSearchCombobox();
await user.click(combobox);
await user.type(combobox, 'nam');
await screen.findByRole('listbox');
await user.click(await screen.findByText('name'));
await user.click(screen.getByText('='));
await waitFor(() =>
expect(jest.mocked(useFieldValuesSuggestion)).toHaveBeenLastCalledWith(
expect.objectContaining({ name: 'name', fieldContext: 'span' }),
undefined,
expect.anything(),
),
);
await user.click(screen.getByText('200'));
expect(mockOnChange).toHaveBeenLastCalledWith(
expect.objectContaining({
items: expect.arrayContaining([
expect.objectContaining({
key: expect.objectContaining({
key: 'name',
type: 'span',
isColumn: true,
}),
}),
]),
}),
);
});
});

View File

@@ -24,6 +24,7 @@ import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
import {
EXCEPTION_CATALOGUE_SIZE,
EXCEPTION_QUICK_FILTER_CAP,
exceptionAttributeKeysResponse,
exceptionAttributeValuesResponse,
exceptionFieldKeysResponse,
exceptionQuickFiltersResponse,
@@ -57,7 +58,7 @@ export const exceptionsMocks = defineStoryMocks({
filterKeys: choiceControl<ResponseState>('Filter keys', {
group: FILTERS,
description:
'How `/fields/keys` answers when the resource filter opens, apart from the page-wide Data control.',
'How `/autocomplete/attribute_keys` answers when the resource filter opens, apart from the page-wide Data control.',
options: RESPONSE_STATES,
value: 'loaded',
}),
@@ -94,7 +95,7 @@ export const exceptionsMocks = defineStoryMocks({
rest.get(
'http://localhost/api/v1/fields/keys',
respondWith(values.filterKeys, (req) =>
response.json((req) =>
exceptionFieldKeysResponse(req.url.searchParams.get('searchText')),
),
),
@@ -108,6 +109,23 @@ export const exceptionsMocks = defineStoryMocks({
),
),
),
rest.get(
'http://localhost/api/v3/autocomplete/attribute_keys',
respondWith(values.filterKeys, (req) =>
exceptionAttributeKeysResponse(req.url.searchParams.get('searchText')),
),
),
rest.get(
'http://localhost/api/v3/autocomplete/attribute_values',
response.json((req) =>
exceptionAttributeValuesResponse(
req.url.searchParams.get('attributeKey'),
req.url.searchParams.get('searchText'),
),
),
),
],
effect: (values) => {
set(LOCALSTORAGE.SHOW_EXCEPTIONS_QUICK_FILTERS, String(values.filterPanel));

View File

@@ -17,6 +17,7 @@ import type {
import {
type BaseAutocompleteData,
DataTypes,
type IQueryAutocompleteResponse,
} from 'types/api/queryBuilder/queryAutocompleteResponse';
import type { Exception, Order, OrderBy } from 'types/api/errors/getAll';
import type { IAttributeValuesResponse } from 'types/api/queryBuilder/getAttributesValues';
@@ -310,6 +311,21 @@ const ATTRIBUTE_KEYS: BaseAutocompleteData[] = [
{ key: 'cloud.region', dataType: DataTypes.String, type: 'resource' },
];
export const exceptionAttributeKeysResponse = (
searchText: string | null,
): { status: string; data: IQueryAutocompleteResponse } => {
const search = (searchText ?? '').toLowerCase();
return {
status: 'success',
data: {
attributeKeys: ATTRIBUTE_KEYS.filter(({ key }) =>
key.toLowerCase().includes(search),
),
},
};
};
/** The quick filter settings panel lists its "other filters" from `/fields/keys`. */
export const exceptionFieldKeysResponse = (
searchText: string | null,

View File

@@ -9,8 +9,6 @@ import type { QueryRangePayload } from 'types/api/metrics/getQueryRange';
import { choiceControl, countControl } from '@/storybook/controls/controls';
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
import { queryRangeV5ScalarTableResponse } from '@/storybook/msw/__story_mockdata__/queryRange';
import {
attributeValuesResponse,
@@ -77,34 +75,30 @@ export const celeryTaskMocks = defineStoryMocks({
return [
rest.post('http://localhost/api/v3/query_range', queryRange),
rest.post('http://localhost/api/v4/query_range', queryRange),
rest.post(
'http://localhost/api/v5/query_range',
response.json(() =>
queryRangeV5ScalarTableResponse({
groupBy: ['worker'],
aggregations: ['A'],
rows: (
attributeValuesResponse('worker', '').data.stringAttributeValues ?? []
)
.slice(0, values.workers)
.map((worker) => [worker, 12]),
}),
),
),
rest.get(
'http://localhost/api/v1/fields/values',
'http://localhost/api/v3/autocomplete/attribute_values',
response.json((req) => {
const name = req.url.searchParams.get('name') ?? '';
const found =
attributeValuesResponse(
name,
req.url.searchParams.get('searchText') ?? '',
).data.stringAttributeValues ?? [];
return fieldValuesResponse(
name === 'worker' ? found.slice(0, values.workers) : found,
const attributeKey = req.url.searchParams.get('attributeKey') ?? '';
const values_ = attributeValuesResponse(
attributeKey,
req.url.searchParams.get('searchText') ?? '',
);
if (attributeKey !== 'worker') {
return values_;
}
return {
...values_,
data: {
...values_.data,
stringAttributeValues: (values_.data.stringAttributeValues ?? []).slice(
0,
values.workers,
),
},
};
}),
),
];

View File

@@ -13,7 +13,6 @@ import {
toggleControl,
} from '@/storybook/controls/controls';
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
import {
attributeValuesResponse,
@@ -92,13 +91,11 @@ export const queueOverviewMocks = defineStoryMocks({
rest.post('http://localhost/api/v4/query_range', queryRange),
rest.get(
'http://localhost/api/v1/fields/values',
'http://localhost/api/v3/autocomplete/attribute_values',
response.json((req) =>
fieldValuesResponse(
attributeValuesResponse(
req.url.searchParams.get('name') ?? '',
req.url.searchParams.get('searchText') ?? '',
).data.stringAttributeValues ?? [],
attributeValuesResponse(
req.url.searchParams.get('attributeKey') ?? '',
req.url.searchParams.get('searchText') ?? '',
),
),
),

View File

@@ -82,15 +82,15 @@ const CELERY_TASKS = [
const CELERY_WORKERS = ['worker-a', 'worker-b', 'worker-c'];
/**
* Values per attribute key. The kafka config selects read the v3 endpoint and
* the celery filter rows read `/fields/values`, so one map answers for all of
* Values per autocomplete attribute key. The kafka config selects and both
* celery filter rows read the same v3 endpoint, so one map answers for all of
* them; a key nobody seeded answers empty rather than 501.
*/
const ATTRIBUTE_VALUES: Record<string, string[]> = {
group: CONSUMER_GROUPS,
topic: TOPICS,
partition: PARTITIONS,
'service.name': KAFKA_SERVICES,
serviceName: KAFKA_SERVICES,
name: [
'order-events send',
'order-events process',

View File

@@ -3,7 +3,6 @@ import { Button, Form, Space, Tooltip } from 'antd';
import { Dropdown, type DropdownItemType } from '@signozhq/ui/dropdown';
import { Divider } from '@signozhq/ui/divider';
import { Switch } from '@signozhq/ui/switch';
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import cx from 'classnames';
import { FilterSelect } from 'components/CeleryOverview/CeleryOverviewConfigOptions/CeleryOverviewConfigOptions';
import { QueryParams } from 'constants/query';
@@ -150,8 +149,7 @@ function FunnelStep({
<FilterSelect
placeholder="Select Service"
queryParam={QueryParams.service}
filterType="service.name"
fieldContext={TelemetrytypesFieldContextDTO.resource}
filterType="serviceName"
shouldSetQueryParams={false}
values={stepData.service_name}
isMultiple={false}
@@ -171,7 +169,6 @@ function FunnelStep({
placeholder="Select Span name"
queryParam={QueryParams.spanName}
filterType="name"
fieldContext={TelemetrytypesFieldContextDTO.span}
shouldSetQueryParams={false}
values={stepData.span_name}
isMultiple={false}

View File

@@ -3,19 +3,15 @@
* Do not hand-edit: regenerate instead.
*/
import {
TelemetrytypesFieldContextDTO,
TelemetrytypesSignalDTO,
} from 'api/generated/services/sigNoz.schemas';
import ROUTES from 'constants/routes';
import { rest } from 'msw';
import { choiceControl, countControl } from '@/storybook/controls/controls';
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
import {
fieldKeysResponse,
fieldValuesResponse,
} from '@/storybook/msw/__story_mockdata__/fields';
attributeKeysResponse,
attributeValuesResponse,
} from '@/storybook/msw/__story_mockdata__/attributes';
import {
FUNNEL_ID,
@@ -45,9 +41,13 @@ const CONVERSION: Record<DropOff, number> = {
const ENTERED = 24_000;
const STEP_RESOURCE_KEYS = ['service.name', 'deployment.environment'];
const STEP_ATTRIBUTE_KEYS = ['name', 'http.status_code', 'hasError'];
const STEP_FILTER_KEYS = [
'service.name',
'name',
'http.status_code',
'deployment.environment',
'hasError',
];
const STEP_FILTER_VALUES: Record<string, string[]> = {
'service.name': ['frontend', 'checkout', 'payment', 'shipping'],
@@ -115,44 +115,20 @@ export const tracesFunnelDetailsMocks = defineStoryMocks({
'http://localhost/api/v1/trace-funnels/analytics/error-traces',
response.json(() => funnelErrorTracesResponse(values.traces)),
),
// The `Where` box on every step reads its suggestions as soon as the step
// renders, and the service and span name selects read `service.name` and
// `name` from the same values endpoint.
// The `Where` box on every step is the trace query builder, which reads
// its suggestions as soon as the step renders.
rest.get(
'http://localhost/api/v1/fields/keys',
response.json(() => {
const signal = TelemetrytypesSignalDTO.traces;
const resource = fieldKeysResponse(STEP_RESOURCE_KEYS, {
signal,
fieldContext: TelemetrytypesFieldContextDTO.resource,
});
const attribute = fieldKeysResponse(STEP_ATTRIBUTE_KEYS, {
signal,
fieldContext: TelemetrytypesFieldContextDTO.attribute,
});
return {
...resource,
data: {
...resource.data,
keys: { ...resource.data.keys, ...attribute.data.keys },
},
};
}),
'http://localhost/api/v3/autocomplete/attribute_keys',
response.json(() => attributeKeysResponse(STEP_FILTER_KEYS)),
),
rest.get(
'http://localhost/api/v1/fields/values',
response.json((req) => {
const searchText = (
req.url.searchParams.get('searchText') ?? ''
).toLowerCase();
return fieldValuesResponse(
(STEP_FILTER_VALUES[req.url.searchParams.get('name') ?? ''] ?? []).filter(
(value) => value.toLowerCase().includes(searchText),
),
);
}),
'http://localhost/api/v3/autocomplete/attribute_values',
response.json((req) =>
attributeValuesResponse(
STEP_FILTER_VALUES[req.url.searchParams.get('attributeKey') ?? ''] ?? [],
req.url.searchParams.get('searchText') ?? '',
),
),
),
];
},

View File

@@ -53,7 +53,6 @@ export type FieldContext =
| 'scope'
| 'attribute'
| 'event'
| 'body'
| '';
export type FieldDataType =

View File

@@ -1,6 +1,4 @@
import { SIGNOZ_ADMIN_EMAIL } from 'constants/app';
import { buildAbsolutePath, isSignozAdminEmail } from '../app';
import { buildAbsolutePath } from '../app';
// buildAbsolutePath reads getCurrentLocation().pathname (basename-relative) rather than
// window.location.pathname, so we mock lib/router/navigation instead of utils/getLocation.
@@ -124,13 +122,3 @@ describe('buildAbsolutePath', () => {
});
});
});
describe('isSignozAdminEmail', () => {
it('matches the SigNoz admin email', () => {
expect(isSignozAdminEmail(SIGNOZ_ADMIN_EMAIL)).toBe(true);
});
it.each(['jon@winterfell.dev', '', undefined])('rejects %p', (email) => {
expect(isSignozAdminEmail(email)).toBe(false);
});
});

View File

@@ -1,76 +0,0 @@
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
import {
attributeTypeToFieldContext,
toLegacyLogsFilter,
} from '../fieldContext';
const item = (type: string): TagFilter['items'][number] => ({
id: type,
key: { key: 'k', dataType: DataTypes.String, type },
op: '=',
value: 'v',
});
describe('toLegacyLogsFilter', () => {
it('sends attribute keys as tag and log fields as top-level fields', () => {
const filter: TagFilter = {
op: 'AND',
items: [
item('attribute'),
item('resource'),
item('scope'),
item('log'),
item('body'),
item('tag'),
item(''),
],
};
expect(
toLegacyLogsFilter(filter).items.map((filterItem) => filterItem.key?.type),
).toStrictEqual(['tag', 'resource', 'scope', '', '', 'tag', '']);
});
it('sends the scope name and version as top-level fields only from the scope context', () => {
const filter: TagFilter = {
op: 'AND',
items: [
['scope_name', 'scope'],
['scope_version', 'scope'],
['scope.attr', 'scope'],
['scope_name', 'attribute'],
['scope_name', 'tag'],
['scope_version', 'resource'],
].map(([key, type]) => ({
id: `${key}-${type}`,
key: { key, dataType: DataTypes.String, type },
op: '=',
value: 'v',
})),
};
expect(
toLegacyLogsFilter(filter).items.map((filterItem) => filterItem.key?.type),
).toStrictEqual(['', '', 'scope', 'tag', 'tag', 'resource']);
});
});
describe('attributeTypeToFieldContext', () => {
it('maps legacy tag to attribute and passes v5 contexts through', () => {
expect(
['tag', 'attribute', 'resource', 'scope', 'span', 'log', ''].map(
attributeTypeToFieldContext,
),
).toStrictEqual([
'attribute',
'attribute',
'resource',
'scope',
'span',
'log',
undefined,
]);
});
});

View File

@@ -1,5 +1,4 @@
import getLocalStorage from 'api/browser/localstorage/get';
import { SIGNOZ_ADMIN_EMAIL } from 'constants/app';
import { FeatureKeys } from 'constants/features';
import { SKIP_ONBOARDING } from 'constants/onboarding';
import dayjs from 'dayjs';
@@ -17,9 +16,6 @@ export function extractDomain(email: string): string {
return emailParts[1];
}
export const isSignozAdminEmail = (email?: string): boolean =>
email === SIGNOZ_ADMIN_EMAIL;
const stripVersionPrefix = (version: string): string =>
version?.replace(/^v/i, '') ?? '';

View File

@@ -1,55 +0,0 @@
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
import type { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
const FIELD_CONTEXT_TO_ATTRIBUTE_TYPE: Record<string, string> = {
[TelemetrytypesFieldContextDTO.attribute]: 'tag',
[TelemetrytypesFieldContextDTO.resource]: 'resource',
};
const ATTRIBUTE_TYPE_TO_FIELD_CONTEXT: Record<
string,
TelemetrytypesFieldContextDTO
> = {
tag: TelemetrytypesFieldContextDTO.attribute,
attribute: TelemetrytypesFieldContextDTO.attribute,
resource: TelemetrytypesFieldContextDTO.resource,
scope: TelemetrytypesFieldContextDTO.scope,
span: TelemetrytypesFieldContextDTO.span,
log: TelemetrytypesFieldContextDTO.log,
};
// The legacy logs builders read `tag` as attributes and an empty type as a top-level field;
// any other type falls through to resources.
const LEGACY_LOGS_KEY_TYPE: Record<string, string> = {
[TelemetrytypesFieldContextDTO.attribute]: 'tag',
tag: 'tag',
[TelemetrytypesFieldContextDTO.resource]: 'resource',
[TelemetrytypesFieldContextDTO.scope]: 'scope',
};
// Top-level log columns that v5 files under the scope context.
const LEGACY_LOGS_TOP_LEVEL_SCOPE_FIELDS = ['scope_name', 'scope_version'];
export const fieldContextToAttributeType = (fieldContext?: string): string =>
(fieldContext && FIELD_CONTEXT_TO_ATTRIBUTE_TYPE[fieldContext]) || '';
export const attributeTypeToFieldContext = (
type?: string | null,
): TelemetrytypesFieldContextDTO | undefined =>
type ? ATTRIBUTE_TYPE_TO_FIELD_CONTEXT[type] : undefined;
export const toLegacyLogsFilter = (filter: TagFilter): TagFilter =>
filter && {
...filter,
items: (filter.items ?? []).map((item) => ({
...item,
key: item.key && {
...item.key,
type:
item.key.type === TelemetrytypesFieldContextDTO.scope &&
LEGACY_LOGS_TOP_LEVEL_SCOPE_FIELDS.includes(item.key.key)
? ''
: (LEGACY_LOGS_KEY_TYPE[item.key.type ?? ''] ?? ''),
},
})),
};

View File

@@ -1,21 +0,0 @@
import { TelemetrytypesTelemetryFieldValuesDTO } from 'api/generated/services/sigNoz.schemas';
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
// Numbers and bools stay raw so tags keep their value type.
export const getValuesForDataType = (
values: TelemetrytypesTelemetryFieldValuesDTO | undefined,
dataType: DataTypes | undefined,
): string[] => {
switch (dataType) {
case DataTypes.Int64:
case DataTypes.Float64:
case DataTypes.ArrayInt64:
case DataTypes.ArrayFloat64:
return (values?.numberValues ?? []) as unknown as string[];
case DataTypes.bool:
case DataTypes.ArrayBool:
return (values?.boolValues ?? []) as unknown as string[];
default:
return values?.stringValues ?? [];
}
};

View File

@@ -26,8 +26,9 @@ const bucketCacheCellSteps = 60
type BucketCache interface {
// cached portion + list of gaps to fetch
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
// store fresh buckets for future hits
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
// store fresh buckets for future hits; only the ranges in the map were
// queried for fresh, the rest of it came from the cache
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats)
}
// bucketCache implements the BucketCache interface.
@@ -116,7 +117,7 @@ func (bc *bucketCache) GetMissRanges(
}
// Merge buckets into a single result
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings)
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings, startMs, endMs)
// Filter the merged result to only include values within the requested time range
mergedResult = bc.filterResultToTimeRange(mergedResult, q, startMs, endMs, stepMs)
@@ -125,7 +126,7 @@ func (bc *bucketCache) GetMissRanges(
}
// Put stores fresh query results in the cache.
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result) {
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) {
// Get query window
startMs, endMs := q.Window()
@@ -188,7 +189,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
// Convert trimmed result to buckets, one per grid cell it spans
cellWidthMs := bucketCacheCellSteps * stepMs
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs)
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs, execStatsForQueriedTimeRanges)
cellKeys := make([]string, len(freshBuckets))
cellEntries := make([]cachetypes.Cacheable, len(freshBuckets))
@@ -210,7 +211,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
cellEntry = cellEntries[i].(*qbtypes.CachedData)
}
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket)
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket, execStatsForQueriedTimeRanges)
if !changed {
continue
}
@@ -463,7 +464,7 @@ func (bc *bucketCache) filterRelevantBuckets(buckets []*qbtypes.CachedBucket, st
}
// mergeBuckets combines multiple cached buckets into a single result.
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string) *qbtypes.Result {
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string, startMs, endMs uint64) *qbtypes.Result {
if len(buckets) == 0 {
return &qbtypes.Result{}
}
@@ -471,12 +472,13 @@ func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.Cach
// All buckets should have the same type
resultType := buckets[0].Type
// Aggregate stats
// A bucket can reach past the window; only the part inside it is served
var totalStats qbtypes.ExecStats
for _, bucket := range buckets {
totalStats.RowsScanned += bucket.Stats.RowsScanned
totalStats.BytesScanned += bucket.Stats.BytesScanned
totalStats.DurationMS += bucket.Stats.DurationMS
servedStats := bucket.ProrateStatsToWindow(startMs, endMs)
totalStats.RowsScanned += servedStats.RowsScanned
totalStats.BytesScanned += servedStats.BytesScanned
totalStats.DurationMS += servedStats.DurationMS
}
// Merge values based on type
@@ -673,7 +675,7 @@ func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFi
}
// resultToBuckets converts a query result into time-based buckets.
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64) []*qbtypes.CachedBucket {
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) []*qbtypes.CachedBucket {
// Check if result is empty
isEmpty, isFiltered := bc.isEmptyResult(result)
@@ -685,7 +687,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
}
// Empty filtered results still become buckets, so the range is not re-queried
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs)
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs, execStatsForQueriedTimeRanges)
if err != nil {
bc.logger.ErrorContext(ctx, "failed to convert result to buckets", errors.Attr(err))
return nil
@@ -697,7 +699,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
// and reports whether the cell changed: a bucket already covering the fresh
// one leaves it as is, buckets inside the fresh one are replaced, buckets
// overlapping or edge to edge with it are unioned, and the rest are kept apart.
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket) ([]*qbtypes.CachedBucket, bool) {
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) ([]*qbtypes.CachedBucket, bool) {
keptBuckets := make([]*qbtypes.CachedBucket, 0, len(cellBuckets)+1)
touchingBuckets := []*qbtypes.CachedBucket{}
for _, cellBucket := range cellBuckets {
@@ -710,7 +712,7 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
keptBuckets = append(keptBuckets, cellBucket)
}
}
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets))
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets, execStatsForQueriedTimeRanges))
slices.SortFunc(keptBuckets, func(a, b *qbtypes.CachedBucket) int {
return cmp.Compare(a.StartMs, b.StartMs)
@@ -721,7 +723,16 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
// unionBuckets widens the fresh bucket by the buckets it touches. Buckets
// inside it are replaced outright, so only a genuine partial overlap decodes
// anything.
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket) *qbtypes.CachedBucket {
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) *qbtypes.CachedBucket {
// Data queried again takes the new query's cost, which the fresh bucket
// carries; a touching bucket keeps its cost only for what was not queried again
for _, touchingBucket := range touchingBuckets {
keptStats := touchingBucket.ProrateStatsToUnqueried(execStatsForQueriedTimeRanges)
freshBucket.Stats.RowsScanned += keptStats.RowsScanned
freshBucket.Stats.BytesScanned += keptStats.BytesScanned
freshBucket.Stats.DurationMS += keptStats.DurationMS
}
bucketsToMerge := []*qbtypes.CachedBucket{freshBucket}
for _, touchingBucket := range touchingBuckets {
if !freshBucket.Contains(touchingBucket) {
@@ -742,14 +753,11 @@ func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.Ca
EndMs: freshBucket.EndMs,
Type: freshBucket.Type,
Value: value,
Stats: qbtypes.ExecStats{StepIntervals: freshBucket.Stats.StepIntervals},
Stats: freshBucket.Stats,
}
for _, mergedBucket := range bucketsToMerge {
unionBucket.StartMs = min(unionBucket.StartMs, mergedBucket.StartMs)
unionBucket.EndMs = max(unionBucket.EndMs, mergedBucket.EndMs)
unionBucket.Stats.RowsScanned += mergedBucket.Stats.RowsScanned
unionBucket.Stats.BytesScanned += mergedBucket.Stats.BytesScanned
unionBucket.Stats.DurationMS += mergedBucket.Stats.DurationMS
}
return unionBucket
}

View File

@@ -30,7 +30,7 @@ func BenchmarkBucketCache_GetMissRanges(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
}
// Create test queries with varying cache hit patterns
@@ -121,7 +121,7 @@ func BenchmarkBucketCache_Put(b *testing.B) {
for i := 0; i < b.N; i++ {
for j := 0; j < tc.numQueries; j++ {
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j])
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j], map[qbtypes.TimeRange]qbtypes.ExecStats{{From: queries[j].startMs, To: queries[j].endMs}: results[j].Stats})
}
}
})
@@ -259,7 +259,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
}
b.ResetTimer()
@@ -284,7 +284,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
endMs: uint64((i + 1) * 10000),
}
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
case 2: // Partial read
query := &mockQuery{
fingerprint: fmt.Sprintf("concurrent-query-%d", i%100),

View File

@@ -59,7 +59,7 @@ func TestBucketCacheStepAlignment(t *testing.T) {
}
// Put result in cache
bc.Put(ctx, orgID, query, step, result)
bc.Put(ctx, orgID, query, step, result, nil)
// Get cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
@@ -109,7 +109,7 @@ func TestBucketCacheNoStepInterval(t *testing.T) {
}
// Should cache the entire range when step is 0
bc.Put(ctx, orgID, query, step, result)
bc.Put(ctx, orgID, query, step, result, nil)
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
assert.NotNil(t, cached)

View File

@@ -156,7 +156,7 @@ func TestBucketCache_Put_And_Get(t *testing.T) {
}
// Store in cache
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
// Wait a bit for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -189,7 +189,7 @@ func TestBucketCache_PartialHit(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 3000, 1000),
}
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
// Wait for cache write
time.Sleep(10 * time.Millisecond)
@@ -225,7 +225,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 2000, 100),
})
}, nil)
query2 := &mockQuery{
fingerprint: "test-query",
@@ -235,7 +235,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
bc.Put(context.Background(), valuer.UUID{}, query2, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 3000, 4000, 100),
})
}, nil)
// Wait for cache writes
time.Sleep(10 * time.Millisecond)
@@ -280,7 +280,7 @@ func TestBucketCache_FluxInterval(t *testing.T) {
}
// This should not be cached due to flux interval
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
// Wait a bit
time.Sleep(10 * time.Millisecond)
@@ -358,7 +358,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
{Series: []*qbtypes.TimeSeries{series1}},
},
},
})
}, nil)
// Cache second part
query2 := &mockQuery{
@@ -374,7 +374,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
{Series: []*qbtypes.TimeSeries{series2}},
},
},
})
}, nil)
// Wait for cache writes
time.Sleep(10 * time.Millisecond)
@@ -441,7 +441,7 @@ func TestBucketCache_RawData(t *testing.T) {
Value: rawData,
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -481,7 +481,7 @@ func TestBucketCache_ScalarData(t *testing.T) {
Value: scalarData,
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -509,7 +509,7 @@ func TestBucketCache_EmptyFingerprint(t *testing.T) {
Value: createTestTimeSeries("A", 1000, 5000, 1000),
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
// Should still be able to retrieve
@@ -564,7 +564,7 @@ func TestBucketCache_ConcurrentAccess(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries(fmt.Sprintf("Q%d", id), query.startMs, query.endMs, 100),
}
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result)
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result, nil)
done <- true
}(i)
}
@@ -624,7 +624,7 @@ func TestBucketCache_GetMissRanges_FluxInterval(t *testing.T) {
},
}
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult, nil)
// Get miss ranges
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -686,7 +686,7 @@ func TestBucketCache_Put_FluxIntervalTrimming(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
// Retrieve cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -756,7 +756,7 @@ func TestBucketCache_Put_EntireRangeInFluxInterval(t *testing.T) {
}
// Put the result - should not cache anything
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
// Try to get cached data - should have no cached data
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
@@ -862,7 +862,7 @@ func TestBucketCache_EmptyDataHandling(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result, nil)
// Wait a bit for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -927,7 +927,7 @@ func TestBucketCache_PartialValues(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
// Wait for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -997,7 +997,7 @@ func TestBucketCache_AllPartialValues(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
// Wait for cache to be written
time.Sleep(10 * time.Millisecond)
@@ -1058,7 +1058,7 @@ func TestBucketCache_FilteredCachedResults(t *testing.T) {
}
// Cache the wide range
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
time.Sleep(10 * time.Millisecond)
// Now query for a smaller range (2000-3500ms)
@@ -1125,7 +1125,7 @@ func TestBucketCache_PromQLKeepsTheValueAtTheWindowEnd(t *testing.T) {
}},
}},
},
})
}, nil)
time.Sleep(10 * time.Millisecond)
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
@@ -1291,7 +1291,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1345,7 +1345,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1397,7 +1397,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
}
// Put the result
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
// Get cached data
@@ -1454,7 +1454,7 @@ func TestBucketCache_NoCache(t *testing.T) {
}
// Put the result in cache
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
time.Sleep(10 * time.Millisecond)
// Verify data is cached
@@ -1475,11 +1475,12 @@ func TestBucketCachePutSlicesAtCellBoundaries(t *testing.T) {
// a 1s step gives 60s cells, so 30s to 150s spans three of them
query := &mockQuery{fingerprint: "sliced-query", startMs: 30000, endMs: 150000}
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
result := &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 30000, 150000, 1000),
Stats: qbtypes.ExecStats{RowsScanned: 120, BytesScanned: 1200, DurationMS: 12},
})
}
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: 30000, To: 150000}: result.Stats})
type bucketShape struct {
cell, startMs, endMs, rowsScanned uint64
@@ -1539,7 +1540,7 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
name: "OverlappingWrites_BecomeOne",
firstWindow: qbtypes.TimeRange{From: 1000, To: 4000},
secondWindow: qbtypes.TimeRange{From: 3000, To: 6000},
expectedBuckets: []bucketShape{{1000, 6000, 6}},
expectedBuckets: []bucketShape{{1000, 6000, 5}},
expectedMissingRanges: []*qbtypes.TimeRange{},
expectedValueCount: 5,
},
@@ -1570,12 +1571,13 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
for _, window := range []qbtypes.TimeRange{testCase.firstWindow, testCase.secondWindow} {
query := &mockQuery{fingerprint: "absorb-query", startMs: window.From, endMs: window.To}
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
result := &qbtypes.Result{
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", window.From, window.To, 1000),
// one row per second, so a bucket's stats read as the seconds it was scanned for
// one row per second, so a bucket's stats read as the seconds of data it holds
Stats: qbtypes.ExecStats{RowsScanned: (window.To - window.From) / 1000},
})
}
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{window: result.Stats})
}
query := &mockQuery{
@@ -1609,7 +1611,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 1000, 10000, 1000),
Warnings: []string{"from the wide write"},
})
}, nil)
// a write already covered by the cell's bucket must not rewrite the entry,
// which shows as its warnings never arriving
@@ -1618,7 +1620,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
Type: qbtypes.RequestTypeTimeSeries,
Value: createTestTimeSeries("A", 3000, 5000, 1000),
Warnings: []string{"from the inside write"},
})
}, nil)
var cellEntry qbtypes.CachedData
require.NoError(t, bc.cache.Get(ctx, orgID, bc.generateCacheKey(wide, 0), &cellEntry))

View File

@@ -828,7 +828,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
return nil, err
}
// Store in cache for future use
q.bucketCache.Put(ctx, orgID, query, step, result)
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
return result, nil
}
}
@@ -884,16 +884,19 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
if err != nil {
return nil, err
}
q.bucketCache.Put(ctx, orgID, query, step, result)
startMs, endMs := query.Window()
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
return result, nil
}
}
// Calculate total stats and filter out nil results
validResults := make([]*qbtypes.Result, 0, len(freshResults))
for _, result := range freshResults {
execStatsForQueriedTimeRanges := make(map[qbtypes.TimeRange]qbtypes.ExecStats, len(freshResults))
for i, result := range freshResults {
if result != nil {
validResults = append(validResults, result)
execStatsForQueriedTimeRanges[*missingRanges[i]] = result.Stats
totalStats.RowsScanned += result.Stats.RowsScanned
totalStats.BytesScanned += result.Stats.BytesScanned
totalStats.DurationMS += result.Stats.DurationMS
@@ -908,7 +911,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
mergedResult.Stats.DurationMS += totalStats.DurationMS
// Store merged result in cache
q.bucketCache.Put(ctx, orgID, query, step, mergedResult)
q.bucketCache.Put(ctx, orgID, query, step, mergedResult, execStatsForQueriedTimeRanges)
return mergedResult, nil
}

View File

@@ -43,6 +43,23 @@ func (c *CachedBucket) Touches(other *CachedBucket) bool {
return c.StartMs <= other.EndMs && other.StartMs <= c.EndMs
}
// ProrateStatsToWindow returns the share of the stats for the part of the
// bucket inside [startMs, endMs).
func (c *CachedBucket) ProrateStatsToWindow(startMs, endMs uint64) ExecStats {
return c.Stats.prorate(overlapMs(c.StartMs, c.EndMs, startMs, endMs), c.EndMs-c.StartMs)
}
// ProrateStatsToUnqueried returns the share of the stats for the part of the
// bucket no queried range covers.
func (c *CachedBucket) ProrateStatsToUnqueried(execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ExecStats {
widthMs := c.EndMs - c.StartMs
queriedMs := uint64(0)
for queried := range execStatsForQueriedTimeRanges {
queriedMs += overlapMs(c.StartMs, c.EndMs, queried.From, queried.To)
}
return c.Stats.prorate(widthMs-min(queriedMs, widthMs), widthMs)
}
// CachedData represents the full cached data for a query.
type CachedData struct {
Buckets []*CachedBucket `json:"buckets"`
@@ -88,7 +105,7 @@ func (c *CachedData) Cost() int64 {
return size
}
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64) ([]*CachedBucket, error) {
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ([]*CachedBucket, error) {
tsData, ok := r.Value.(*TimeSeriesData)
if !ok || tsData == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "result of type %s cannot be bucketed", r.Type.StringValue())
@@ -111,18 +128,39 @@ func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64) (
if err != nil {
return nil, err
}
// A slice costs what was queried inside it; the rest of the result came
// out of the cache and is costed there already
sliceStats := ExecStats{StepIntervals: maps.Clone(r.Stats.StepIntervals)}
for queried, stats := range execStatsForQueriedTimeRanges {
queriedMs := overlapMs(from, to, queried.From, queried.To)
if queriedMs == 0 {
continue
}
share := stats.prorate(queriedMs, queried.To-queried.From)
sliceStats.RowsScanned += share.RowsScanned
sliceStats.BytesScanned += share.BytesScanned
sliceStats.DurationMS += share.DurationMS
}
cellBuckets = append(cellBuckets, &CachedBucket{
StartMs: from,
EndMs: to,
Type: r.Type,
Value: value,
Stats: r.Stats.prorate(to-from, endMs-startMs),
Stats: sliceStats,
})
from = to
}
return cellBuckets, nil
}
// overlapMs is the length of [aFromMs, aToMs) ∩ [bFromMs, bToMs).
func overlapMs(aFromMs, aToMs, bFromMs, bToMs uint64) uint64 {
if fromMs, toMs := max(aFromMs, bFromMs), min(aToMs, bToMs); fromMs < toMs {
return toMs - fromMs
}
return 0
}
// prorate returns the share of the stats that partMs out of wholeMs of the
// scanned window accounts for.
func (e ExecStats) prorate(partMs, wholeMs uint64) ExecStats {

View File

@@ -1,51 +0,0 @@
import json
import time
import docker
from fixtures import types
AUDIT_FILE_PATH = "/tmp/audit.log"
def list_audit_events(signoz: types.SigNoz) -> list[dict]:
"""Each line of the audit file is one OTLP JSON batch of events; this flattens all batches into events."""
exit_code, output = docker.from_env().containers.get(signoz.self.id).exec_run(["cat", AUDIT_FILE_PATH])
assert exit_code == 0, output.decode()
events = []
for line in output.decode().splitlines():
for resource_logs in json.loads(line)["resourceLogs"]:
resource = _attributes(resource_logs["resource"]["attributes"])
for scope_logs in resource_logs["scopeLogs"]:
for log_record in scope_logs["logRecords"]:
events.append(
{
"event_name": log_record["eventName"],
"severity_text": log_record["severityText"],
"body": log_record["body"]["stringValue"],
"scope_name": scope_logs["scope"]["name"],
"resource": resource,
"attributes": _attributes(log_record["attributes"]),
}
)
return events
def wait_for_audit_event(signoz: types.SigNoz, event_name: str, resource_id: str, target_id: str | None = None, wait_seconds: int = 10) -> dict:
deadline = time.time() + wait_seconds
events = []
while time.time() < deadline:
events = list_audit_events(signoz)
matched = [event for event in events if event["event_name"] == event_name and event["resource"].get("signoz.resource.id") == resource_id and (target_id is None or event["attributes"].get("signoz.resource.target.id") == target_id)]
if len(matched) > 0:
return matched[0]
time.sleep(0.5)
raise AssertionError(f"No {event_name} audit event for resource {resource_id} (target {target_id}) within {wait_seconds}s, events: {events}")
def _attributes(otlp_attributes: list[dict]) -> dict:
# OTLP JSON encodes int64 values as decimal strings.
return {attribute["key"]: int(attribute["value"]["intValue"]) if "intValue" in attribute["value"] else attribute["value"]["stringValue"] for attribute in otlp_attributes}

View File

@@ -1,151 +0,0 @@
import uuid
from collections.abc import Callable
from http import HTTPStatus
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, find_user_by_email
from fixtures.fileauditor import wait_for_audit_event
from fixtures.savedview import SAVED_VIEW_BASE, create_saved_view
from fixtures.serviceaccount import SERVICE_ACCOUNT_BASE, create_service_account, create_service_account_with_key, delete_service_account, get_first_key_id
def test_create_saved_view_writes_create_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
admin = find_user_by_email(signoz, token, USER_ADMIN_EMAIL)
view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
event = wait_for_audit_event(signoz, "signoz.audit.create", view_id)
assert event["severity_text"] == "INFO"
assert event["scope_name"] == "signoz.audit"
assert event["resource"]["service.name"] == "signoz"
assert event["resource"]["signoz.resource.kind"] == "saved-view"
assert event["attributes"]["signoz.audit.verb"] == "create"
assert event["attributes"]["signoz.audit.outcome"] == "success"
assert event["attributes"]["signoz.principal.id"] == admin["id"]
assert event["attributes"]["signoz.principal.email"] == USER_ADMIN_EMAIL
assert event["attributes"]["signoz.organization.id"] == admin["orgId"]
assert event["attributes"]["http.request.method"] == "POST"
assert event["attributes"]["http.route"] == "/api/v2/saved_views"
assert event["attributes"]["http.response.status_code"] == HTTPStatus.CREATED
def test_update_service_account_writes_update_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
service_account_id = create_service_account(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
response = requests.put(
signoz.self.host_configs["8080"].get(f"{SERVICE_ACCOUNT_BASE}/{service_account_id}"),
json={"name": f"fileauditor-{uuid.uuid4().hex[:8]}"},
headers={"Authorization": f"Bearer {token}"},
timeout=5,
)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
event = wait_for_audit_event(signoz, "signoz.audit.update", service_account_id)
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
assert event["attributes"]["signoz.audit.verb"] == "update"
assert event["attributes"]["signoz.audit.outcome"] == "success"
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}"
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NO_CONTENT
def test_delete_service_account_writes_delete_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
service_account_id = create_service_account(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
delete_service_account(signoz, token, service_account_id)
event = wait_for_audit_event(signoz, "signoz.audit.delete", service_account_id)
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
assert event["attributes"]["signoz.audit.verb"] == "delete"
assert event["attributes"]["signoz.audit.outcome"] == "success"
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}"
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NO_CONTENT
def test_create_service_account_key_writes_attach_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
service_account_id = create_service_account(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
response = requests.post(
signoz.self.host_configs["8080"].get(f"{SERVICE_ACCOUNT_BASE}/{service_account_id}/keys"),
json={"name": "fileauditor", "expiresAt": 0},
headers={"Authorization": f"Bearer {token}"},
timeout=5,
)
assert response.status_code == HTTPStatus.CREATED, response.text
key_id = response.json()["data"]["id"]
event = wait_for_audit_event(signoz, "signoz.audit.attach", service_account_id, target_id=key_id)
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
assert event["attributes"]["signoz.resource.target.kind"] == "factor-api-key"
assert event["attributes"]["signoz.audit.verb"] == "attach"
assert event["attributes"]["signoz.audit.outcome"] == "success"
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}/keys"
assert event["attributes"]["http.response.status_code"] == HTTPStatus.CREATED
def test_revoke_service_account_key_writes_detach_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
service_account_id, _ = create_service_account_with_key(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
key_id = get_first_key_id(signoz, token, service_account_id)
response = requests.delete(
signoz.self.host_configs["8080"].get(f"{SERVICE_ACCOUNT_BASE}/{service_account_id}/keys/{key_id}"),
headers={"Authorization": f"Bearer {token}"},
timeout=5,
)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
event = wait_for_audit_event(signoz, "signoz.audit.detach", service_account_id, target_id=key_id)
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
assert event["attributes"]["signoz.resource.target.kind"] == "factor-api-key"
assert event["attributes"]["signoz.audit.verb"] == "detach"
assert event["attributes"]["signoz.audit.outcome"] == "success"
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}/keys/{fid}"
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NO_CONTENT
def test_delete_missing_saved_view_writes_failure_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
missing_view_id = str(uuid.uuid4())
response = requests.delete(
signoz.self.host_configs["8080"].get(f"{SAVED_VIEW_BASE}/{missing_view_id}"),
headers={"Authorization": f"Bearer {token}"},
timeout=5,
)
assert response.status_code == HTTPStatus.NOT_FOUND, response.text
event = wait_for_audit_event(signoz, "signoz.audit.delete", missing_view_id)
assert event["severity_text"] == "ERROR"
assert event["attributes"]["signoz.audit.outcome"] == "failure"
assert event["attributes"]["exception.type"] == "not-found"
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NOT_FOUND

View File

@@ -1,49 +0,0 @@
import uuid
from collections.abc import Callable
from sqlalchemy import sql
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.fileauditor import list_audit_events, wait_for_audit_event
from fixtures.savedview import create_saved_view
def test_unlicensed_create_writes_no_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
with signoz.sqlstore.conn.begin() as conn:
licenses = conn.execute(sql.text("SELECT id, created_at, updated_at, key, data, last_validated_at, org_id FROM license")).mappings().all()
assert len(licenses) > 0
conn.execute(sql.text("DELETE FROM license"))
try:
unlicensed_view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
finally:
with signoz.sqlstore.conn.begin() as conn:
conn.execute(
sql.text("INSERT INTO license (id, created_at, updated_at, key, data, last_validated_at, org_id) VALUES (:id, :created_at, :updated_at, :key, :data, :last_validated_at, :org_id)"),
[dict(row) for row in licenses],
)
marker_view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
wait_for_audit_event(signoz, "signoz.audit.create", marker_view_id)
assert all(event["resource"].get("signoz.resource.id") != unlicensed_view_id for event in list_audit_events(signoz))
def test_login_writes_no_event(
signoz: types.SigNoz,
apply_license: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
marker_view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
wait_for_audit_event(signoz, "signoz.audit.create", marker_view_id)
assert all(event["attributes"].get("http.route") != "/api/v2/sessions/email_password" for event in list_audit_events(signoz))

View File

@@ -1,32 +0,0 @@
import pytest
from testcontainers.core.container import Network
from fixtures import types
from fixtures.fileauditor import AUDIT_FILE_PATH
from fixtures.signoz import create_signoz
@pytest.fixture(name="signoz", scope="package")
def signoz_fileauditor(
network: Network,
zeus: types.TestContainerDocker,
gateway: types.TestContainerDocker,
sqlstore: types.TestContainerSQL,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.SigNoz:
return create_signoz(
network=network,
zeus=zeus,
gateway=gateway,
sqlstore=sqlstore,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
cache_key="signoz-fileauditor",
env_overrides={
"SIGNOZ_AUDITOR_PROVIDER": "file",
"SIGNOZ_AUDITOR_FILE_PATH": AUDIT_FILE_PATH,
},
)

View File

@@ -442,3 +442,55 @@ def test_builder_refreshing_a_sliding_time_range(
assert uncached.status_code == HTTPStatus.OK, uncached.text
assert_results_equal(from_cache.json(), uncached.json(), "A", "the twentieth refresh")
def test_builder_refreshing_does_not_inflate_rows_scanned(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_metrics: Callable[[list[Metrics]], None],
) -> None:
metric_name = f"cache_rows_scanned_{uuid4().hex[:8]}"
# 90 minutes back so even the last refresh closes clear of the flux
# interval, which holds recent data out of the cache
start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=90)).timestamp()) // 60 * 60, tz=UTC)
start_time_ms = int(start_time.timestamp() * 1000)
query = [build_builder_query("A", metric_name, "max", "max")]
insert_metrics(
[
Metrics(
metric_name=metric_name,
labels={"service": "api"},
timestamp=start_time + timedelta(minutes=minute),
value=minute,
type_="Gauge",
is_monotonic=False,
)
for minute in range(70)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# a dashboard left open on a thirty minute range, re-running a minute later
# each time. rowsScanned is a cost indicator, so a refresh served mostly out
# of the cache cannot claim more than every read made so far: the first
# full read and then one new minute per refresh, measured with the cache off
uncached_rows_scanned_so_far = 0
for refresh in range(10):
refresh_start_ms = start_time_ms + refresh * MINUTE_MS
refresh_end_ms = refresh_start_ms + 30 * MINUTE_MS
uncached_start_ms = refresh_start_ms if refresh == 0 else refresh_end_ms - MINUTE_MS
uncached_response = make_query_request(signoz, token, uncached_start_ms, refresh_end_ms, query, no_cache=True)
assert uncached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: uncached read failed: {uncached_response.text}"
uncached_rows_scanned_so_far += uncached_response.json()["data"]["meta"]["rowsScanned"]
assert uncached_rows_scanned_so_far > 0, f"refresh {refresh}: uncached read reported no rows scanned"
cached_response = make_query_request(signoz, token, refresh_start_ms, refresh_end_ms, query, no_cache=False)
assert cached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: cached read failed: {cached_response.text}"
cached_rows_scanned = cached_response.json()["data"]["meta"]["rowsScanned"]
assert cached_rows_scanned <= uncached_rows_scanned_so_far, f"refresh {refresh}: cached read reported {cached_rows_scanned} rows scanned, more than the {uncached_rows_scanned_so_far} rows all uncached reads scanned so far"