mirror of
https://github.com/SigNoz/signoz.git
synced 2026-11-16 13:21:15 +00:00
Compare commits
7 Commits
test/filea
...
nv/exec-st
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8101a79371 | ||
|
|
62d477e920 | ||
|
|
fe45b9dcae | ||
|
|
63ee60a40c | ||
|
|
29a393e47e | ||
|
|
c58f76f998 | ||
|
|
e39ad9e221 |
28
.github/workflows/e2eci.yaml
vendored
28
.github/workflows/e2eci.yaml
vendored
@@ -2,50 +2,44 @@ name: e2eci
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- labeled
|
||||
merge_group:
|
||||
types:
|
||||
- checks_requested
|
||||
|
||||
# Labels other than safe-to-e2e get a group of their own, so they never cancel a running e2e.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event.action == 'labeled' && github.event.label.name != 'safe-to-e2e' && format('-{0}', github.run_id) || '' }}
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
changes:
|
||||
if: github.event.action != 'labeled' || github.event.label.name == 'safe-to-e2e'
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
e2e: ${{ steps.changes.outputs.e2e }}
|
||||
steps:
|
||||
- name: merge-queue-pass
|
||||
if: github.event_name == 'merge_group'
|
||||
run: echo "::notice::Merge queue run. E2E already gated the PR, so the required e2e check auto-passes."
|
||||
- name: changes
|
||||
id: changes
|
||||
if: github.event_name == 'pull_request'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
FORCE: ${{ github.event_name == 'merge_group' || contains(github.event.pull_request.labels.*.name, 'safe-to-e2e') }}
|
||||
FORCE: ${{ contains(github.event.pull_request.labels.*.name, 'safe-to-e2e') }}
|
||||
run: |
|
||||
if [ "$FORCE" = "true" ]; then
|
||||
echo "e2e=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CHANGED_FILES=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER/files" --paginate --jq '.[].filename | select((startswith("tests/integration/") or startswith("frontend/src/api/generated/")) | not)')
|
||||
PATTERN='^(frontend/|tests/|\.github/workflows/e2eci\.yaml$|cmd/enterprise/Dockerfile\.with-web\.integration$)'
|
||||
|
||||
if printf '%s\n' "$CHANGED_FILES" | grep -qE "$PATTERN"; then
|
||||
if [ "$FORCE" = "true" ]; then
|
||||
echo "e2e=true" >> "$GITHUB_OUTPUT"
|
||||
elif printf '%s\n' "$CHANGED_FILES" | grep -qE "$PATTERN"; then
|
||||
echo "e2e=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "e2e=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
fmtlint:
|
||||
needs: changes
|
||||
if: github.event_name != 'merge_group' && needs.changes.outputs.e2e == 'true'
|
||||
if: needs.changes.outputs.e2e == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
@@ -186,8 +180,6 @@ jobs:
|
||||
retention-days: 5
|
||||
# Required status check. Fails when e2e was needed but did not pass.
|
||||
e2e:
|
||||
# Runs for other labels must not report as e2e, or their pass would hide the real result.
|
||||
name: ${{ github.event.action == 'labeled' && github.event.label.name != 'safe-to-e2e' && 'ignored-label' || 'e2e' }}
|
||||
if: always()
|
||||
needs: [changes, fmtlint, test]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
1
.github/workflows/integrationci.yaml
vendored
1
.github/workflows/integrationci.yaml
vendored
@@ -45,7 +45,6 @@ jobs:
|
||||
- callbackauthn
|
||||
- cloudintegrations
|
||||
- dashboard
|
||||
- fileauditor
|
||||
- ingestionkeys
|
||||
- inframonitoring
|
||||
- llmpricingrules
|
||||
|
||||
@@ -13,7 +13,6 @@ import { useAppContext } from 'providers/App/App';
|
||||
import { LicensePlatform, LicenseState } from 'types/api/licensesV3/getActive';
|
||||
import { OrgPreference } from 'types/api/preferences/preference';
|
||||
import { USER_ROLES } from 'types/roles';
|
||||
import { isSignozAdminEmail } from 'utils/app';
|
||||
import {
|
||||
routePermission,
|
||||
routeWithInitialAuthZSupport,
|
||||
@@ -76,7 +75,7 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
|
||||
const users = usersData?.data || [];
|
||||
|
||||
const remainingUsers = (Array.isArray(users) ? users : []).filter(
|
||||
(u) => !isSignozAdminEmail(u.email),
|
||||
(user) => user.email !== 'admin@signoz.cloud',
|
||||
);
|
||||
|
||||
return remainingUsers.length === 1;
|
||||
@@ -204,7 +203,6 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
|
||||
const isFirstUser = checkFirstTimeUser();
|
||||
if (
|
||||
isFirstUser &&
|
||||
!isSignozAdminEmail(user.email) &&
|
||||
!isOnboardingComplete &&
|
||||
!ROUTES_NOT_TO_BE_OVERRIDEN.includes(pathname) &&
|
||||
pathname !== ROUTES.ONBOARDING
|
||||
|
||||
@@ -2,7 +2,6 @@ import { ReactElement } from 'react';
|
||||
import { QueryClient, QueryClientProvider } from 'react-query';
|
||||
import { MemoryRouter, Route, Switch, useLocation } from 'react-router-dom';
|
||||
import { act, render, screen, waitFor } from '@testing-library/react';
|
||||
import { SIGNOZ_ADMIN_EMAIL } from 'constants/app';
|
||||
import { LOCALSTORAGE } from 'constants/localStorage';
|
||||
import { ORG_PREFERENCES } from 'constants/orgPreferences';
|
||||
import ROUTES from 'constants/routes';
|
||||
@@ -1090,33 +1089,6 @@ describe('PrivateRoute', () => {
|
||||
await assertRedirectsTo(ROUTES.ONBOARDING);
|
||||
});
|
||||
|
||||
it('should not redirect the signoz admin to onboarding while the first user has not completed it', async () => {
|
||||
mockUsersData = [
|
||||
{ email: 'test@example.com' },
|
||||
{ email: SIGNOZ_ADMIN_EMAIL },
|
||||
];
|
||||
|
||||
renderPrivateRoute({
|
||||
initialRoute: ROUTES.HOME,
|
||||
appContext: {
|
||||
isLoggedIn: true,
|
||||
isFetchingOrgPreferences: false,
|
||||
orgPreferences: createMockOrgPreferences(false),
|
||||
user: createMockUser({
|
||||
role: USER_ROLES.ADMIN as ROLES,
|
||||
email: SIGNOZ_ADMIN_EMAIL,
|
||||
}),
|
||||
},
|
||||
isCloudUser: true,
|
||||
});
|
||||
|
||||
await act(async () => {
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
assertStaysOnRoute(ROUTES.HOME);
|
||||
});
|
||||
|
||||
it('should not redirect to onboarding when org preferences are still fetching', () => {
|
||||
renderPrivateRoute({
|
||||
initialRoute: ROUTES.HOME,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { useCallback, useMemo, useState } from 'react';
|
||||
import { Row, Select, Spin } from 'antd';
|
||||
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { navigate } from 'lib/router/navigation';
|
||||
import { useAppLocation } from 'lib/router/useAppLocation';
|
||||
import {
|
||||
@@ -18,7 +17,6 @@ export interface SelectOptionConfig {
|
||||
placeholder: string;
|
||||
queryParam: QueryParams;
|
||||
filterType: string | string[];
|
||||
fieldContext?: TelemetrytypesFieldContextDTO;
|
||||
shouldSetQueryParams?: boolean;
|
||||
onChange?: (value: string | string[]) => void;
|
||||
values?: string | string[];
|
||||
@@ -29,16 +27,13 @@ export function FilterSelect({
|
||||
placeholder,
|
||||
queryParam,
|
||||
filterType,
|
||||
fieldContext,
|
||||
values,
|
||||
shouldSetQueryParams,
|
||||
onChange,
|
||||
isMultiple,
|
||||
}: SelectOptionConfig): JSX.Element {
|
||||
const { handleSearch, isFetching, options } = useCeleryFilterOptions(
|
||||
filterType,
|
||||
fieldContext,
|
||||
);
|
||||
const { handleSearch, isFetching, options } =
|
||||
useCeleryFilterOptions(filterType);
|
||||
|
||||
const urlQuery = useUrlQuery();
|
||||
const location = useAppLocation();
|
||||
@@ -123,7 +118,6 @@ export function FilterSelect({
|
||||
}
|
||||
|
||||
FilterSelect.defaultProps = {
|
||||
fieldContext: undefined,
|
||||
shouldSetQueryParams: true,
|
||||
onChange: (): void => {},
|
||||
values: [],
|
||||
@@ -135,14 +129,12 @@ function CeleryOverviewConfigOptions(): JSX.Element {
|
||||
{
|
||||
placeholder: 'Service Name',
|
||||
queryParam: QueryParams.service,
|
||||
filterType: 'service.name',
|
||||
fieldContext: TelemetrytypesFieldContextDTO.resource,
|
||||
filterType: 'serviceName',
|
||||
},
|
||||
{
|
||||
placeholder: 'Span Name',
|
||||
queryParam: QueryParams.spanName,
|
||||
filterType: 'name',
|
||||
fieldContext: TelemetrytypesFieldContextDTO.span,
|
||||
},
|
||||
{
|
||||
placeholder: 'Msg System',
|
||||
@@ -170,7 +162,6 @@ function CeleryOverviewConfigOptions(): JSX.Element {
|
||||
placeholder={config.placeholder}
|
||||
queryParam={config.queryParam}
|
||||
filterType={config.filterType}
|
||||
fieldContext={config.fieldContext}
|
||||
/>
|
||||
))}
|
||||
</Row>
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import type { GetFieldsValues200 } from 'api/generated/services/sigNoz.schemas';
|
||||
import { rest, server } from 'mocks-server/server';
|
||||
import { render, waitFor } from 'tests/test-utils';
|
||||
|
||||
import CeleryOverviewConfigOptions from '../CeleryOverviewConfigOptions';
|
||||
|
||||
const emptyValues: GetFieldsValues200 = {
|
||||
status: 'success',
|
||||
data: { complete: true, values: { stringValues: [] } },
|
||||
};
|
||||
|
||||
describe('CeleryOverviewConfigOptions', () => {
|
||||
it('scopes the service and span name lookups to their field context', async () => {
|
||||
const contextByKey = new Map<string, string | null>();
|
||||
server.use(
|
||||
rest.get('*/api/v1/fields/values', (req, res, ctx) => {
|
||||
contextByKey.set(
|
||||
req.url.searchParams.get('name') ?? '',
|
||||
req.url.searchParams.get('fieldContext'),
|
||||
);
|
||||
return res(ctx.status(200), ctx.json(emptyValues));
|
||||
}),
|
||||
);
|
||||
|
||||
render(<CeleryOverviewConfigOptions />);
|
||||
|
||||
await waitFor(() => expect(contextByKey.size).toBe(6));
|
||||
expect(Object.fromEntries(contextByKey)).toStrictEqual({
|
||||
'service.name': 'resource',
|
||||
name: 'span',
|
||||
'messaging.system': null,
|
||||
'messaging.destination.name': null,
|
||||
'messaging.destination': null,
|
||||
kind_string: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,21 +1,23 @@
|
||||
import { useQuery } from 'react-query';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { useSelector } from 'react-redux';
|
||||
import type { DefaultOptionType } from 'antd/es/select';
|
||||
import type { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { getFieldValueSuggestions } from 'api/querySuggestions/getFieldValueSuggestions';
|
||||
import { FIELD_API_CACHE_TIME } from 'constants/queryCacheTime';
|
||||
import { getAttributesValues } from 'api/queryBuilder/getAttributesValues';
|
||||
import { DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY } from 'constants/queryBuilder';
|
||||
import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
|
||||
import { toFieldValues } from 'hooks/querySuggestions/useFieldValuesSuggestion';
|
||||
import { AppState } from 'store/reducers';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { DATA_SOURCE_TO_SIGNAL, DataSource } from 'types/common/queryBuilder';
|
||||
import { getValuesForDataType } from 'utils/fieldValues';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
import { GlobalReducer } from 'types/reducer/globalTime';
|
||||
|
||||
export interface Filters {
|
||||
searchText: string;
|
||||
attributeKey: string | string[];
|
||||
aggregateOperator?: string;
|
||||
dataSource?: DataSource;
|
||||
metricName?: string;
|
||||
aggregateAttribute?: string;
|
||||
filterAttributeKeyDataType?: DataTypes;
|
||||
fieldContext?: TelemetrytypesFieldContextDTO;
|
||||
tagType?: string;
|
||||
}
|
||||
|
||||
export interface GetAllFiltersResponse {
|
||||
@@ -27,41 +29,54 @@ export function useGetAllFilters(props: Filters): GetAllFiltersResponse {
|
||||
const {
|
||||
searchText,
|
||||
attributeKey,
|
||||
aggregateOperator,
|
||||
dataSource,
|
||||
metricName,
|
||||
aggregateAttribute,
|
||||
filterAttributeKeyDataType,
|
||||
fieldContext,
|
||||
tagType,
|
||||
} = props;
|
||||
|
||||
const { minTime, maxTime } = useSelector<AppState, GlobalReducer>(
|
||||
(state) => state.globalTime,
|
||||
);
|
||||
|
||||
const { data, isLoading } = useQuery(
|
||||
[
|
||||
REACT_QUERY_KEY.GET_ATTRIBUTE_VALUES,
|
||||
dataSource,
|
||||
metricName,
|
||||
attributeKey,
|
||||
fieldContext,
|
||||
searchText,
|
||||
minTime,
|
||||
maxTime,
|
||||
],
|
||||
async () => {
|
||||
const keys = Array.isArray(attributeKey) ? attributeKey : [attributeKey];
|
||||
|
||||
const responses = await Promise.all(
|
||||
keys.map((key) =>
|
||||
getFieldValueSuggestions({
|
||||
signal: DATA_SOURCE_TO_SIGNAL[dataSource || DataSource.TRACES],
|
||||
name: key,
|
||||
fieldContext,
|
||||
getAttributesValues({
|
||||
aggregateOperator: aggregateOperator || 'noop',
|
||||
dataSource: dataSource || DataSource.TRACES,
|
||||
aggregateAttribute: aggregateAttribute || '',
|
||||
attributeKey: key,
|
||||
searchText: searchText ?? '',
|
||||
metricName,
|
||||
filterAttributeKeyDataType:
|
||||
filterAttributeKeyDataType || DataTypes.String,
|
||||
tagType: tagType || 'tag',
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const uniqueValues = [
|
||||
...new Set(
|
||||
responses.flatMap((response) =>
|
||||
getValuesForDataType(toFieldValues(response), filterAttributeKeyDataType),
|
||||
),
|
||||
responses.flatMap(({ payload }) => {
|
||||
if (!payload) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const dataType = filterAttributeKeyDataType || DataTypes.String;
|
||||
const key = DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY[dataType];
|
||||
return key ? payload[key] || [] : [];
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
@@ -70,7 +85,6 @@ export function useGetAllFilters(props: Filters): GetAllFiltersResponse {
|
||||
value: val,
|
||||
}));
|
||||
},
|
||||
{ staleTime: FIELD_API_CACHE_TIME },
|
||||
);
|
||||
|
||||
return { options: data ?? [], isFetching: isLoading };
|
||||
|
||||
@@ -8,9 +8,12 @@ import { CardContainer } from 'container/WidgetCard/styles';
|
||||
import { useIsDarkMode } from 'hooks/useDarkMode';
|
||||
import { ChevronDown, ChevronUp } from '@signozhq/icons';
|
||||
import { AppState } from 'store/reducers';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
import { GlobalReducer } from 'types/reducer/globalTime';
|
||||
|
||||
import { CaptureDataProps } from '../CeleryTaskDetail/CeleryTaskDetail';
|
||||
import { useCeleryFilterOptions } from '../useCeleryFilterOptions';
|
||||
import CeleryTaskBar from './CeleryTaskBar';
|
||||
import CeleryTaskGraph from './CeleryTaskGraph';
|
||||
import {
|
||||
@@ -20,7 +23,6 @@ import {
|
||||
celeryTasksByWorkerWidgetData,
|
||||
} from './CeleryTaskGraphUtils';
|
||||
import CeleryTaskLatencyGraph from './CeleryTaskLatencyGraph';
|
||||
import { useCeleryWorkerCount } from './useCeleryWorkerCount';
|
||||
|
||||
import './CeleryTaskGraph.style.scss';
|
||||
|
||||
@@ -69,7 +71,14 @@ export default function CeleryTaskGraphGrid({
|
||||
'Latency by worker',
|
||||
];
|
||||
|
||||
const { data: workerCount } = useCeleryWorkerCount();
|
||||
const { options } = useCeleryFilterOptions(
|
||||
'worker',
|
||||
'rate',
|
||||
DataSource.METRICS,
|
||||
'flower_task_runtime_seconds_sum',
|
||||
DataTypes.String,
|
||||
'tag',
|
||||
);
|
||||
const isDarkMode = useIsDarkMode();
|
||||
|
||||
const [collapsedSections, setCollapsedSections] = useState<{
|
||||
@@ -142,11 +151,8 @@ export default function CeleryTaskGraphGrid({
|
||||
</Typography.Text>
|
||||
</div>
|
||||
<div className="worker-count-text-container">
|
||||
<Typography.Text
|
||||
className="celery-task-graph-worker-count-text"
|
||||
data-testid="celery-worker-count"
|
||||
>
|
||||
{workerCount ?? '-'}
|
||||
<Typography.Text className="celery-task-graph-worker-count-text">
|
||||
{options.filter((option) => option.value).length}
|
||||
</Typography.Text>
|
||||
</div>
|
||||
</Card>
|
||||
|
||||
@@ -1,12 +1,3 @@
|
||||
import {
|
||||
MetrictypesSpaceAggregationDTO,
|
||||
MetrictypesTimeAggregationDTO,
|
||||
Querybuildertypesv5QueryBuilderQueryGithubComSigNozSignozPkgTypesQuerybuildertypesQuerybuildertypesv5MetricAggregationDTOSignal as MetricSignal,
|
||||
Querybuildertypesv5QueryEnvelopeBuilderDTOType,
|
||||
type Querybuildertypesv5QueryRangeRequestDTO,
|
||||
Querybuildertypesv5ReduceToDTO,
|
||||
Querybuildertypesv5RequestTypeDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import { getWidgetQueryBuilder } from 'container/MetricsApplication/MetricsApplication.factory';
|
||||
import { getWidgetQuery } from 'pages/MessagingQueues/MQDetails/MetricPage/MetricPageUtil';
|
||||
@@ -1051,34 +1042,3 @@ export const celeryRetryStateCountWidgetData = getWidgetQueryBuilder(
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
// One row per worker that reported in the range; the card counts the rows.
|
||||
export const getCeleryWorkerCountRequest = (
|
||||
startTime: number,
|
||||
endTime: number,
|
||||
): Querybuildertypesv5QueryRangeRequestDTO => ({
|
||||
schemaVersion: 'v1',
|
||||
start: Math.floor(startTime / 1e6),
|
||||
end: Math.floor(endTime / 1e6),
|
||||
requestType: Querybuildertypesv5RequestTypeDTO.scalar,
|
||||
compositeQuery: {
|
||||
queries: [
|
||||
{
|
||||
type: Querybuildertypesv5QueryEnvelopeBuilderDTOType.builder_query,
|
||||
spec: {
|
||||
name: 'A',
|
||||
signal: MetricSignal.metrics,
|
||||
aggregations: [
|
||||
{
|
||||
metricName: 'flower_task_runtime_seconds_sum',
|
||||
timeAggregation: MetrictypesTimeAggregationDTO.count,
|
||||
spaceAggregation: MetrictypesSpaceAggregationDTO.sum,
|
||||
reduceTo: Querybuildertypesv5ReduceToDTO.sum,
|
||||
},
|
||||
],
|
||||
groupBy: [{ name: 'worker' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
import {
|
||||
Querybuildertypesv5ColumnTypeDTO,
|
||||
type Querybuildertypesv5QueryRangeRequestDTO,
|
||||
Querybuildertypesv5RequestTypeDTO,
|
||||
type QueryRangeV5200,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
import { rest, server } from 'mocks-server/server';
|
||||
import store from 'store';
|
||||
import { render, screen, waitFor } from 'tests/test-utils';
|
||||
|
||||
import CeleryTaskGraphGrid from '../CeleryTaskGraphGrid';
|
||||
|
||||
const workerCountResponse = (workers: string[]): QueryRangeV5200 => ({
|
||||
status: 'success',
|
||||
data: {
|
||||
type: Querybuildertypesv5RequestTypeDTO.scalar,
|
||||
data: {
|
||||
results: [
|
||||
{
|
||||
columns: [
|
||||
{
|
||||
name: 'worker',
|
||||
queryName: 'A',
|
||||
columnType: Querybuildertypesv5ColumnTypeDTO.group,
|
||||
},
|
||||
{
|
||||
name: '__result_0',
|
||||
queryName: 'A',
|
||||
aggregationIndex: 0,
|
||||
columnType: Querybuildertypesv5ColumnTypeDTO.aggregation,
|
||||
},
|
||||
],
|
||||
data: workers.map((worker) => [worker, 4]),
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
describe('CeleryTaskGraphGrid worker count', () => {
|
||||
let requests: Querybuildertypesv5QueryRangeRequestDTO[];
|
||||
|
||||
beforeEach(() => {
|
||||
requests = [];
|
||||
server.use(
|
||||
rest.post('*/api/v5/query_range', async (req, res, ctx) => {
|
||||
requests.push(await req.json());
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(workerCountResponse(['worker-a', 'worker-b', ''])),
|
||||
);
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('counts the workers that reported in the selected range', async () => {
|
||||
render(<CeleryTaskGraphGrid onClick={jest.fn()} queryEnabled={false} />);
|
||||
|
||||
await waitFor(() =>
|
||||
expect(screen.getByTestId('celery-worker-count')).toHaveTextContent('2'),
|
||||
);
|
||||
|
||||
const { minTime, maxTime } = store.getState().globalTime;
|
||||
expect(requests).toStrictEqual([
|
||||
{
|
||||
schemaVersion: 'v1',
|
||||
start: Math.floor(minTime / 1e6),
|
||||
end: Math.floor(maxTime / 1e6),
|
||||
requestType: 'scalar',
|
||||
compositeQuery: {
|
||||
queries: [
|
||||
{
|
||||
type: 'builder_query',
|
||||
spec: {
|
||||
name: 'A',
|
||||
signal: 'metrics',
|
||||
aggregations: [
|
||||
{
|
||||
metricName: 'flower_task_runtime_seconds_sum',
|
||||
timeAggregation: 'count',
|
||||
spaceAggregation: 'sum',
|
||||
reduceTo: 'sum',
|
||||
},
|
||||
],
|
||||
groupBy: [{ name: 'worker' }],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,31 +0,0 @@
|
||||
import { useMemo } from 'react';
|
||||
import { useQuery, UseQueryResult } from 'react-query';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { useSelector } from 'react-redux';
|
||||
import { queryRangeV5 } from 'api/generated/services/querier';
|
||||
import type { QueryRangeV5200 } from 'api/generated/services/sigNoz.schemas';
|
||||
import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
|
||||
import { getScalarResults } from 'pages/DashboardPage/DashboardContainer/queryV5/v5ResponseData';
|
||||
import { AppState } from 'store/reducers';
|
||||
import { GlobalReducer } from 'types/reducer/globalTime';
|
||||
|
||||
import { getCeleryWorkerCountRequest } from './CeleryTaskGraphUtils';
|
||||
|
||||
export function useCeleryWorkerCount(): UseQueryResult<number, Error> {
|
||||
const { minTime, maxTime } = useSelector<AppState, GlobalReducer>(
|
||||
(state) => state.globalTime,
|
||||
);
|
||||
|
||||
const request = useMemo(
|
||||
() => getCeleryWorkerCountRequest(minTime, maxTime),
|
||||
[minTime, maxTime],
|
||||
);
|
||||
|
||||
return useQuery<QueryRangeV5200, Error, number>({
|
||||
queryKey: [REACT_QUERY_KEY.GET_CELERY_WORKER_COUNT, request],
|
||||
queryFn: ({ signal }) => queryRangeV5(request, signal),
|
||||
select: (response) =>
|
||||
(getScalarResults(response)[0]?.data ?? []).filter(([worker]) => worker)
|
||||
.length,
|
||||
});
|
||||
}
|
||||
@@ -1,13 +1,18 @@
|
||||
import { useState } from 'react';
|
||||
import type { DefaultOptionType } from 'antd/es/select';
|
||||
import type { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import useDebouncedFn from 'hooks/useDebouncedFunction';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
|
||||
import { useGetAllFilters } from './CeleryTaskConfigOptions/useGetCeleryFilters';
|
||||
|
||||
export const useCeleryFilterOptions = (
|
||||
type: string | string[],
|
||||
fieldContext?: TelemetrytypesFieldContextDTO,
|
||||
aggregateOperator?: string,
|
||||
dataSource?: DataSource,
|
||||
aggregateAttribute?: string,
|
||||
filterAttributeKeyDataType?: DataTypes,
|
||||
tagType?: string,
|
||||
): {
|
||||
searchText: string;
|
||||
handleSearch: (value: string) => void;
|
||||
@@ -18,7 +23,11 @@ export const useCeleryFilterOptions = (
|
||||
const { isFetching, options } = useGetAllFilters({
|
||||
attributeKey: type,
|
||||
searchText,
|
||||
fieldContext,
|
||||
aggregateOperator,
|
||||
dataSource,
|
||||
aggregateAttribute,
|
||||
filterAttributeKeyDataType,
|
||||
tagType,
|
||||
});
|
||||
const handleDebouncedSearch = useDebouncedFn((searchText): void => {
|
||||
setSearchText(searchText as string);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { TelemetryFieldKey } from 'types/api/v5/queryRange';
|
||||
import { fieldContextToAttributeType } from 'utils/fieldContext';
|
||||
import { fieldDataTypeToDataType } from 'utils/fieldDataType';
|
||||
|
||||
import { FiltersType, IQuickFiltersConfig } from './types';
|
||||
@@ -14,6 +14,19 @@ const FILTER_TYPE_MAP: Record<string, FiltersType> = {
|
||||
duration_nano: FiltersType.DURATION,
|
||||
};
|
||||
|
||||
// The map below exists only for the old v3 attribute-values fetch
|
||||
// (useCheckboxFilterValues), the sole reader of attributeKey.dataType/type.
|
||||
// Once the values fetch moves to fields/values, remove this and reduce
|
||||
// attributeKey to { id, key }.
|
||||
|
||||
const FIELD_CONTEXT_TO_ATTRIBUTE_TYPE: Record<string, string> = {
|
||||
[TelemetrytypesFieldContextDTO.attribute]: 'tag',
|
||||
[TelemetrytypesFieldContextDTO.resource]: 'resource',
|
||||
};
|
||||
|
||||
const mapFieldContext = (fieldContext?: string): string =>
|
||||
(fieldContext && FIELD_CONTEXT_TO_ATTRIBUTE_TYPE[fieldContext]) || '';
|
||||
|
||||
const getFilterName = (str: string): string => {
|
||||
if (FILTER_TITLE_MAP[str]) {
|
||||
return FILTER_TITLE_MAP[str];
|
||||
@@ -48,15 +61,11 @@ export const getFilterConfig = (
|
||||
({
|
||||
type: getFilterType(att),
|
||||
title: getFilterName(att.name),
|
||||
// The type mapping below exists only for the old v3 attribute-values fetch
|
||||
// (useCheckboxFilterValues), the sole reader of attributeKey.dataType/type.
|
||||
// Once the values fetch moves to fields/values, remove this and reduce
|
||||
// attributeKey to { id, key }.
|
||||
attributeKey: {
|
||||
id: att.name,
|
||||
key: att.name,
|
||||
dataType: fieldDataTypeToDataType(att.fieldDataType),
|
||||
type: fieldContextToAttributeType(att.fieldContext),
|
||||
type: mapFieldContext(att.fieldContext),
|
||||
},
|
||||
defaultOpen: index < 2,
|
||||
}) as IQuickFiltersConfig,
|
||||
|
||||
@@ -15,5 +15,3 @@ export const DASHBOARD_TIME_IN_DURATION = 'refreshInterval';
|
||||
export const DEFAULT_ENTITY_VERSION = 'v3';
|
||||
export const ENTITY_VERSION_V4 = 'v4';
|
||||
export const ENTITY_VERSION_V5 = 'v5';
|
||||
|
||||
export const SIGNOZ_ADMIN_EMAIL = 'admin@signoz.cloud';
|
||||
|
||||
@@ -109,9 +109,6 @@ export const REACT_QUERY_KEY = {
|
||||
// Field Values Suggestion Query Keys
|
||||
FIELD_VALUES_SUGGESTION: 'FIELD_VALUES_SUGGESTION',
|
||||
|
||||
// Celery Query Keys
|
||||
GET_CELERY_WORKER_COUNT: 'GET_CELERY_WORKER_COUNT',
|
||||
|
||||
// AI Assistant Query Keys
|
||||
AI_ASSISTANT_EMPTY_STATE_CHIPS: 'AI_ASSISTANT_EMPTY_STATE_CHIPS',
|
||||
} as const;
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import { Form } from 'antd';
|
||||
import QueryBuilderSearchV2 from 'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2';
|
||||
import { rest, server } from 'mocks-server/server';
|
||||
import { act, render, waitFor } from 'tests/test-utils';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import type { QueryRangePayload } from 'types/api/metrics/getQueryRange';
|
||||
import type {
|
||||
IBuilderQuery,
|
||||
TagFilter,
|
||||
} from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
import { pipelineFields } from '../../../../config';
|
||||
import FilterInput from '..';
|
||||
|
||||
jest.mock(
|
||||
'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2',
|
||||
() => ({ __esModule: true, default: jest.fn(() => null) }),
|
||||
);
|
||||
|
||||
const filterField = pipelineFields.find((field) => field.name === 'filter');
|
||||
|
||||
const v5Filter: TagFilter = {
|
||||
op: 'AND',
|
||||
items: [
|
||||
{
|
||||
id: 'attribute-pill',
|
||||
key: { key: 'http.method', dataType: DataTypes.String, type: 'attribute' },
|
||||
op: '=',
|
||||
value: 'GET',
|
||||
},
|
||||
{
|
||||
id: 'log-pill',
|
||||
key: { key: 'severity_text', dataType: DataTypes.String, type: 'log' },
|
||||
op: '=',
|
||||
value: 'ERROR',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe('Pipeline FilterInput', () => {
|
||||
it('previews sample logs with the legacy key types the v3 logs builder reads', async () => {
|
||||
const previewFilters: TagFilter[] = [];
|
||||
server.use(
|
||||
rest.post('*/api/v3/query_range', async (req, res, ctx) => {
|
||||
const body: QueryRangePayload = await req.json();
|
||||
const query = body.compositeQuery.builderQueries?.A as IBuilderQuery;
|
||||
if (query.filters) {
|
||||
previewFilters.push(query.filters);
|
||||
}
|
||||
return res(ctx.status(200), ctx.json({ data: { result: [] } }));
|
||||
}),
|
||||
);
|
||||
|
||||
render(<Form>{filterField && <FilterInput fieldData={filterField} />}</Form>);
|
||||
|
||||
const { onChange } = jest.mocked(QueryBuilderSearchV2).mock.calls[0][0];
|
||||
act(() => onChange(v5Filter));
|
||||
|
||||
await waitFor(() => expect(previewFilters).not.toHaveLength(0));
|
||||
expect(
|
||||
previewFilters[previewFilters.length - 1].items.map((item) => [
|
||||
item.key?.key,
|
||||
item.key?.type,
|
||||
]),
|
||||
).toStrictEqual([
|
||||
['http.method', 'tag'],
|
||||
['severity_text', ''],
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,6 @@ import { initialQueryBuilderFormValuesMap } from 'constants/queryBuilder';
|
||||
import QueryBuilderSearchV2 from 'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2';
|
||||
import isEqual from 'lodash-es/isEqual';
|
||||
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { toLegacyLogsFilter } from 'utils/fieldContext';
|
||||
|
||||
import { ProcessorFormField } from '../../../AddNewProcessor/config';
|
||||
import { formValidationRules } from '../../../config';
|
||||
@@ -23,10 +22,9 @@ function TagFilterInput({
|
||||
}
|
||||
|
||||
const onQueryChange = (newValue: TagFilter): void => {
|
||||
const legacyValue = toLegacyLogsFilter(newValue);
|
||||
// Avoid unnecessary onChange calls
|
||||
if (!isEqual(legacyValue, query.filters)) {
|
||||
onChange(legacyValue);
|
||||
if (!isEqual(newValue, query.filters)) {
|
||||
onChange(newValue);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
userEvent,
|
||||
waitFor,
|
||||
} from 'tests/test-utils';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
|
||||
import { pipelineApiResponseMockData } from '../mocks/pipeline';
|
||||
import PipelineListsView from '../PipelineListsView';
|
||||
@@ -84,7 +85,7 @@ jest.mock('providers/preferences/sync/usePreferenceSync', () => ({
|
||||
}));
|
||||
|
||||
const BASE_URL = ENVIRONMENT.baseURL;
|
||||
const fieldKeysURL = `${BASE_URL}/api/v1/fields/keys`;
|
||||
const attributeKeysURL = `${BASE_URL}/api/v3/filter_suggestions`;
|
||||
|
||||
describe('PipelinePage container test', () => {
|
||||
beforeAll(() => {
|
||||
@@ -338,41 +339,42 @@ describe('PipelinePage container test', () => {
|
||||
expect(screen.queryAllByText('source = nginx')).toHaveLength(2);
|
||||
|
||||
server.use(
|
||||
rest.get(fieldKeysURL, (_req, res, ctx) =>
|
||||
rest.get(attributeKeysURL, (_req, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
status: 'success',
|
||||
data: {
|
||||
complete: true,
|
||||
keys: {
|
||||
otelServiceName: [
|
||||
{
|
||||
name: 'otelServiceName',
|
||||
fieldContext: 'attribute',
|
||||
fieldDataType: 'string',
|
||||
},
|
||||
],
|
||||
'service.name': [
|
||||
{
|
||||
name: 'service.name',
|
||||
fieldContext: 'resource',
|
||||
fieldDataType: 'string',
|
||||
},
|
||||
{
|
||||
name: 'service.name',
|
||||
fieldContext: 'attribute',
|
||||
fieldDataType: 'string',
|
||||
},
|
||||
],
|
||||
'service.instance.id': [
|
||||
{
|
||||
name: 'service.instance.id',
|
||||
fieldContext: 'resource',
|
||||
fieldDataType: 'string',
|
||||
},
|
||||
],
|
||||
},
|
||||
attributes: [
|
||||
{
|
||||
key: 'otelServiceName',
|
||||
dataType: DataTypes.String,
|
||||
type: 'tag',
|
||||
isColumn: false,
|
||||
isJSON: false,
|
||||
},
|
||||
{
|
||||
key: 'service.name',
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
isColumn: false,
|
||||
isJSON: false,
|
||||
},
|
||||
{
|
||||
key: 'service.instance.id',
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
isColumn: false,
|
||||
isJSON: false,
|
||||
},
|
||||
{
|
||||
key: 'service.name',
|
||||
dataType: DataTypes.String,
|
||||
type: 'tag',
|
||||
isColumn: false,
|
||||
isJSON: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
),
|
||||
|
||||
@@ -10,9 +10,9 @@ import {
|
||||
} from 'react';
|
||||
import { type RefSelectProps, Select, Spin } from 'antd';
|
||||
import { Pill } from '@signozhq/ui/pill';
|
||||
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import cx from 'classnames';
|
||||
import {
|
||||
DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY,
|
||||
OperatorConfigKeys,
|
||||
OPERATORS,
|
||||
QUERY_BUILDER_OPERATORS_BY_TYPES,
|
||||
@@ -22,14 +22,23 @@ import { DEBOUNCE_DELAY } from 'constants/queryBuilderFilterConfig';
|
||||
import { LogsExplorerShortcuts } from 'constants/shortcuts/logsExplorerShortcuts';
|
||||
import { useDynamicVariableSuggestions } from 'hooks/dashboard/useDynamicVariableSuggestions';
|
||||
import { useKeyboardHotkeys } from 'hooks/hotkeys/useKeyboardHotkeys';
|
||||
import { useGetAggregateKeys } from 'hooks/queryBuilder/useGetAggregateKeys';
|
||||
import { useGetAggregateValues } from 'hooks/queryBuilder/useGetAggregateValues';
|
||||
import { useGetAttributeSuggestions } from 'hooks/queryBuilder/useGetAttributeSuggestions';
|
||||
import { validationMapper } from 'hooks/queryBuilder/useIsValidTag';
|
||||
import { operatorTypeMapper } from 'hooks/queryBuilder/useOperatorType';
|
||||
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
|
||||
import { useRunQueryWithFilterItems } from 'hooks/queryBuilder/useRunQueryWithFilterItems';
|
||||
import { useFieldKeysSuggestion } from 'hooks/querySuggestions/useFieldKeysSuggestion';
|
||||
import { useFieldValuesSuggestion } from 'hooks/querySuggestions/useFieldValuesSuggestion';
|
||||
import useDebounceValue from 'hooks/useDebounce';
|
||||
import { isArray, isEmpty, isEqual, isObject, isUndefined } from 'lodash-es';
|
||||
import {
|
||||
cloneDeep,
|
||||
isArray,
|
||||
isEmpty,
|
||||
isEqual,
|
||||
isObject,
|
||||
isUndefined,
|
||||
unset,
|
||||
} from 'lodash-es';
|
||||
import { ChevronDown, ChevronUp } from '@signozhq/icons';
|
||||
import {
|
||||
BaseAutocompleteData,
|
||||
@@ -39,13 +48,7 @@ import {
|
||||
IBuilderQuery,
|
||||
TagFilter,
|
||||
} from 'types/api/queryBuilder/queryBuilderData';
|
||||
import {
|
||||
DATA_SOURCE_TO_SIGNAL,
|
||||
DataSource,
|
||||
QUERY_SOURCE_TO_FIELD_SOURCE,
|
||||
} from 'types/common/queryBuilder';
|
||||
import { attributeTypeToFieldContext } from 'utils/fieldContext';
|
||||
import { getValuesForDataType } from 'utils/fieldValues';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
import { useSelectPopupContainer } from 'utils/selectPopupContainer';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
|
||||
@@ -58,8 +61,7 @@ import {
|
||||
getTagToken,
|
||||
isInNInOperator,
|
||||
} from './utils';
|
||||
import { toAttributeKey } from '../GroupByFilter/utils';
|
||||
import { filterByOperatorConfig, getMetricNameForQuery } from '../utils';
|
||||
import { filterByOperatorConfig } from '../utils';
|
||||
import QueryBuilderSearchDropdown from './QueryBuilderSearchDropdown';
|
||||
import SpanScopeSelector from './SpanScopeSelector';
|
||||
import Suggestions from './Suggestions';
|
||||
@@ -73,8 +75,6 @@ export interface ITag {
|
||||
value: (string | number | boolean)[] | string | number | boolean;
|
||||
}
|
||||
|
||||
const SPAN_SEARCH_SCOPE_KEYS = ['isRoot', 'isEntryPoint'];
|
||||
|
||||
interface CustomTagProps {
|
||||
label: React.ReactNode;
|
||||
value: string;
|
||||
@@ -186,14 +186,69 @@ function QueryBuilderSearchV2(
|
||||
[query.dataSource],
|
||||
);
|
||||
|
||||
const debouncedKeySearchText = useDebounceValue(
|
||||
searchValue?.split(' ')[0] ?? '',
|
||||
DEBOUNCE_DELAY,
|
||||
const memoizedSearchParams = useMemo(
|
||||
() => [
|
||||
searchValue,
|
||||
query.dataSource,
|
||||
query.aggregateOperator,
|
||||
query.aggregateAttribute?.key,
|
||||
],
|
||||
[
|
||||
searchValue,
|
||||
query.dataSource,
|
||||
query.aggregateOperator,
|
||||
query.aggregateAttribute?.key,
|
||||
],
|
||||
);
|
||||
|
||||
const valueSearchText = isArray(currentFilterItem?.value)
|
||||
? String(currentFilterItem?.value?.[currentFilterItem.value.length - 1] ?? '')
|
||||
: currentFilterItem?.value?.toString() || '';
|
||||
const queryFiltersWithoutId = useMemo(
|
||||
() => ({
|
||||
...query.filters,
|
||||
items: query.filters?.items.map((item) => {
|
||||
const filterWithoutId = cloneDeep(item);
|
||||
unset(filterWithoutId, 'id');
|
||||
return filterWithoutId;
|
||||
}),
|
||||
}),
|
||||
[query.filters],
|
||||
);
|
||||
|
||||
const memoizedSuggestionsParams = useMemo(
|
||||
() => [searchValue, query.dataSource, queryFiltersWithoutId],
|
||||
[query.dataSource, queryFiltersWithoutId, searchValue],
|
||||
);
|
||||
|
||||
const memoizedValueParams = useMemo(
|
||||
() => [
|
||||
query.aggregateOperator,
|
||||
query.dataSource,
|
||||
query.aggregateAttribute?.key,
|
||||
currentFilterItem?.key?.key || '',
|
||||
currentFilterItem?.key?.dataType,
|
||||
currentFilterItem?.key?.type ?? '',
|
||||
isArray(currentFilterItem?.value)
|
||||
? currentFilterItem?.value?.[currentFilterItem.value.length - 1]
|
||||
: currentFilterItem?.value,
|
||||
],
|
||||
[
|
||||
query.aggregateOperator,
|
||||
query.dataSource,
|
||||
query.aggregateAttribute?.key,
|
||||
currentFilterItem?.key?.key,
|
||||
currentFilterItem?.key?.dataType,
|
||||
currentFilterItem?.key?.type,
|
||||
currentFilterItem?.value,
|
||||
],
|
||||
);
|
||||
|
||||
const searchParams = useDebounceValue(memoizedSearchParams, DEBOUNCE_DELAY);
|
||||
|
||||
const valueParams = useDebounceValue(memoizedValueParams, DEBOUNCE_DELAY);
|
||||
|
||||
const suggestionsParams = useDebounceValue(
|
||||
memoizedSuggestionsParams,
|
||||
DEBOUNCE_DELAY,
|
||||
);
|
||||
|
||||
const isQueryEnabled = useMemo(() => {
|
||||
if (currentState === DropdownState.ATTRIBUTE_KEY) {
|
||||
@@ -207,76 +262,53 @@ function QueryBuilderSearchV2(
|
||||
|
||||
const dashboardDynamicVariables = useDynamicVariableSuggestions();
|
||||
|
||||
const signal = DATA_SOURCE_TO_SIGNAL[query.dataSource];
|
||||
const source = QUERY_SOURCE_TO_FIELD_SOURCE[query.source ?? ''];
|
||||
const metricName = getMetricNameForQuery(query);
|
||||
|
||||
const { data: fieldKeys, isFetching } = useFieldKeysSuggestion(
|
||||
const { data, isFetching } = useGetAggregateKeys(
|
||||
{
|
||||
signal,
|
||||
source,
|
||||
metricName,
|
||||
fieldContext: attributeTypeToFieldContext(query.aggregateAttribute?.type),
|
||||
searchText: debouncedKeySearchText,
|
||||
searchText: searchValue?.split(' ')[0],
|
||||
dataSource: query.dataSource,
|
||||
aggregateOperator: query.aggregateOperator || '',
|
||||
aggregateAttribute: query.aggregateAttribute?.key || '',
|
||||
tagType: query.aggregateAttribute?.type ?? null,
|
||||
},
|
||||
query.builderQueryType,
|
||||
{
|
||||
enabled: isQueryEnabled && !hardcodedAttributeKeys,
|
||||
keepPreviousData: true,
|
||||
queryKey: [searchParams],
|
||||
enabled: isQueryEnabled && !isLogsDataSource && !hardcodedAttributeKeys,
|
||||
},
|
||||
);
|
||||
|
||||
// The legacy traces builder checks `exists` on the column only for keys marked as columns.
|
||||
// Body JSON keys come back as bare paths, which the legacy builders read as top-level fields,
|
||||
// and the span search scope selectors are not columns the legacy filter builder can read.
|
||||
const attributeKeys = useMemo(
|
||||
() =>
|
||||
(fieldKeys ?? [])
|
||||
.filter(
|
||||
(fieldKey) =>
|
||||
fieldKey.fieldContext !== TelemetrytypesFieldContextDTO.body &&
|
||||
!(
|
||||
fieldKey.fieldContext === TelemetrytypesFieldContextDTO.span &&
|
||||
SPAN_SEARCH_SCOPE_KEYS.includes(fieldKey.name)
|
||||
),
|
||||
)
|
||||
.map((fieldKey) => ({
|
||||
...toAttributeKey(fieldKey),
|
||||
isColumn:
|
||||
fieldKey.fieldContext === TelemetrytypesFieldContextDTO.span ||
|
||||
fieldKey.fieldContext === TelemetrytypesFieldContextDTO.log,
|
||||
})),
|
||||
[fieldKeys],
|
||||
);
|
||||
const { data: suggestionsData, isFetching: isFetchingSuggestions } =
|
||||
useGetAttributeSuggestions(
|
||||
{
|
||||
searchText: searchValue?.split(' ')[0],
|
||||
dataSource: query.dataSource,
|
||||
filters: query.filters || { items: [], op: 'AND' },
|
||||
},
|
||||
{
|
||||
queryKey: [suggestionsParams],
|
||||
enabled: isQueryEnabled && isLogsDataSource,
|
||||
},
|
||||
);
|
||||
|
||||
const fieldValuesConfig = useMemo(
|
||||
() => ({
|
||||
signal,
|
||||
source,
|
||||
metricName,
|
||||
name: currentFilterItem?.key?.key || '',
|
||||
fieldContext: attributeTypeToFieldContext(currentFilterItem?.key?.type),
|
||||
searchText: valueSearchText,
|
||||
}),
|
||||
[
|
||||
signal,
|
||||
source,
|
||||
metricName,
|
||||
currentFilterItem?.key?.key,
|
||||
currentFilterItem?.key?.type,
|
||||
valueSearchText,
|
||||
],
|
||||
);
|
||||
|
||||
const debouncedFieldValuesConfig = useDebounceValue(
|
||||
fieldValuesConfig,
|
||||
DEBOUNCE_DELAY,
|
||||
);
|
||||
|
||||
const { data: fieldValues, isFetching: isFetchingAttributeValues } =
|
||||
useFieldValuesSuggestion(debouncedFieldValuesConfig, query.builderQueryType, {
|
||||
enabled: currentState === DropdownState.ATTRIBUTE_VALUE,
|
||||
});
|
||||
const { data: attributeValues, isFetching: isFetchingAttributeValues } =
|
||||
useGetAggregateValues(
|
||||
{
|
||||
aggregateOperator: query.aggregateOperator || '',
|
||||
dataSource: query.dataSource,
|
||||
aggregateAttribute: query.aggregateAttribute?.key || '',
|
||||
attributeKey: currentFilterItem?.key?.key || '',
|
||||
filterAttributeKeyDataType:
|
||||
currentFilterItem?.key?.dataType ?? DataTypes.EMPTY,
|
||||
tagType: currentFilterItem?.key?.type ?? '',
|
||||
searchText: isArray(currentFilterItem?.value)
|
||||
? String(currentFilterItem?.value?.[currentFilterItem.value.length - 1]) ||
|
||||
''
|
||||
: currentFilterItem?.value?.toString() || '',
|
||||
},
|
||||
{
|
||||
enabled: currentState === DropdownState.ATTRIBUTE_VALUE,
|
||||
queryKey: [valueParams],
|
||||
},
|
||||
);
|
||||
|
||||
const handleDropdownSelect = useCallback(
|
||||
(value: string) => {
|
||||
@@ -501,14 +533,16 @@ function QueryBuilderSearchV2(
|
||||
if (tagKey && isUndefined(currentFilterItem?.key)) {
|
||||
let currentRunningAttributeKey;
|
||||
const isSuggestedKeyInAutocomplete =
|
||||
isLogsDataSource &&
|
||||
attributeKeys.some((value) => value.key === tagKey.split(' ')[0]);
|
||||
|
||||
if (isSuggestedKeyInAutocomplete) {
|
||||
const allAttributesMatchingTheKey = attributeKeys.filter(
|
||||
suggestionsData?.payload?.attributes?.some(
|
||||
(value) => value.key === tagKey.split(' ')[0],
|
||||
);
|
||||
|
||||
if (isSuggestedKeyInAutocomplete) {
|
||||
const allAttributesMatchingTheKey =
|
||||
suggestionsData?.payload?.attributes?.filter(
|
||||
(value) => value.key === tagKey.split(' ')[0],
|
||||
) || [];
|
||||
|
||||
if (allAttributesMatchingTheKey?.length === 1) {
|
||||
[currentRunningAttributeKey] = allAttributesMatchingTheKey;
|
||||
}
|
||||
@@ -614,10 +648,9 @@ function QueryBuilderSearchV2(
|
||||
currentFilterItem,
|
||||
currentFilterItem?.key,
|
||||
currentFilterItem?.op,
|
||||
attributeKeys,
|
||||
isLogsDataSource,
|
||||
suggestionsData?.payload?.attributes,
|
||||
searchValue,
|
||||
isFetching,
|
||||
isFetchingSuggestions,
|
||||
currentState,
|
||||
]);
|
||||
|
||||
@@ -629,7 +662,9 @@ function QueryBuilderSearchV2(
|
||||
// add the user typed option in the dropdown to select that and move ahead irrespective of the matches and all
|
||||
setDropdownOptions([
|
||||
...(!isEmpty(tagKey) &&
|
||||
!attributeKeys.some((val) => isEqual(val.key, tagKey))
|
||||
!suggestionsData?.payload?.attributes?.some((val) =>
|
||||
isEqual(val.key, tagKey),
|
||||
)
|
||||
? [
|
||||
{
|
||||
label: tagKey,
|
||||
@@ -641,10 +676,10 @@ function QueryBuilderSearchV2(
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...attributeKeys.map((key) => ({
|
||||
...(suggestionsData?.payload?.attributes?.map((key) => ({
|
||||
label: key.key,
|
||||
value: key,
|
||||
})),
|
||||
})) || []),
|
||||
]);
|
||||
} else if (hardcodedAttributeKeys) {
|
||||
const filteredKeys = hardcodedAttributeKeys.filter((key) =>
|
||||
@@ -662,7 +697,7 @@ function QueryBuilderSearchV2(
|
||||
setDropdownOptions([
|
||||
// Add user typed option if it doesn't exist in the payload
|
||||
...(tagKey.trim().length > 0 &&
|
||||
!attributeKeys.some((val) => val.key === tagKey)
|
||||
!data?.payload?.attributeKeys?.some((val) => val.key === tagKey)
|
||||
? [
|
||||
{
|
||||
label: tagKey,
|
||||
@@ -675,10 +710,10 @@ function QueryBuilderSearchV2(
|
||||
]
|
||||
: []),
|
||||
// Map existing attribute keys from payload
|
||||
...attributeKeys.map((key) => ({
|
||||
...(data?.payload?.attributeKeys?.map((key) => ({
|
||||
label: key.key,
|
||||
value: key,
|
||||
})),
|
||||
})) || []),
|
||||
]);
|
||||
}
|
||||
}
|
||||
@@ -743,10 +778,10 @@ function QueryBuilderSearchV2(
|
||||
values.push(tagValue);
|
||||
}
|
||||
|
||||
if (fieldValues) {
|
||||
values.push(
|
||||
...getValuesForDataType(fieldValues, currentFilterItem?.key?.dataType),
|
||||
);
|
||||
if (attributeValues?.payload) {
|
||||
const dataType = currentFilterItem?.key?.dataType || DataTypes.String;
|
||||
const key = DATA_TYPE_VS_ATTRIBUTE_VALUES_KEY[dataType];
|
||||
values.push(...(attributeValues?.payload?.[key] || []));
|
||||
|
||||
// here we want to suggest the variable name matching with the key here, we will go over the dynamic variables for the keys
|
||||
const variableName = dashboardDynamicVariables.find(
|
||||
@@ -770,12 +805,13 @@ function QueryBuilderSearchV2(
|
||||
}
|
||||
}, [
|
||||
hardcodedAttributeKeys,
|
||||
fieldValues,
|
||||
attributeValues?.payload,
|
||||
currentFilterItem?.key?.dataType,
|
||||
currentState,
|
||||
attributeKeys,
|
||||
data?.payload?.attributeKeys,
|
||||
isLogsDataSource,
|
||||
searchValue,
|
||||
suggestionsData?.payload?.attributes,
|
||||
operatorConfigKey,
|
||||
currentFilterItem?.key?.key,
|
||||
dashboardDynamicVariables,
|
||||
@@ -844,8 +880,8 @@ function QueryBuilderSearchV2(
|
||||
}, [deregisterShortcut, isLastQuery, registerShortcut]);
|
||||
|
||||
const loading = useMemo(
|
||||
() => isFetching || isFetchingAttributeValues,
|
||||
[isFetching, isFetchingAttributeValues],
|
||||
() => isFetching || isFetchingAttributeValues || isFetchingSuggestions,
|
||||
[isFetching, isFetchingAttributeValues, isFetchingSuggestions],
|
||||
);
|
||||
|
||||
const isMetricsDataSource = useMemo(
|
||||
@@ -955,7 +991,7 @@ function QueryBuilderSearchV2(
|
||||
setTags((prev) => [...prev, ...(val.items as ITag[])]);
|
||||
}}
|
||||
searchValue={searchValue}
|
||||
exampleQueries={[]}
|
||||
exampleQueries={suggestionsData?.payload?.example_queries || []}
|
||||
tags={tags}
|
||||
currentFilterItem={currentFilterItem}
|
||||
isLogsDataSource={isLogsDataSource}
|
||||
|
||||
@@ -1,23 +1,13 @@
|
||||
import { QueryClient, QueryClientProvider } from 'react-query';
|
||||
import {
|
||||
render,
|
||||
RenderResult,
|
||||
screen,
|
||||
waitFor,
|
||||
within,
|
||||
} from '@testing-library/react';
|
||||
import { render, RenderResult, screen, within } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import {
|
||||
initialQueriesMap,
|
||||
initialQueryBuilderFormValues,
|
||||
} from 'constants/queryBuilder';
|
||||
import { TelemetrytypesTelemetryFieldValuesDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { DynamicVariableSuggestion } from 'providers/Dashboard/store/dynamicVariableSuggestions';
|
||||
import { useFieldKeysSuggestion } from 'hooks/querySuggestions/useFieldKeysSuggestion';
|
||||
import { useFieldValuesSuggestion } from 'hooks/querySuggestions/useFieldValuesSuggestion';
|
||||
import { QueryBuilderContext } from 'providers/QueryBuilder';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { FieldDataType, TelemetryFieldKey } from 'types/api/v5/queryRange';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
|
||||
import QueryBuilderSearchV2 from '../QueryBuilderSearchV2';
|
||||
@@ -98,34 +88,57 @@ const renderWithContext = (props = {}): RenderResult => {
|
||||
const getSearchCombobox = (): HTMLElement =>
|
||||
within(screen.getByTestId('qb-search-select')).getByRole('combobox');
|
||||
|
||||
const mockFieldKeys: TelemetryFieldKey[] = [
|
||||
{ name: 'http.status', fieldContext: 'attribute', fieldDataType: 'string' },
|
||||
{ name: 'service.name', fieldContext: 'attribute', fieldDataType: 'string' },
|
||||
{ name: 'name', fieldContext: 'span', fieldDataType: 'string' },
|
||||
{ name: 'user.id', fieldContext: 'body', fieldDataType: 'string' },
|
||||
{ name: 'isRoot', fieldContext: 'span', fieldDataType: 'bool' },
|
||||
{
|
||||
name: 'unmapped.attribute',
|
||||
fieldContext: 'attribute',
|
||||
fieldDataType: 'not-a-real-data-type' as FieldDataType,
|
||||
},
|
||||
];
|
||||
// Constants to fix linter errors
|
||||
const TYPE_TAG = 'tag';
|
||||
const IS_COLUMN_FALSE = false;
|
||||
const IS_JSON_FALSE = false;
|
||||
|
||||
jest.mock('hooks/querySuggestions/useFieldKeysSuggestion', () => ({
|
||||
useFieldKeysSuggestion: jest.fn(() => ({
|
||||
data: mockFieldKeys,
|
||||
const mockAggregateKeysData = {
|
||||
payload: {
|
||||
attributeKeys: [
|
||||
{
|
||||
key: 'http.status',
|
||||
dataType: DataTypes.String,
|
||||
type: TYPE_TAG,
|
||||
id: 'http.status--string--tag--false',
|
||||
},
|
||||
{
|
||||
key: 'service.name',
|
||||
dataType: DataTypes.String,
|
||||
type: TYPE_TAG,
|
||||
isColumn: IS_COLUMN_FALSE,
|
||||
isJSON: IS_JSON_FALSE,
|
||||
id: 'service.name--string--tag--false',
|
||||
},
|
||||
{
|
||||
key: 'unmapped.attribute',
|
||||
dataType: 'not-a-real-data-type' as unknown as DataTypes,
|
||||
type: TYPE_TAG,
|
||||
isColumn: IS_COLUMN_FALSE,
|
||||
isJSON: IS_JSON_FALSE,
|
||||
id: 'unmapped.attribute--String--tag--false',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
jest.mock('hooks/queryBuilder/useGetAggregateKeys', () => ({
|
||||
useGetAggregateKeys: jest.fn(() => ({
|
||||
data: mockAggregateKeysData,
|
||||
isFetching: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
const mockFieldValues: TelemetrytypesTelemetryFieldValuesDTO = {
|
||||
stringValues: ['200', '404', '500'],
|
||||
numberValues: [200, 404, 500],
|
||||
const mockAggregateValuesData = {
|
||||
payload: {
|
||||
stringAttributeValues: ['200', '404', '500'],
|
||||
numberAttributeValues: [200, 404, 500],
|
||||
},
|
||||
};
|
||||
|
||||
jest.mock('hooks/querySuggestions/useFieldValuesSuggestion', () => ({
|
||||
useFieldValuesSuggestion: jest.fn(() => ({
|
||||
data: mockFieldValues,
|
||||
jest.mock('hooks/queryBuilder/useGetAggregateValues', () => ({
|
||||
useGetAggregateValues: jest.fn(() => ({
|
||||
data: mockAggregateValuesData,
|
||||
isFetching: false,
|
||||
})),
|
||||
}));
|
||||
@@ -183,11 +196,7 @@ describe('Suggestion Key -> Operator -> Value Flow', () => {
|
||||
expect.objectContaining({
|
||||
items: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
key: expect.objectContaining({
|
||||
key: 'http.status',
|
||||
type: 'attribute',
|
||||
dataType: 'string',
|
||||
}),
|
||||
key: expect.objectContaining({ key: 'http.status' }),
|
||||
op: '=',
|
||||
value: '200',
|
||||
}),
|
||||
@@ -283,7 +292,7 @@ describe('Filter chips', () => {
|
||||
key: {
|
||||
key: 'http.status',
|
||||
dataType: DataTypes.String,
|
||||
type: 'tag',
|
||||
type: TYPE_TAG,
|
||||
id: 'http.status--string--tag--false',
|
||||
},
|
||||
op: '=',
|
||||
@@ -323,87 +332,3 @@ describe('Filter chips', () => {
|
||||
expect(getSearchCombobox()).toHaveDisplayValue(CHIP_LABEL);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Key fetch context', () => {
|
||||
it('limits keys to the aggregate attribute context, as the resource filter needs', () => {
|
||||
renderWithContext({
|
||||
query: {
|
||||
...defaultProps.query,
|
||||
aggregateAttribute: {
|
||||
...defaultProps.query.aggregateAttribute,
|
||||
type: 'resource',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(useFieldKeysSuggestion).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ fieldContext: 'resource' }),
|
||||
undefined,
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('fetches keys from every context when the aggregate attribute has no type', () => {
|
||||
renderWithContext();
|
||||
|
||||
expect(useFieldKeysSuggestion).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ fieldContext: undefined }),
|
||||
undefined,
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Keys the legacy filter builders cannot read', () => {
|
||||
it('does not suggest body JSON keys or span search scope selectors', async () => {
|
||||
const user = userEvent.setup({ delay: null });
|
||||
renderWithContext();
|
||||
|
||||
const combobox = getSearchCombobox();
|
||||
await user.click(combobox);
|
||||
await user.type(combobox, 'user');
|
||||
await screen.findByRole('listbox');
|
||||
|
||||
await expect(screen.findByText('http.status')).resolves.toBeInTheDocument();
|
||||
expect(screen.queryByText('user.id')).not.toBeInTheDocument();
|
||||
expect(screen.queryByText('isRoot')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('Span field keys', () => {
|
||||
it('marks span fields as columns and looks up their values in the span context', async () => {
|
||||
const user = userEvent.setup({ delay: null });
|
||||
renderWithContext();
|
||||
|
||||
const combobox = getSearchCombobox();
|
||||
await user.click(combobox);
|
||||
await user.type(combobox, 'nam');
|
||||
await screen.findByRole('listbox');
|
||||
await user.click(await screen.findByText('name'));
|
||||
await user.click(screen.getByText('='));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(jest.mocked(useFieldValuesSuggestion)).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ name: 'name', fieldContext: 'span' }),
|
||||
undefined,
|
||||
expect.anything(),
|
||||
),
|
||||
);
|
||||
|
||||
await user.click(screen.getByText('200'));
|
||||
|
||||
expect(mockOnChange).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({
|
||||
items: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
key: expect.objectContaining({
|
||||
key: 'name',
|
||||
type: 'span',
|
||||
isColumn: true,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,6 +24,7 @@ import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
|
||||
import {
|
||||
EXCEPTION_CATALOGUE_SIZE,
|
||||
EXCEPTION_QUICK_FILTER_CAP,
|
||||
exceptionAttributeKeysResponse,
|
||||
exceptionAttributeValuesResponse,
|
||||
exceptionFieldKeysResponse,
|
||||
exceptionQuickFiltersResponse,
|
||||
@@ -57,7 +58,7 @@ export const exceptionsMocks = defineStoryMocks({
|
||||
filterKeys: choiceControl<ResponseState>('Filter keys', {
|
||||
group: FILTERS,
|
||||
description:
|
||||
'How `/fields/keys` answers when the resource filter opens, apart from the page-wide Data control.',
|
||||
'How `/autocomplete/attribute_keys` answers when the resource filter opens, apart from the page-wide Data control.',
|
||||
options: RESPONSE_STATES,
|
||||
value: 'loaded',
|
||||
}),
|
||||
@@ -94,7 +95,7 @@ export const exceptionsMocks = defineStoryMocks({
|
||||
|
||||
rest.get(
|
||||
'http://localhost/api/v1/fields/keys',
|
||||
respondWith(values.filterKeys, (req) =>
|
||||
response.json((req) =>
|
||||
exceptionFieldKeysResponse(req.url.searchParams.get('searchText')),
|
||||
),
|
||||
),
|
||||
@@ -108,6 +109,23 @@ export const exceptionsMocks = defineStoryMocks({
|
||||
),
|
||||
),
|
||||
),
|
||||
|
||||
rest.get(
|
||||
'http://localhost/api/v3/autocomplete/attribute_keys',
|
||||
respondWith(values.filterKeys, (req) =>
|
||||
exceptionAttributeKeysResponse(req.url.searchParams.get('searchText')),
|
||||
),
|
||||
),
|
||||
|
||||
rest.get(
|
||||
'http://localhost/api/v3/autocomplete/attribute_values',
|
||||
response.json((req) =>
|
||||
exceptionAttributeValuesResponse(
|
||||
req.url.searchParams.get('attributeKey'),
|
||||
req.url.searchParams.get('searchText'),
|
||||
),
|
||||
),
|
||||
),
|
||||
],
|
||||
effect: (values) => {
|
||||
set(LOCALSTORAGE.SHOW_EXCEPTIONS_QUICK_FILTERS, String(values.filterPanel));
|
||||
|
||||
@@ -17,6 +17,7 @@ import type {
|
||||
import {
|
||||
type BaseAutocompleteData,
|
||||
DataTypes,
|
||||
type IQueryAutocompleteResponse,
|
||||
} from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import type { Exception, Order, OrderBy } from 'types/api/errors/getAll';
|
||||
import type { IAttributeValuesResponse } from 'types/api/queryBuilder/getAttributesValues';
|
||||
@@ -310,6 +311,21 @@ const ATTRIBUTE_KEYS: BaseAutocompleteData[] = [
|
||||
{ key: 'cloud.region', dataType: DataTypes.String, type: 'resource' },
|
||||
];
|
||||
|
||||
export const exceptionAttributeKeysResponse = (
|
||||
searchText: string | null,
|
||||
): { status: string; data: IQueryAutocompleteResponse } => {
|
||||
const search = (searchText ?? '').toLowerCase();
|
||||
|
||||
return {
|
||||
status: 'success',
|
||||
data: {
|
||||
attributeKeys: ATTRIBUTE_KEYS.filter(({ key }) =>
|
||||
key.toLowerCase().includes(search),
|
||||
),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/** The quick filter settings panel lists its "other filters" from `/fields/keys`. */
|
||||
export const exceptionFieldKeysResponse = (
|
||||
searchText: string | null,
|
||||
|
||||
@@ -9,8 +9,6 @@ import type { QueryRangePayload } from 'types/api/metrics/getQueryRange';
|
||||
|
||||
import { choiceControl, countControl } from '@/storybook/controls/controls';
|
||||
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
|
||||
import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
|
||||
import { queryRangeV5ScalarTableResponse } from '@/storybook/msw/__story_mockdata__/queryRange';
|
||||
|
||||
import {
|
||||
attributeValuesResponse,
|
||||
@@ -77,34 +75,30 @@ export const celeryTaskMocks = defineStoryMocks({
|
||||
return [
|
||||
rest.post('http://localhost/api/v3/query_range', queryRange),
|
||||
rest.post('http://localhost/api/v4/query_range', queryRange),
|
||||
rest.post(
|
||||
'http://localhost/api/v5/query_range',
|
||||
response.json(() =>
|
||||
queryRangeV5ScalarTableResponse({
|
||||
groupBy: ['worker'],
|
||||
aggregations: ['A'],
|
||||
rows: (
|
||||
attributeValuesResponse('worker', '').data.stringAttributeValues ?? []
|
||||
)
|
||||
.slice(0, values.workers)
|
||||
.map((worker) => [worker, 12]),
|
||||
}),
|
||||
),
|
||||
),
|
||||
|
||||
rest.get(
|
||||
'http://localhost/api/v1/fields/values',
|
||||
'http://localhost/api/v3/autocomplete/attribute_values',
|
||||
response.json((req) => {
|
||||
const name = req.url.searchParams.get('name') ?? '';
|
||||
const found =
|
||||
attributeValuesResponse(
|
||||
name,
|
||||
req.url.searchParams.get('searchText') ?? '',
|
||||
).data.stringAttributeValues ?? [];
|
||||
|
||||
return fieldValuesResponse(
|
||||
name === 'worker' ? found.slice(0, values.workers) : found,
|
||||
const attributeKey = req.url.searchParams.get('attributeKey') ?? '';
|
||||
const values_ = attributeValuesResponse(
|
||||
attributeKey,
|
||||
req.url.searchParams.get('searchText') ?? '',
|
||||
);
|
||||
|
||||
if (attributeKey !== 'worker') {
|
||||
return values_;
|
||||
}
|
||||
|
||||
return {
|
||||
...values_,
|
||||
data: {
|
||||
...values_.data,
|
||||
stringAttributeValues: (values_.data.stringAttributeValues ?? []).slice(
|
||||
0,
|
||||
values.workers,
|
||||
),
|
||||
},
|
||||
};
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
@@ -13,7 +13,6 @@ import {
|
||||
toggleControl,
|
||||
} from '@/storybook/controls/controls';
|
||||
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
|
||||
import { fieldValuesResponse } from '@/storybook/msw/__story_mockdata__/fields';
|
||||
|
||||
import {
|
||||
attributeValuesResponse,
|
||||
@@ -92,13 +91,11 @@ export const queueOverviewMocks = defineStoryMocks({
|
||||
rest.post('http://localhost/api/v4/query_range', queryRange),
|
||||
|
||||
rest.get(
|
||||
'http://localhost/api/v1/fields/values',
|
||||
'http://localhost/api/v3/autocomplete/attribute_values',
|
||||
response.json((req) =>
|
||||
fieldValuesResponse(
|
||||
attributeValuesResponse(
|
||||
req.url.searchParams.get('name') ?? '',
|
||||
req.url.searchParams.get('searchText') ?? '',
|
||||
).data.stringAttributeValues ?? [],
|
||||
attributeValuesResponse(
|
||||
req.url.searchParams.get('attributeKey') ?? '',
|
||||
req.url.searchParams.get('searchText') ?? '',
|
||||
),
|
||||
),
|
||||
),
|
||||
|
||||
@@ -82,15 +82,15 @@ const CELERY_TASKS = [
|
||||
const CELERY_WORKERS = ['worker-a', 'worker-b', 'worker-c'];
|
||||
|
||||
/**
|
||||
* Values per attribute key. The kafka config selects read the v3 endpoint and
|
||||
* the celery filter rows read `/fields/values`, so one map answers for all of
|
||||
* Values per autocomplete attribute key. The kafka config selects and both
|
||||
* celery filter rows read the same v3 endpoint, so one map answers for all of
|
||||
* them; a key nobody seeded answers empty rather than 501.
|
||||
*/
|
||||
const ATTRIBUTE_VALUES: Record<string, string[]> = {
|
||||
group: CONSUMER_GROUPS,
|
||||
topic: TOPICS,
|
||||
partition: PARTITIONS,
|
||||
'service.name': KAFKA_SERVICES,
|
||||
serviceName: KAFKA_SERVICES,
|
||||
name: [
|
||||
'order-events send',
|
||||
'order-events process',
|
||||
|
||||
@@ -3,7 +3,6 @@ import { Button, Form, Space, Tooltip } from 'antd';
|
||||
import { Dropdown, type DropdownItemType } from '@signozhq/ui/dropdown';
|
||||
import { Divider } from '@signozhq/ui/divider';
|
||||
import { Switch } from '@signozhq/ui/switch';
|
||||
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import cx from 'classnames';
|
||||
import { FilterSelect } from 'components/CeleryOverview/CeleryOverviewConfigOptions/CeleryOverviewConfigOptions';
|
||||
import { QueryParams } from 'constants/query';
|
||||
@@ -150,8 +149,7 @@ function FunnelStep({
|
||||
<FilterSelect
|
||||
placeholder="Select Service"
|
||||
queryParam={QueryParams.service}
|
||||
filterType="service.name"
|
||||
fieldContext={TelemetrytypesFieldContextDTO.resource}
|
||||
filterType="serviceName"
|
||||
shouldSetQueryParams={false}
|
||||
values={stepData.service_name}
|
||||
isMultiple={false}
|
||||
@@ -171,7 +169,6 @@ function FunnelStep({
|
||||
placeholder="Select Span name"
|
||||
queryParam={QueryParams.spanName}
|
||||
filterType="name"
|
||||
fieldContext={TelemetrytypesFieldContextDTO.span}
|
||||
shouldSetQueryParams={false}
|
||||
values={stepData.span_name}
|
||||
isMultiple={false}
|
||||
|
||||
@@ -3,19 +3,15 @@
|
||||
* Do not hand-edit: regenerate instead.
|
||||
*/
|
||||
|
||||
import {
|
||||
TelemetrytypesFieldContextDTO,
|
||||
TelemetrytypesSignalDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
import ROUTES from 'constants/routes';
|
||||
import { rest } from 'msw';
|
||||
|
||||
import { choiceControl, countControl } from '@/storybook/controls/controls';
|
||||
import { defineStoryMocks } from '@/storybook/controls/defineStoryMocks';
|
||||
import {
|
||||
fieldKeysResponse,
|
||||
fieldValuesResponse,
|
||||
} from '@/storybook/msw/__story_mockdata__/fields';
|
||||
attributeKeysResponse,
|
||||
attributeValuesResponse,
|
||||
} from '@/storybook/msw/__story_mockdata__/attributes';
|
||||
|
||||
import {
|
||||
FUNNEL_ID,
|
||||
@@ -45,9 +41,13 @@ const CONVERSION: Record<DropOff, number> = {
|
||||
|
||||
const ENTERED = 24_000;
|
||||
|
||||
const STEP_RESOURCE_KEYS = ['service.name', 'deployment.environment'];
|
||||
|
||||
const STEP_ATTRIBUTE_KEYS = ['name', 'http.status_code', 'hasError'];
|
||||
const STEP_FILTER_KEYS = [
|
||||
'service.name',
|
||||
'name',
|
||||
'http.status_code',
|
||||
'deployment.environment',
|
||||
'hasError',
|
||||
];
|
||||
|
||||
const STEP_FILTER_VALUES: Record<string, string[]> = {
|
||||
'service.name': ['frontend', 'checkout', 'payment', 'shipping'],
|
||||
@@ -115,44 +115,20 @@ export const tracesFunnelDetailsMocks = defineStoryMocks({
|
||||
'http://localhost/api/v1/trace-funnels/analytics/error-traces',
|
||||
response.json(() => funnelErrorTracesResponse(values.traces)),
|
||||
),
|
||||
// The `Where` box on every step reads its suggestions as soon as the step
|
||||
// renders, and the service and span name selects read `service.name` and
|
||||
// `name` from the same values endpoint.
|
||||
// The `Where` box on every step is the trace query builder, which reads
|
||||
// its suggestions as soon as the step renders.
|
||||
rest.get(
|
||||
'http://localhost/api/v1/fields/keys',
|
||||
response.json(() => {
|
||||
const signal = TelemetrytypesSignalDTO.traces;
|
||||
const resource = fieldKeysResponse(STEP_RESOURCE_KEYS, {
|
||||
signal,
|
||||
fieldContext: TelemetrytypesFieldContextDTO.resource,
|
||||
});
|
||||
const attribute = fieldKeysResponse(STEP_ATTRIBUTE_KEYS, {
|
||||
signal,
|
||||
fieldContext: TelemetrytypesFieldContextDTO.attribute,
|
||||
});
|
||||
|
||||
return {
|
||||
...resource,
|
||||
data: {
|
||||
...resource.data,
|
||||
keys: { ...resource.data.keys, ...attribute.data.keys },
|
||||
},
|
||||
};
|
||||
}),
|
||||
'http://localhost/api/v3/autocomplete/attribute_keys',
|
||||
response.json(() => attributeKeysResponse(STEP_FILTER_KEYS)),
|
||||
),
|
||||
rest.get(
|
||||
'http://localhost/api/v1/fields/values',
|
||||
response.json((req) => {
|
||||
const searchText = (
|
||||
req.url.searchParams.get('searchText') ?? ''
|
||||
).toLowerCase();
|
||||
|
||||
return fieldValuesResponse(
|
||||
(STEP_FILTER_VALUES[req.url.searchParams.get('name') ?? ''] ?? []).filter(
|
||||
(value) => value.toLowerCase().includes(searchText),
|
||||
),
|
||||
);
|
||||
}),
|
||||
'http://localhost/api/v3/autocomplete/attribute_values',
|
||||
response.json((req) =>
|
||||
attributeValuesResponse(
|
||||
STEP_FILTER_VALUES[req.url.searchParams.get('attributeKey') ?? ''] ?? [],
|
||||
req.url.searchParams.get('searchText') ?? '',
|
||||
),
|
||||
),
|
||||
),
|
||||
];
|
||||
},
|
||||
|
||||
@@ -53,7 +53,6 @@ export type FieldContext =
|
||||
| 'scope'
|
||||
| 'attribute'
|
||||
| 'event'
|
||||
| 'body'
|
||||
| '';
|
||||
|
||||
export type FieldDataType =
|
||||
|
||||
@@ -1,6 +1,4 @@
|
||||
import { SIGNOZ_ADMIN_EMAIL } from 'constants/app';
|
||||
|
||||
import { buildAbsolutePath, isSignozAdminEmail } from '../app';
|
||||
import { buildAbsolutePath } from '../app';
|
||||
|
||||
// buildAbsolutePath reads getCurrentLocation().pathname (basename-relative) rather than
|
||||
// window.location.pathname, so we mock lib/router/navigation instead of utils/getLocation.
|
||||
@@ -124,13 +122,3 @@ describe('buildAbsolutePath', () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('isSignozAdminEmail', () => {
|
||||
it('matches the SigNoz admin email', () => {
|
||||
expect(isSignozAdminEmail(SIGNOZ_ADMIN_EMAIL)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['jon@winterfell.dev', '', undefined])('rejects %p', (email) => {
|
||||
expect(isSignozAdminEmail(email)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
import {
|
||||
attributeTypeToFieldContext,
|
||||
toLegacyLogsFilter,
|
||||
} from '../fieldContext';
|
||||
|
||||
const item = (type: string): TagFilter['items'][number] => ({
|
||||
id: type,
|
||||
key: { key: 'k', dataType: DataTypes.String, type },
|
||||
op: '=',
|
||||
value: 'v',
|
||||
});
|
||||
|
||||
describe('toLegacyLogsFilter', () => {
|
||||
it('sends attribute keys as tag and log fields as top-level fields', () => {
|
||||
const filter: TagFilter = {
|
||||
op: 'AND',
|
||||
items: [
|
||||
item('attribute'),
|
||||
item('resource'),
|
||||
item('scope'),
|
||||
item('log'),
|
||||
item('body'),
|
||||
item('tag'),
|
||||
item(''),
|
||||
],
|
||||
};
|
||||
|
||||
expect(
|
||||
toLegacyLogsFilter(filter).items.map((filterItem) => filterItem.key?.type),
|
||||
).toStrictEqual(['tag', 'resource', 'scope', '', '', 'tag', '']);
|
||||
});
|
||||
|
||||
it('sends the scope name and version as top-level fields only from the scope context', () => {
|
||||
const filter: TagFilter = {
|
||||
op: 'AND',
|
||||
items: [
|
||||
['scope_name', 'scope'],
|
||||
['scope_version', 'scope'],
|
||||
['scope.attr', 'scope'],
|
||||
['scope_name', 'attribute'],
|
||||
['scope_name', 'tag'],
|
||||
['scope_version', 'resource'],
|
||||
].map(([key, type]) => ({
|
||||
id: `${key}-${type}`,
|
||||
key: { key, dataType: DataTypes.String, type },
|
||||
op: '=',
|
||||
value: 'v',
|
||||
})),
|
||||
};
|
||||
|
||||
expect(
|
||||
toLegacyLogsFilter(filter).items.map((filterItem) => filterItem.key?.type),
|
||||
).toStrictEqual(['', '', 'scope', 'tag', 'tag', 'resource']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('attributeTypeToFieldContext', () => {
|
||||
it('maps legacy tag to attribute and passes v5 contexts through', () => {
|
||||
expect(
|
||||
['tag', 'attribute', 'resource', 'scope', 'span', 'log', ''].map(
|
||||
attributeTypeToFieldContext,
|
||||
),
|
||||
).toStrictEqual([
|
||||
'attribute',
|
||||
'attribute',
|
||||
'resource',
|
||||
'scope',
|
||||
'span',
|
||||
'log',
|
||||
undefined,
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,4 @@
|
||||
import getLocalStorage from 'api/browser/localstorage/get';
|
||||
import { SIGNOZ_ADMIN_EMAIL } from 'constants/app';
|
||||
import { FeatureKeys } from 'constants/features';
|
||||
import { SKIP_ONBOARDING } from 'constants/onboarding';
|
||||
import dayjs from 'dayjs';
|
||||
@@ -17,9 +16,6 @@ export function extractDomain(email: string): string {
|
||||
return emailParts[1];
|
||||
}
|
||||
|
||||
export const isSignozAdminEmail = (email?: string): boolean =>
|
||||
email === SIGNOZ_ADMIN_EMAIL;
|
||||
|
||||
const stripVersionPrefix = (version: string): string =>
|
||||
version?.replace(/^v/i, '') ?? '';
|
||||
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import { TelemetrytypesFieldContextDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import type { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
const FIELD_CONTEXT_TO_ATTRIBUTE_TYPE: Record<string, string> = {
|
||||
[TelemetrytypesFieldContextDTO.attribute]: 'tag',
|
||||
[TelemetrytypesFieldContextDTO.resource]: 'resource',
|
||||
};
|
||||
|
||||
const ATTRIBUTE_TYPE_TO_FIELD_CONTEXT: Record<
|
||||
string,
|
||||
TelemetrytypesFieldContextDTO
|
||||
> = {
|
||||
tag: TelemetrytypesFieldContextDTO.attribute,
|
||||
attribute: TelemetrytypesFieldContextDTO.attribute,
|
||||
resource: TelemetrytypesFieldContextDTO.resource,
|
||||
scope: TelemetrytypesFieldContextDTO.scope,
|
||||
span: TelemetrytypesFieldContextDTO.span,
|
||||
log: TelemetrytypesFieldContextDTO.log,
|
||||
};
|
||||
|
||||
// The legacy logs builders read `tag` as attributes and an empty type as a top-level field;
|
||||
// any other type falls through to resources.
|
||||
const LEGACY_LOGS_KEY_TYPE: Record<string, string> = {
|
||||
[TelemetrytypesFieldContextDTO.attribute]: 'tag',
|
||||
tag: 'tag',
|
||||
[TelemetrytypesFieldContextDTO.resource]: 'resource',
|
||||
[TelemetrytypesFieldContextDTO.scope]: 'scope',
|
||||
};
|
||||
|
||||
// Top-level log columns that v5 files under the scope context.
|
||||
const LEGACY_LOGS_TOP_LEVEL_SCOPE_FIELDS = ['scope_name', 'scope_version'];
|
||||
|
||||
export const fieldContextToAttributeType = (fieldContext?: string): string =>
|
||||
(fieldContext && FIELD_CONTEXT_TO_ATTRIBUTE_TYPE[fieldContext]) || '';
|
||||
|
||||
export const attributeTypeToFieldContext = (
|
||||
type?: string | null,
|
||||
): TelemetrytypesFieldContextDTO | undefined =>
|
||||
type ? ATTRIBUTE_TYPE_TO_FIELD_CONTEXT[type] : undefined;
|
||||
|
||||
export const toLegacyLogsFilter = (filter: TagFilter): TagFilter =>
|
||||
filter && {
|
||||
...filter,
|
||||
items: (filter.items ?? []).map((item) => ({
|
||||
...item,
|
||||
key: item.key && {
|
||||
...item.key,
|
||||
type:
|
||||
item.key.type === TelemetrytypesFieldContextDTO.scope &&
|
||||
LEGACY_LOGS_TOP_LEVEL_SCOPE_FIELDS.includes(item.key.key)
|
||||
? ''
|
||||
: (LEGACY_LOGS_KEY_TYPE[item.key.type ?? ''] ?? ''),
|
||||
},
|
||||
})),
|
||||
};
|
||||
@@ -1,21 +0,0 @@
|
||||
import { TelemetrytypesTelemetryFieldValuesDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
|
||||
// Numbers and bools stay raw so tags keep their value type.
|
||||
export const getValuesForDataType = (
|
||||
values: TelemetrytypesTelemetryFieldValuesDTO | undefined,
|
||||
dataType: DataTypes | undefined,
|
||||
): string[] => {
|
||||
switch (dataType) {
|
||||
case DataTypes.Int64:
|
||||
case DataTypes.Float64:
|
||||
case DataTypes.ArrayInt64:
|
||||
case DataTypes.ArrayFloat64:
|
||||
return (values?.numberValues ?? []) as unknown as string[];
|
||||
case DataTypes.bool:
|
||||
case DataTypes.ArrayBool:
|
||||
return (values?.boolValues ?? []) as unknown as string[];
|
||||
default:
|
||||
return values?.stringValues ?? [];
|
||||
}
|
||||
};
|
||||
@@ -26,8 +26,9 @@ const bucketCacheCellSteps = 60
|
||||
type BucketCache interface {
|
||||
// cached portion + list of gaps to fetch
|
||||
GetMissRanges(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step) (cached *qbtypes.Result, missing []*qbtypes.TimeRange)
|
||||
// store fresh buckets for future hits
|
||||
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result)
|
||||
// store fresh buckets for future hits; only the ranges in the map were
|
||||
// queried for fresh, the rest of it came from the cache
|
||||
Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats)
|
||||
}
|
||||
|
||||
// bucketCache implements the BucketCache interface.
|
||||
@@ -116,7 +117,7 @@ func (bc *bucketCache) GetMissRanges(
|
||||
}
|
||||
|
||||
// Merge buckets into a single result
|
||||
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings)
|
||||
mergedResult := bc.mergeBuckets(ctx, relevantBuckets, cachedWarnings, startMs, endMs)
|
||||
|
||||
// Filter the merged result to only include values within the requested time range
|
||||
mergedResult = bc.filterResultToTimeRange(mergedResult, q, startMs, endMs, stepMs)
|
||||
@@ -125,7 +126,7 @@ func (bc *bucketCache) GetMissRanges(
|
||||
}
|
||||
|
||||
// Put stores fresh query results in the cache.
|
||||
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result) {
|
||||
func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Query, step qbtypes.Step, fresh *qbtypes.Result, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) {
|
||||
// Get query window
|
||||
startMs, endMs := q.Window()
|
||||
|
||||
@@ -188,7 +189,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
|
||||
|
||||
// Convert trimmed result to buckets, one per grid cell it spans
|
||||
cellWidthMs := bucketCacheCellSteps * stepMs
|
||||
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs)
|
||||
freshBuckets := bc.resultToBuckets(ctx, trimmedResult, cachableStartMs, cachableEndMs, cellWidthMs, execStatsForQueriedTimeRanges)
|
||||
|
||||
cellKeys := make([]string, len(freshBuckets))
|
||||
cellEntries := make([]cachetypes.Cacheable, len(freshBuckets))
|
||||
@@ -210,7 +211,7 @@ func (bc *bucketCache) Put(ctx context.Context, orgID valuer.UUID, q qbtypes.Que
|
||||
cellEntry = cellEntries[i].(*qbtypes.CachedData)
|
||||
}
|
||||
|
||||
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket)
|
||||
cellBuckets, changed := bc.mergeFreshBucketIntoCell(ctx, cellEntry.Buckets, freshBucket, execStatsForQueriedTimeRanges)
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
@@ -463,7 +464,7 @@ func (bc *bucketCache) filterRelevantBuckets(buckets []*qbtypes.CachedBucket, st
|
||||
}
|
||||
|
||||
// mergeBuckets combines multiple cached buckets into a single result.
|
||||
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string) *qbtypes.Result {
|
||||
func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.CachedBucket, warnings []string, startMs, endMs uint64) *qbtypes.Result {
|
||||
if len(buckets) == 0 {
|
||||
return &qbtypes.Result{}
|
||||
}
|
||||
@@ -471,12 +472,13 @@ func (bc *bucketCache) mergeBuckets(ctx context.Context, buckets []*qbtypes.Cach
|
||||
// All buckets should have the same type
|
||||
resultType := buckets[0].Type
|
||||
|
||||
// Aggregate stats
|
||||
// A bucket can reach past the window; only the part inside it is served
|
||||
var totalStats qbtypes.ExecStats
|
||||
for _, bucket := range buckets {
|
||||
totalStats.RowsScanned += bucket.Stats.RowsScanned
|
||||
totalStats.BytesScanned += bucket.Stats.BytesScanned
|
||||
totalStats.DurationMS += bucket.Stats.DurationMS
|
||||
servedStats := bucket.ProrateStatsToWindow(startMs, endMs)
|
||||
totalStats.RowsScanned += servedStats.RowsScanned
|
||||
totalStats.BytesScanned += servedStats.BytesScanned
|
||||
totalStats.DurationMS += servedStats.DurationMS
|
||||
}
|
||||
|
||||
// Merge values based on type
|
||||
@@ -673,7 +675,7 @@ func (bc *bucketCache) isEmptyResult(result *qbtypes.Result) (isEmpty bool, isFi
|
||||
}
|
||||
|
||||
// resultToBuckets converts a query result into time-based buckets.
|
||||
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64) []*qbtypes.CachedBucket {
|
||||
func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Result, startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) []*qbtypes.CachedBucket {
|
||||
// Check if result is empty
|
||||
isEmpty, isFiltered := bc.isEmptyResult(result)
|
||||
|
||||
@@ -685,7 +687,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
|
||||
}
|
||||
|
||||
// Empty filtered results still become buckets, so the range is not re-queried
|
||||
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs)
|
||||
perCellBuckets, err := result.SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs, execStatsForQueriedTimeRanges)
|
||||
if err != nil {
|
||||
bc.logger.ErrorContext(ctx, "failed to convert result to buckets", errors.Attr(err))
|
||||
return nil
|
||||
@@ -697,7 +699,7 @@ func (bc *bucketCache) resultToBuckets(ctx context.Context, result *qbtypes.Resu
|
||||
// and reports whether the cell changed: a bucket already covering the fresh
|
||||
// one leaves it as is, buckets inside the fresh one are replaced, buckets
|
||||
// overlapping or edge to edge with it are unioned, and the rest are kept apart.
|
||||
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket) ([]*qbtypes.CachedBucket, bool) {
|
||||
func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets []*qbtypes.CachedBucket, freshBucket *qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) ([]*qbtypes.CachedBucket, bool) {
|
||||
keptBuckets := make([]*qbtypes.CachedBucket, 0, len(cellBuckets)+1)
|
||||
touchingBuckets := []*qbtypes.CachedBucket{}
|
||||
for _, cellBucket := range cellBuckets {
|
||||
@@ -710,7 +712,7 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
|
||||
keptBuckets = append(keptBuckets, cellBucket)
|
||||
}
|
||||
}
|
||||
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets))
|
||||
keptBuckets = append(keptBuckets, bc.unionBuckets(ctx, freshBucket, touchingBuckets, execStatsForQueriedTimeRanges))
|
||||
|
||||
slices.SortFunc(keptBuckets, func(a, b *qbtypes.CachedBucket) int {
|
||||
return cmp.Compare(a.StartMs, b.StartMs)
|
||||
@@ -721,7 +723,16 @@ func (bc *bucketCache) mergeFreshBucketIntoCell(ctx context.Context, cellBuckets
|
||||
// unionBuckets widens the fresh bucket by the buckets it touches. Buckets
|
||||
// inside it are replaced outright, so only a genuine partial overlap decodes
|
||||
// anything.
|
||||
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket) *qbtypes.CachedBucket {
|
||||
func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.CachedBucket, touchingBuckets []*qbtypes.CachedBucket, execStatsForQueriedTimeRanges map[qbtypes.TimeRange]qbtypes.ExecStats) *qbtypes.CachedBucket {
|
||||
// Data queried again takes the new query's cost, which the fresh bucket
|
||||
// carries; a touching bucket keeps its cost only for what was not queried again
|
||||
for _, touchingBucket := range touchingBuckets {
|
||||
keptStats := touchingBucket.ProrateStatsToUnqueried(execStatsForQueriedTimeRanges)
|
||||
freshBucket.Stats.RowsScanned += keptStats.RowsScanned
|
||||
freshBucket.Stats.BytesScanned += keptStats.BytesScanned
|
||||
freshBucket.Stats.DurationMS += keptStats.DurationMS
|
||||
}
|
||||
|
||||
bucketsToMerge := []*qbtypes.CachedBucket{freshBucket}
|
||||
for _, touchingBucket := range touchingBuckets {
|
||||
if !freshBucket.Contains(touchingBucket) {
|
||||
@@ -742,14 +753,11 @@ func (bc *bucketCache) unionBuckets(ctx context.Context, freshBucket *qbtypes.Ca
|
||||
EndMs: freshBucket.EndMs,
|
||||
Type: freshBucket.Type,
|
||||
Value: value,
|
||||
Stats: qbtypes.ExecStats{StepIntervals: freshBucket.Stats.StepIntervals},
|
||||
Stats: freshBucket.Stats,
|
||||
}
|
||||
for _, mergedBucket := range bucketsToMerge {
|
||||
unionBucket.StartMs = min(unionBucket.StartMs, mergedBucket.StartMs)
|
||||
unionBucket.EndMs = max(unionBucket.EndMs, mergedBucket.EndMs)
|
||||
unionBucket.Stats.RowsScanned += mergedBucket.Stats.RowsScanned
|
||||
unionBucket.Stats.BytesScanned += mergedBucket.Stats.BytesScanned
|
||||
unionBucket.Stats.DurationMS += mergedBucket.Stats.DurationMS
|
||||
}
|
||||
return unionBucket
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ func BenchmarkBucketCache_GetMissRanges(b *testing.B) {
|
||||
endMs: uint64((i + 1) * 10000),
|
||||
}
|
||||
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
|
||||
}
|
||||
|
||||
// Create test queries with varying cache hit patterns
|
||||
@@ -121,7 +121,7 @@ func BenchmarkBucketCache_Put(b *testing.B) {
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
for j := 0; j < tc.numQueries; j++ {
|
||||
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j])
|
||||
bc.Put(ctx, orgID, queries[j], qbtypes.Step{Duration: 1000 * time.Millisecond}, results[j], map[qbtypes.TimeRange]qbtypes.ExecStats{{From: queries[j].startMs, To: queries[j].endMs}: results[j].Stats})
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -259,7 +259,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
|
||||
endMs: uint64((i + 1) * 10000),
|
||||
}
|
||||
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
@@ -284,7 +284,7 @@ func BenchmarkBucketCache_ConcurrentOperations(b *testing.B) {
|
||||
endMs: uint64((i + 1) * 10000),
|
||||
}
|
||||
result := createBenchmarkResult(query.startMs, query.endMs, 1000)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
|
||||
case 2: // Partial read
|
||||
query := &mockQuery{
|
||||
fingerprint: fmt.Sprintf("concurrent-query-%d", i%100),
|
||||
|
||||
@@ -59,7 +59,7 @@ func TestBucketCacheStepAlignment(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put result in cache
|
||||
bc.Put(ctx, orgID, query, step, result)
|
||||
bc.Put(ctx, orgID, query, step, result, nil)
|
||||
|
||||
// Get cached data
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
|
||||
@@ -109,7 +109,7 @@ func TestBucketCacheNoStepInterval(t *testing.T) {
|
||||
}
|
||||
|
||||
// Should cache the entire range when step is 0
|
||||
bc.Put(ctx, orgID, query, step, result)
|
||||
bc.Put(ctx, orgID, query, step, result, nil)
|
||||
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
|
||||
assert.NotNil(t, cached)
|
||||
|
||||
@@ -156,7 +156,7 @@ func TestBucketCache_Put_And_Get(t *testing.T) {
|
||||
}
|
||||
|
||||
// Store in cache
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
|
||||
|
||||
// Wait a bit for cache to be written
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -189,7 +189,7 @@ func TestBucketCache_PartialHit(t *testing.T) {
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", 1000, 3000, 1000),
|
||||
}
|
||||
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
|
||||
|
||||
// Wait for cache write
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -225,7 +225,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
|
||||
bc.Put(context.Background(), valuer.UUID{}, query1, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", 1000, 2000, 100),
|
||||
})
|
||||
}, nil)
|
||||
|
||||
query2 := &mockQuery{
|
||||
fingerprint: "test-query",
|
||||
@@ -235,7 +235,7 @@ func TestBucketCache_MultipleBuckets(t *testing.T) {
|
||||
bc.Put(context.Background(), valuer.UUID{}, query2, qbtypes.Step{Duration: 100 * time.Millisecond}, &qbtypes.Result{
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", 3000, 4000, 100),
|
||||
})
|
||||
}, nil)
|
||||
|
||||
// Wait for cache writes
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -280,7 +280,7 @@ func TestBucketCache_FluxInterval(t *testing.T) {
|
||||
}
|
||||
|
||||
// This should not be cached due to flux interval
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
|
||||
// Wait a bit
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -358,7 +358,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
|
||||
{Series: []*qbtypes.TimeSeries{series1}},
|
||||
},
|
||||
},
|
||||
})
|
||||
}, nil)
|
||||
|
||||
// Cache second part
|
||||
query2 := &mockQuery{
|
||||
@@ -374,7 +374,7 @@ func TestBucketCache_MergeTimeSeriesResults(t *testing.T) {
|
||||
{Series: []*qbtypes.TimeSeries{series2}},
|
||||
},
|
||||
},
|
||||
})
|
||||
}, nil)
|
||||
|
||||
// Wait for cache writes
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -441,7 +441,7 @@ func TestBucketCache_RawData(t *testing.T) {
|
||||
Value: rawData,
|
||||
}
|
||||
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
|
||||
@@ -481,7 +481,7 @@ func TestBucketCache_ScalarData(t *testing.T) {
|
||||
Value: scalarData,
|
||||
}
|
||||
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
cached, missing := bc.GetMissRanges(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
|
||||
@@ -509,7 +509,7 @@ func TestBucketCache_EmptyFingerprint(t *testing.T) {
|
||||
Value: createTestTimeSeries("A", 1000, 5000, 1000),
|
||||
}
|
||||
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Should still be able to retrieve
|
||||
@@ -564,7 +564,7 @@ func TestBucketCache_ConcurrentAccess(t *testing.T) {
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries(fmt.Sprintf("Q%d", id), query.startMs, query.endMs, 100),
|
||||
}
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result)
|
||||
bc.Put(context.Background(), valuer.UUID{}, query, qbtypes.Step{Duration: 100 * time.Microsecond}, result, nil)
|
||||
done <- true
|
||||
}(i)
|
||||
}
|
||||
@@ -624,7 +624,7 @@ func TestBucketCache_GetMissRanges_FluxInterval(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, cachedResult, nil)
|
||||
|
||||
// Get miss ranges
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
|
||||
@@ -686,7 +686,7 @@ func TestBucketCache_Put_FluxIntervalTrimming(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
|
||||
|
||||
// Retrieve cached data
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
|
||||
@@ -756,7 +756,7 @@ func TestBucketCache_Put_EntireRangeInFluxInterval(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result - should not cache anything
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
|
||||
// Try to get cached data - should have no cached data
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond})
|
||||
@@ -862,7 +862,7 @@ func TestBucketCache_EmptyDataHandling(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, tt.result, nil)
|
||||
|
||||
// Wait a bit for cache to be written
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -927,7 +927,7 @@ func TestBucketCache_PartialValues(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: query.startMs, To: query.endMs}: result.Stats})
|
||||
|
||||
// Wait for cache to be written
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -997,7 +997,7 @@ func TestBucketCache_AllPartialValues(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
|
||||
// Wait for cache to be written
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
@@ -1058,7 +1058,7 @@ func TestBucketCache_FilteredCachedResults(t *testing.T) {
|
||||
}
|
||||
|
||||
// Cache the wide range
|
||||
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1)
|
||||
bc.Put(ctx, orgID, query1, qbtypes.Step{Duration: 1000 * time.Millisecond}, result1, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Now query for a smaller range (2000-3500ms)
|
||||
@@ -1125,7 +1125,7 @@ func TestBucketCache_PromQLKeepsTheValueAtTheWindowEnd(t *testing.T) {
|
||||
}},
|
||||
}},
|
||||
},
|
||||
})
|
||||
}, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
cached, missing := bc.GetMissRanges(ctx, orgID, query, step)
|
||||
@@ -1291,7 +1291,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Get cached data
|
||||
@@ -1345,7 +1345,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Get cached data
|
||||
@@ -1397,7 +1397,7 @@ func TestBucketCache_PartialValueDetection(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Get cached data
|
||||
@@ -1454,7 +1454,7 @@ func TestBucketCache_NoCache(t *testing.T) {
|
||||
}
|
||||
|
||||
// Put the result in cache
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result)
|
||||
bc.Put(ctx, orgID, query, qbtypes.Step{Duration: 1000 * time.Millisecond}, result, nil)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
// Verify data is cached
|
||||
@@ -1475,11 +1475,12 @@ func TestBucketCachePutSlicesAtCellBoundaries(t *testing.T) {
|
||||
|
||||
// a 1s step gives 60s cells, so 30s to 150s spans three of them
|
||||
query := &mockQuery{fingerprint: "sliced-query", startMs: 30000, endMs: 150000}
|
||||
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
|
||||
result := &qbtypes.Result{
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", 30000, 150000, 1000),
|
||||
Stats: qbtypes.ExecStats{RowsScanned: 120, BytesScanned: 1200, DurationMS: 12},
|
||||
})
|
||||
}
|
||||
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: 30000, To: 150000}: result.Stats})
|
||||
|
||||
type bucketShape struct {
|
||||
cell, startMs, endMs, rowsScanned uint64
|
||||
@@ -1539,7 +1540,7 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
|
||||
name: "OverlappingWrites_BecomeOne",
|
||||
firstWindow: qbtypes.TimeRange{From: 1000, To: 4000},
|
||||
secondWindow: qbtypes.TimeRange{From: 3000, To: 6000},
|
||||
expectedBuckets: []bucketShape{{1000, 6000, 6}},
|
||||
expectedBuckets: []bucketShape{{1000, 6000, 5}},
|
||||
expectedMissingRanges: []*qbtypes.TimeRange{},
|
||||
expectedValueCount: 5,
|
||||
},
|
||||
@@ -1570,12 +1571,13 @@ func TestBucketCachePutAbsorbsWritesWithinACell(t *testing.T) {
|
||||
|
||||
for _, window := range []qbtypes.TimeRange{testCase.firstWindow, testCase.secondWindow} {
|
||||
query := &mockQuery{fingerprint: "absorb-query", startMs: window.From, endMs: window.To}
|
||||
bc.Put(ctx, orgID, query, step, &qbtypes.Result{
|
||||
result := &qbtypes.Result{
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", window.From, window.To, 1000),
|
||||
// one row per second, so a bucket's stats read as the seconds it was scanned for
|
||||
// one row per second, so a bucket's stats read as the seconds of data it holds
|
||||
Stats: qbtypes.ExecStats{RowsScanned: (window.To - window.From) / 1000},
|
||||
})
|
||||
}
|
||||
bc.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{window: result.Stats})
|
||||
}
|
||||
|
||||
query := &mockQuery{
|
||||
@@ -1609,7 +1611,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", 1000, 10000, 1000),
|
||||
Warnings: []string{"from the wide write"},
|
||||
})
|
||||
}, nil)
|
||||
|
||||
// a write already covered by the cell's bucket must not rewrite the entry,
|
||||
// which shows as its warnings never arriving
|
||||
@@ -1618,7 +1620,7 @@ func TestBucketCachePutLeavesAnUnchangedCellAlone(t *testing.T) {
|
||||
Type: qbtypes.RequestTypeTimeSeries,
|
||||
Value: createTestTimeSeries("A", 3000, 5000, 1000),
|
||||
Warnings: []string{"from the inside write"},
|
||||
})
|
||||
}, nil)
|
||||
|
||||
var cellEntry qbtypes.CachedData
|
||||
require.NoError(t, bc.cache.Get(ctx, orgID, bc.generateCacheKey(wide, 0), &cellEntry))
|
||||
|
||||
@@ -828,7 +828,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
|
||||
return nil, err
|
||||
}
|
||||
// Store in cache for future use
|
||||
q.bucketCache.Put(ctx, orgID, query, step, result)
|
||||
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
@@ -884,16 +884,19 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q.bucketCache.Put(ctx, orgID, query, step, result)
|
||||
startMs, endMs := query.Window()
|
||||
q.bucketCache.Put(ctx, orgID, query, step, result, map[qbtypes.TimeRange]qbtypes.ExecStats{{From: startMs, To: endMs}: result.Stats})
|
||||
return result, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate total stats and filter out nil results
|
||||
validResults := make([]*qbtypes.Result, 0, len(freshResults))
|
||||
for _, result := range freshResults {
|
||||
execStatsForQueriedTimeRanges := make(map[qbtypes.TimeRange]qbtypes.ExecStats, len(freshResults))
|
||||
for i, result := range freshResults {
|
||||
if result != nil {
|
||||
validResults = append(validResults, result)
|
||||
execStatsForQueriedTimeRanges[*missingRanges[i]] = result.Stats
|
||||
totalStats.RowsScanned += result.Stats.RowsScanned
|
||||
totalStats.BytesScanned += result.Stats.BytesScanned
|
||||
totalStats.DurationMS += result.Stats.DurationMS
|
||||
@@ -908,7 +911,7 @@ func (q *querier) executeWithCache(ctx context.Context, orgID valuer.UUID, query
|
||||
mergedResult.Stats.DurationMS += totalStats.DurationMS
|
||||
|
||||
// Store merged result in cache
|
||||
q.bucketCache.Put(ctx, orgID, query, step, mergedResult)
|
||||
q.bucketCache.Put(ctx, orgID, query, step, mergedResult, execStatsForQueriedTimeRanges)
|
||||
|
||||
return mergedResult, nil
|
||||
}
|
||||
|
||||
@@ -43,6 +43,23 @@ func (c *CachedBucket) Touches(other *CachedBucket) bool {
|
||||
return c.StartMs <= other.EndMs && other.StartMs <= c.EndMs
|
||||
}
|
||||
|
||||
// ProrateStatsToWindow returns the share of the stats for the part of the
|
||||
// bucket inside [startMs, endMs).
|
||||
func (c *CachedBucket) ProrateStatsToWindow(startMs, endMs uint64) ExecStats {
|
||||
return c.Stats.prorate(overlapMs(c.StartMs, c.EndMs, startMs, endMs), c.EndMs-c.StartMs)
|
||||
}
|
||||
|
||||
// ProrateStatsToUnqueried returns the share of the stats for the part of the
|
||||
// bucket no queried range covers.
|
||||
func (c *CachedBucket) ProrateStatsToUnqueried(execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ExecStats {
|
||||
widthMs := c.EndMs - c.StartMs
|
||||
queriedMs := uint64(0)
|
||||
for queried := range execStatsForQueriedTimeRanges {
|
||||
queriedMs += overlapMs(c.StartMs, c.EndMs, queried.From, queried.To)
|
||||
}
|
||||
return c.Stats.prorate(widthMs-min(queriedMs, widthMs), widthMs)
|
||||
}
|
||||
|
||||
// CachedData represents the full cached data for a query.
|
||||
type CachedData struct {
|
||||
Buckets []*CachedBucket `json:"buckets"`
|
||||
@@ -88,7 +105,7 @@ func (c *CachedData) Cost() int64 {
|
||||
return size
|
||||
}
|
||||
|
||||
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64) ([]*CachedBucket, error) {
|
||||
func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64, execStatsForQueriedTimeRanges map[TimeRange]ExecStats) ([]*CachedBucket, error) {
|
||||
tsData, ok := r.Value.(*TimeSeriesData)
|
||||
if !ok || tsData == nil {
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "result of type %s cannot be bucketed", r.Type.StringValue())
|
||||
@@ -111,18 +128,39 @@ func (r *Result) SplitIntoCacheCellBuckets(startMs, endMs, cellWidthMs uint64) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A slice costs what was queried inside it; the rest of the result came
|
||||
// out of the cache and is costed there already
|
||||
sliceStats := ExecStats{StepIntervals: maps.Clone(r.Stats.StepIntervals)}
|
||||
for queried, stats := range execStatsForQueriedTimeRanges {
|
||||
queriedMs := overlapMs(from, to, queried.From, queried.To)
|
||||
if queriedMs == 0 {
|
||||
continue
|
||||
}
|
||||
share := stats.prorate(queriedMs, queried.To-queried.From)
|
||||
sliceStats.RowsScanned += share.RowsScanned
|
||||
sliceStats.BytesScanned += share.BytesScanned
|
||||
sliceStats.DurationMS += share.DurationMS
|
||||
}
|
||||
cellBuckets = append(cellBuckets, &CachedBucket{
|
||||
StartMs: from,
|
||||
EndMs: to,
|
||||
Type: r.Type,
|
||||
Value: value,
|
||||
Stats: r.Stats.prorate(to-from, endMs-startMs),
|
||||
Stats: sliceStats,
|
||||
})
|
||||
from = to
|
||||
}
|
||||
return cellBuckets, nil
|
||||
}
|
||||
|
||||
// overlapMs is the length of [aFromMs, aToMs) ∩ [bFromMs, bToMs).
|
||||
func overlapMs(aFromMs, aToMs, bFromMs, bToMs uint64) uint64 {
|
||||
if fromMs, toMs := max(aFromMs, bFromMs), min(aToMs, bToMs); fromMs < toMs {
|
||||
return toMs - fromMs
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// prorate returns the share of the stats that partMs out of wholeMs of the
|
||||
// scanned window accounts for.
|
||||
func (e ExecStats) prorate(partMs, wholeMs uint64) ExecStats {
|
||||
|
||||
51
tests/fixtures/fileauditor.py
vendored
51
tests/fixtures/fileauditor.py
vendored
@@ -1,51 +0,0 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
import docker
|
||||
|
||||
from fixtures import types
|
||||
|
||||
AUDIT_FILE_PATH = "/tmp/audit.log"
|
||||
|
||||
|
||||
def list_audit_events(signoz: types.SigNoz) -> list[dict]:
|
||||
"""Each line of the audit file is one OTLP JSON batch of events; this flattens all batches into events."""
|
||||
exit_code, output = docker.from_env().containers.get(signoz.self.id).exec_run(["cat", AUDIT_FILE_PATH])
|
||||
assert exit_code == 0, output.decode()
|
||||
|
||||
events = []
|
||||
for line in output.decode().splitlines():
|
||||
for resource_logs in json.loads(line)["resourceLogs"]:
|
||||
resource = _attributes(resource_logs["resource"]["attributes"])
|
||||
for scope_logs in resource_logs["scopeLogs"]:
|
||||
for log_record in scope_logs["logRecords"]:
|
||||
events.append(
|
||||
{
|
||||
"event_name": log_record["eventName"],
|
||||
"severity_text": log_record["severityText"],
|
||||
"body": log_record["body"]["stringValue"],
|
||||
"scope_name": scope_logs["scope"]["name"],
|
||||
"resource": resource,
|
||||
"attributes": _attributes(log_record["attributes"]),
|
||||
}
|
||||
)
|
||||
|
||||
return events
|
||||
|
||||
|
||||
def wait_for_audit_event(signoz: types.SigNoz, event_name: str, resource_id: str, target_id: str | None = None, wait_seconds: int = 10) -> dict:
|
||||
deadline = time.time() + wait_seconds
|
||||
events = []
|
||||
while time.time() < deadline:
|
||||
events = list_audit_events(signoz)
|
||||
matched = [event for event in events if event["event_name"] == event_name and event["resource"].get("signoz.resource.id") == resource_id and (target_id is None or event["attributes"].get("signoz.resource.target.id") == target_id)]
|
||||
if len(matched) > 0:
|
||||
return matched[0]
|
||||
time.sleep(0.5)
|
||||
|
||||
raise AssertionError(f"No {event_name} audit event for resource {resource_id} (target {target_id}) within {wait_seconds}s, events: {events}")
|
||||
|
||||
|
||||
def _attributes(otlp_attributes: list[dict]) -> dict:
|
||||
# OTLP JSON encodes int64 values as decimal strings.
|
||||
return {attribute["key"]: int(attribute["value"]["intValue"]) if "intValue" in attribute["value"] else attribute["value"]["stringValue"] for attribute in otlp_attributes}
|
||||
@@ -1,151 +0,0 @@
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, find_user_by_email
|
||||
from fixtures.fileauditor import wait_for_audit_event
|
||||
from fixtures.savedview import SAVED_VIEW_BASE, create_saved_view
|
||||
from fixtures.serviceaccount import SERVICE_ACCOUNT_BASE, create_service_account, create_service_account_with_key, delete_service_account, get_first_key_id
|
||||
|
||||
|
||||
def test_create_saved_view_writes_create_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
admin = find_user_by_email(signoz, token, USER_ADMIN_EMAIL)
|
||||
|
||||
view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
|
||||
event = wait_for_audit_event(signoz, "signoz.audit.create", view_id)
|
||||
assert event["severity_text"] == "INFO"
|
||||
assert event["scope_name"] == "signoz.audit"
|
||||
assert event["resource"]["service.name"] == "signoz"
|
||||
assert event["resource"]["signoz.resource.kind"] == "saved-view"
|
||||
assert event["attributes"]["signoz.audit.verb"] == "create"
|
||||
assert event["attributes"]["signoz.audit.outcome"] == "success"
|
||||
assert event["attributes"]["signoz.principal.id"] == admin["id"]
|
||||
assert event["attributes"]["signoz.principal.email"] == USER_ADMIN_EMAIL
|
||||
assert event["attributes"]["signoz.organization.id"] == admin["orgId"]
|
||||
assert event["attributes"]["http.request.method"] == "POST"
|
||||
assert event["attributes"]["http.route"] == "/api/v2/saved_views"
|
||||
assert event["attributes"]["http.response.status_code"] == HTTPStatus.CREATED
|
||||
|
||||
|
||||
def test_update_service_account_writes_update_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
service_account_id = create_service_account(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
|
||||
response = requests.put(
|
||||
signoz.self.host_configs["8080"].get(f"{SERVICE_ACCOUNT_BASE}/{service_account_id}"),
|
||||
json={"name": f"fileauditor-{uuid.uuid4().hex[:8]}"},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=5,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
|
||||
|
||||
event = wait_for_audit_event(signoz, "signoz.audit.update", service_account_id)
|
||||
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
|
||||
assert event["attributes"]["signoz.audit.verb"] == "update"
|
||||
assert event["attributes"]["signoz.audit.outcome"] == "success"
|
||||
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}"
|
||||
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NO_CONTENT
|
||||
|
||||
|
||||
def test_delete_service_account_writes_delete_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
service_account_id = create_service_account(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
|
||||
delete_service_account(signoz, token, service_account_id)
|
||||
|
||||
event = wait_for_audit_event(signoz, "signoz.audit.delete", service_account_id)
|
||||
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
|
||||
assert event["attributes"]["signoz.audit.verb"] == "delete"
|
||||
assert event["attributes"]["signoz.audit.outcome"] == "success"
|
||||
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}"
|
||||
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NO_CONTENT
|
||||
|
||||
|
||||
def test_create_service_account_key_writes_attach_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
service_account_id = create_service_account(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get(f"{SERVICE_ACCOUNT_BASE}/{service_account_id}/keys"),
|
||||
json={"name": "fileauditor", "expiresAt": 0},
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=5,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.CREATED, response.text
|
||||
key_id = response.json()["data"]["id"]
|
||||
|
||||
event = wait_for_audit_event(signoz, "signoz.audit.attach", service_account_id, target_id=key_id)
|
||||
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
|
||||
assert event["attributes"]["signoz.resource.target.kind"] == "factor-api-key"
|
||||
assert event["attributes"]["signoz.audit.verb"] == "attach"
|
||||
assert event["attributes"]["signoz.audit.outcome"] == "success"
|
||||
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}/keys"
|
||||
assert event["attributes"]["http.response.status_code"] == HTTPStatus.CREATED
|
||||
|
||||
|
||||
def test_revoke_service_account_key_writes_detach_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
service_account_id, _ = create_service_account_with_key(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
key_id = get_first_key_id(signoz, token, service_account_id)
|
||||
|
||||
response = requests.delete(
|
||||
signoz.self.host_configs["8080"].get(f"{SERVICE_ACCOUNT_BASE}/{service_account_id}/keys/{key_id}"),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=5,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
|
||||
|
||||
event = wait_for_audit_event(signoz, "signoz.audit.detach", service_account_id, target_id=key_id)
|
||||
assert event["resource"]["signoz.resource.kind"] == "serviceaccount"
|
||||
assert event["attributes"]["signoz.resource.target.kind"] == "factor-api-key"
|
||||
assert event["attributes"]["signoz.audit.verb"] == "detach"
|
||||
assert event["attributes"]["signoz.audit.outcome"] == "success"
|
||||
assert event["attributes"]["http.route"] == "/api/v1/service_accounts/{id}/keys/{fid}"
|
||||
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NO_CONTENT
|
||||
|
||||
|
||||
def test_delete_missing_saved_view_writes_failure_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
missing_view_id = str(uuid.uuid4())
|
||||
|
||||
response = requests.delete(
|
||||
signoz.self.host_configs["8080"].get(f"{SAVED_VIEW_BASE}/{missing_view_id}"),
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=5,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND, response.text
|
||||
|
||||
event = wait_for_audit_event(signoz, "signoz.audit.delete", missing_view_id)
|
||||
assert event["severity_text"] == "ERROR"
|
||||
assert event["attributes"]["signoz.audit.outcome"] == "failure"
|
||||
assert event["attributes"]["exception.type"] == "not-found"
|
||||
assert event["attributes"]["http.response.status_code"] == HTTPStatus.NOT_FOUND
|
||||
@@ -1,49 +0,0 @@
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
|
||||
from sqlalchemy import sql
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.fileauditor import list_audit_events, wait_for_audit_event
|
||||
from fixtures.savedview import create_saved_view
|
||||
|
||||
|
||||
def test_unlicensed_create_writes_no_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
with signoz.sqlstore.conn.begin() as conn:
|
||||
licenses = conn.execute(sql.text("SELECT id, created_at, updated_at, key, data, last_validated_at, org_id FROM license")).mappings().all()
|
||||
assert len(licenses) > 0
|
||||
conn.execute(sql.text("DELETE FROM license"))
|
||||
|
||||
try:
|
||||
unlicensed_view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
finally:
|
||||
with signoz.sqlstore.conn.begin() as conn:
|
||||
conn.execute(
|
||||
sql.text("INSERT INTO license (id, created_at, updated_at, key, data, last_validated_at, org_id) VALUES (:id, :created_at, :updated_at, :key, :data, :last_validated_at, :org_id)"),
|
||||
[dict(row) for row in licenses],
|
||||
)
|
||||
|
||||
marker_view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
wait_for_audit_event(signoz, "signoz.audit.create", marker_view_id)
|
||||
|
||||
assert all(event["resource"].get("signoz.resource.id") != unlicensed_view_id for event in list_audit_events(signoz))
|
||||
|
||||
|
||||
def test_login_writes_no_event(
|
||||
signoz: types.SigNoz,
|
||||
apply_license: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
marker_view_id = create_saved_view(signoz, token, f"fileauditor-{uuid.uuid4().hex[:8]}")
|
||||
wait_for_audit_event(signoz, "signoz.audit.create", marker_view_id)
|
||||
|
||||
assert all(event["attributes"].get("http.route") != "/api/v2/sessions/email_password" for event in list_audit_events(signoz))
|
||||
@@ -1,32 +0,0 @@
|
||||
import pytest
|
||||
from testcontainers.core.container import Network
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.fileauditor import AUDIT_FILE_PATH
|
||||
from fixtures.signoz import create_signoz
|
||||
|
||||
|
||||
@pytest.fixture(name="signoz", scope="package")
|
||||
def signoz_fileauditor(
|
||||
network: Network,
|
||||
zeus: types.TestContainerDocker,
|
||||
gateway: types.TestContainerDocker,
|
||||
sqlstore: types.TestContainerSQL,
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.SigNoz:
|
||||
return create_signoz(
|
||||
network=network,
|
||||
zeus=zeus,
|
||||
gateway=gateway,
|
||||
sqlstore=sqlstore,
|
||||
clickhouse=clickhouse,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="signoz-fileauditor",
|
||||
env_overrides={
|
||||
"SIGNOZ_AUDITOR_PROVIDER": "file",
|
||||
"SIGNOZ_AUDITOR_FILE_PATH": AUDIT_FILE_PATH,
|
||||
},
|
||||
)
|
||||
@@ -442,3 +442,55 @@ def test_builder_refreshing_a_sliding_time_range(
|
||||
assert uncached.status_code == HTTPStatus.OK, uncached.text
|
||||
|
||||
assert_results_equal(from_cache.json(), uncached.json(), "A", "the twentieth refresh")
|
||||
|
||||
|
||||
def test_builder_refreshing_does_not_inflate_rows_scanned(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
metric_name = f"cache_rows_scanned_{uuid4().hex[:8]}"
|
||||
|
||||
# 90 minutes back so even the last refresh closes clear of the flux
|
||||
# interval, which holds recent data out of the cache
|
||||
start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=90)).timestamp()) // 60 * 60, tz=UTC)
|
||||
start_time_ms = int(start_time.timestamp() * 1000)
|
||||
|
||||
query = [build_builder_query("A", metric_name, "max", "max")]
|
||||
|
||||
insert_metrics(
|
||||
[
|
||||
Metrics(
|
||||
metric_name=metric_name,
|
||||
labels={"service": "api"},
|
||||
timestamp=start_time + timedelta(minutes=minute),
|
||||
value=minute,
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
)
|
||||
for minute in range(70)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# a dashboard left open on a thirty minute range, re-running a minute later
|
||||
# each time. rowsScanned is a cost indicator, so a refresh served mostly out
|
||||
# of the cache cannot claim more than every read made so far: the first
|
||||
# full read and then one new minute per refresh, measured with the cache off
|
||||
uncached_rows_scanned_so_far = 0
|
||||
for refresh in range(10):
|
||||
refresh_start_ms = start_time_ms + refresh * MINUTE_MS
|
||||
refresh_end_ms = refresh_start_ms + 30 * MINUTE_MS
|
||||
|
||||
uncached_start_ms = refresh_start_ms if refresh == 0 else refresh_end_ms - MINUTE_MS
|
||||
uncached_response = make_query_request(signoz, token, uncached_start_ms, refresh_end_ms, query, no_cache=True)
|
||||
assert uncached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: uncached read failed: {uncached_response.text}"
|
||||
uncached_rows_scanned_so_far += uncached_response.json()["data"]["meta"]["rowsScanned"]
|
||||
assert uncached_rows_scanned_so_far > 0, f"refresh {refresh}: uncached read reported no rows scanned"
|
||||
|
||||
cached_response = make_query_request(signoz, token, refresh_start_ms, refresh_end_ms, query, no_cache=False)
|
||||
assert cached_response.status_code == HTTPStatus.OK, f"refresh {refresh}: cached read failed: {cached_response.text}"
|
||||
cached_rows_scanned = cached_response.json()["data"]["meta"]["rowsScanned"]
|
||||
assert cached_rows_scanned <= uncached_rows_scanned_so_far, f"refresh {refresh}: cached read reported {cached_rows_scanned} rows scanned, more than the {uncached_rows_scanned_so_far} rows all uncached reads scanned so far"
|
||||
|
||||
Reference in New Issue
Block a user