mirror of
https://github.com/SigNoz/signoz.git
synced 2026-03-27 14:40:25 +00:00
Compare commits
7 Commits
issue_3017
...
fix/hosts-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2314c6ab6c | ||
|
|
64dab58fad | ||
|
|
d95a2264d6 | ||
|
|
a63e3408d6 | ||
|
|
558de5a950 | ||
|
|
7f9ad2ac0a | ||
|
|
3cc9ab6265 |
@@ -190,7 +190,7 @@ services:
|
||||
# - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
|
||||
signoz:
|
||||
!!merge <<: *db-depend
|
||||
image: signoz/signoz:v0.117.1
|
||||
image: signoz/signoz:v0.117.0
|
||||
ports:
|
||||
- "8080:8080" # signoz port
|
||||
# - "6060:6060" # pprof port
|
||||
|
||||
@@ -117,7 +117,7 @@ services:
|
||||
# - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
|
||||
signoz:
|
||||
!!merge <<: *db-depend
|
||||
image: signoz/signoz:v0.117.1
|
||||
image: signoz/signoz:v0.117.0
|
||||
ports:
|
||||
- "8080:8080" # signoz port
|
||||
volumes:
|
||||
|
||||
@@ -181,7 +181,7 @@ services:
|
||||
# - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
|
||||
signoz:
|
||||
!!merge <<: *db-depend
|
||||
image: signoz/signoz:${VERSION:-v0.117.1}
|
||||
image: signoz/signoz:${VERSION:-v0.117.0}
|
||||
container_name: signoz
|
||||
ports:
|
||||
- "8080:8080" # signoz port
|
||||
|
||||
@@ -109,7 +109,7 @@ services:
|
||||
# - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
|
||||
signoz:
|
||||
!!merge <<: *db-depend
|
||||
image: signoz/signoz:${VERSION:-v0.117.1}
|
||||
image: signoz/signoz:${VERSION:-v0.117.0}
|
||||
container_name: signoz
|
||||
ports:
|
||||
- "8080:8080" # signoz port
|
||||
|
||||
@@ -116,12 +116,7 @@ describe.each([
|
||||
expect(screen.getByRole('dialog')).toBeInTheDocument();
|
||||
expect(screen.getByText('FORMAT')).toBeInTheDocument();
|
||||
expect(screen.getByText('Number of Rows')).toBeInTheDocument();
|
||||
|
||||
if (dataSource === DataSource.TRACES) {
|
||||
expect(screen.queryByText('Columns')).not.toBeInTheDocument();
|
||||
} else {
|
||||
expect(screen.getByText('Columns')).toBeInTheDocument();
|
||||
}
|
||||
expect(screen.getByText('Columns')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('allows changing export format', () => {
|
||||
@@ -151,17 +146,6 @@ describe.each([
|
||||
});
|
||||
|
||||
it('allows changing columns scope', () => {
|
||||
if (dataSource === DataSource.TRACES) {
|
||||
renderWithStore(dataSource);
|
||||
fireEvent.click(screen.getByTestId(testId));
|
||||
|
||||
expect(screen.queryByRole('radio', { name: 'All' })).not.toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByRole('radio', { name: 'Selected' }),
|
||||
).not.toBeInTheDocument();
|
||||
return;
|
||||
}
|
||||
|
||||
renderWithStore(dataSource);
|
||||
fireEvent.click(screen.getByTestId(testId));
|
||||
|
||||
@@ -226,12 +210,7 @@ describe.each([
|
||||
mockUseQueryBuilder.mockReturnValue({ stagedQuery: mockQuery });
|
||||
renderWithStore(dataSource);
|
||||
fireEvent.click(screen.getByTestId(testId));
|
||||
|
||||
// For traces, column scope is always Selected and the radio is hidden
|
||||
if (dataSource !== DataSource.TRACES) {
|
||||
fireEvent.click(screen.getByRole('radio', { name: 'Selected' }));
|
||||
}
|
||||
|
||||
fireEvent.click(screen.getByRole('radio', { name: 'Selected' }));
|
||||
fireEvent.click(screen.getByText('Export'));
|
||||
|
||||
await waitFor(() => {
|
||||
@@ -248,11 +227,6 @@ describe.each([
|
||||
});
|
||||
|
||||
it('sends no selectFields when column scope is All', async () => {
|
||||
// For traces, column scope is always Selected — this test only applies to other sources
|
||||
if (dataSource === DataSource.TRACES) {
|
||||
return;
|
||||
}
|
||||
|
||||
renderWithStore(dataSource);
|
||||
fireEvent.click(screen.getByTestId(testId));
|
||||
fireEvent.click(screen.getByRole('radio', { name: 'All' }));
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { useCallback, useMemo, useState } from 'react';
|
||||
import { Button, Popover, Radio, Tooltip, Typography } from 'antd';
|
||||
import { TelemetryFieldKey } from 'api/v5/v5';
|
||||
import { useExportRawData } from 'hooks/useDownloadOptionsMenu/useDownloadOptionsMenu';
|
||||
import { Download, DownloadIcon, Loader2 } from 'lucide-react';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
@@ -15,12 +14,10 @@ import './DownloadOptionsMenu.styles.scss';
|
||||
|
||||
interface DownloadOptionsMenuProps {
|
||||
dataSource: DataSource;
|
||||
selectedColumns?: TelemetryFieldKey[];
|
||||
}
|
||||
|
||||
export default function DownloadOptionsMenu({
|
||||
dataSource,
|
||||
selectedColumns,
|
||||
}: DownloadOptionsMenuProps): JSX.Element {
|
||||
const [exportFormat, setExportFormat] = useState<string>(DownloadFormats.CSV);
|
||||
const [rowLimit, setRowLimit] = useState<number>(DownloadRowCounts.TEN_K);
|
||||
@@ -38,19 +35,9 @@ export default function DownloadOptionsMenu({
|
||||
await handleExportRawData({
|
||||
format: exportFormat,
|
||||
rowLimit,
|
||||
clearSelectColumns:
|
||||
dataSource !== DataSource.TRACES &&
|
||||
columnsScope === DownloadColumnsScopes.ALL,
|
||||
selectedColumns,
|
||||
clearSelectColumns: columnsScope === DownloadColumnsScopes.ALL,
|
||||
});
|
||||
}, [
|
||||
exportFormat,
|
||||
rowLimit,
|
||||
columnsScope,
|
||||
selectedColumns,
|
||||
handleExportRawData,
|
||||
dataSource,
|
||||
]);
|
||||
}, [exportFormat, rowLimit, columnsScope, handleExportRawData]);
|
||||
|
||||
const popoverContent = useMemo(
|
||||
() => (
|
||||
@@ -85,22 +72,18 @@ export default function DownloadOptionsMenu({
|
||||
</Radio.Group>
|
||||
</div>
|
||||
|
||||
{dataSource !== DataSource.TRACES && (
|
||||
<>
|
||||
<div className="horizontal-line" />
|
||||
<div className="horizontal-line" />
|
||||
|
||||
<div className="columns-scope">
|
||||
<Typography.Text className="title">Columns</Typography.Text>
|
||||
<Radio.Group
|
||||
value={columnsScope}
|
||||
onChange={(e): void => setColumnsScope(e.target.value)}
|
||||
>
|
||||
<Radio value={DownloadColumnsScopes.ALL}>All</Radio>
|
||||
<Radio value={DownloadColumnsScopes.SELECTED}>Selected</Radio>
|
||||
</Radio.Group>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
<div className="columns-scope">
|
||||
<Typography.Text className="title">Columns</Typography.Text>
|
||||
<Radio.Group
|
||||
value={columnsScope}
|
||||
onChange={(e): void => setColumnsScope(e.target.value)}
|
||||
>
|
||||
<Radio value={DownloadColumnsScopes.ALL}>All</Radio>
|
||||
<Radio value={DownloadColumnsScopes.SELECTED}>Selected</Radio>
|
||||
</Radio.Group>
|
||||
</div>
|
||||
|
||||
<Button
|
||||
type="primary"
|
||||
@@ -114,14 +97,7 @@ export default function DownloadOptionsMenu({
|
||||
</Button>
|
||||
</div>
|
||||
),
|
||||
[
|
||||
exportFormat,
|
||||
rowLimit,
|
||||
columnsScope,
|
||||
isDownloading,
|
||||
handleExport,
|
||||
dataSource,
|
||||
],
|
||||
[exportFormat, rowLimit, columnsScope, isDownloading, handleExport],
|
||||
);
|
||||
|
||||
return (
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
ScrollText,
|
||||
X,
|
||||
} from 'lucide-react';
|
||||
import { parseAsString, useQueryState } from 'nuqs';
|
||||
import { AppState } from 'store/reducers';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import {
|
||||
@@ -52,15 +53,18 @@ import {
|
||||
import { GlobalReducer } from 'types/reducer/globalTime';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
import { convertFiltersToExpression } from '../QueryBuilderV2/utils';
|
||||
import { VIEW_TYPES, VIEWS } from './constants';
|
||||
import Containers from './Containers/Containers';
|
||||
import { HostDetailProps } from './HostMetricDetail.interfaces';
|
||||
import HostMetricLogsDetailedView from './HostMetricsLogs/HostMetricLogsDetailedView';
|
||||
import { HOST_METRICS_LOGS_EXPR_QUERY_KEY } from './HostMetricsLogs/constants';
|
||||
import HostMetricsLogs from './HostMetricsLogs/HostMetricsLogs';
|
||||
import HostMetricTraces from './HostMetricTraces/HostMetricTraces';
|
||||
import Metrics from './Metrics/Metrics';
|
||||
import Processes from './Processes/Processes';
|
||||
|
||||
import './HostMetricsDetail.styles.scss';
|
||||
|
||||
// eslint-disable-next-line sonarjs/cognitive-complexity
|
||||
function HostMetricsDetails({
|
||||
host,
|
||||
@@ -129,10 +133,6 @@ function HostMetricsDetails({
|
||||
};
|
||||
}, [host?.hostName, searchParams]);
|
||||
|
||||
const [logFilters, setLogFilters] = useState<IBuilderQuery['filters']>(
|
||||
initialFilters,
|
||||
);
|
||||
|
||||
const [tracesFilters, setTracesFilters] = useState<IBuilderQuery['filters']>(
|
||||
initialFilters,
|
||||
);
|
||||
@@ -147,7 +147,6 @@ function HostMetricsDetails({
|
||||
}, [host]);
|
||||
|
||||
useEffect(() => {
|
||||
setLogFilters(initialFilters);
|
||||
setTracesFilters(initialFilters);
|
||||
}, [initialFilters]);
|
||||
|
||||
@@ -172,7 +171,6 @@ function HostMetricsDetails({
|
||||
setSearchParams({
|
||||
...Object.fromEntries(searchParams.entries()),
|
||||
[INFRA_MONITORING_K8S_PARAMS_KEYS.VIEW]: e.target.value,
|
||||
[INFRA_MONITORING_K8S_PARAMS_KEYS.LOG_FILTERS]: JSON.stringify(null),
|
||||
[INFRA_MONITORING_K8S_PARAMS_KEYS.TRACES_FILTERS]: JSON.stringify(null),
|
||||
});
|
||||
}
|
||||
@@ -210,48 +208,30 @@ function HostMetricsDetails({
|
||||
[],
|
||||
);
|
||||
|
||||
const handleChangeLogFilters = useCallback(
|
||||
(value: IBuilderQuery['filters'], view: VIEWS) => {
|
||||
setLogFilters((prevFilters) => {
|
||||
const hostNameFilter = prevFilters?.items?.find(
|
||||
(item) => item.key?.key === 'host.name',
|
||||
);
|
||||
const paginationFilter = value?.items?.find(
|
||||
(item) => item.key?.key === 'id',
|
||||
);
|
||||
const newFilters = value?.items?.filter(
|
||||
(item) => item.key?.key !== 'id' && item.key?.key !== 'host.name',
|
||||
);
|
||||
const initialLogsExpression = useMemo(
|
||||
() =>
|
||||
convertFiltersToExpression({
|
||||
items: [
|
||||
{
|
||||
id: uuidv4(),
|
||||
key: {
|
||||
key: 'host.name',
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
id: 'host.name--string--resource--false',
|
||||
},
|
||||
op: '=',
|
||||
value: host?.hostName || '',
|
||||
},
|
||||
],
|
||||
op: 'AND',
|
||||
}).expression,
|
||||
[host?.hostName],
|
||||
);
|
||||
|
||||
if (newFilters && newFilters?.length > 0) {
|
||||
logEvent(InfraMonitoringEvents.FilterApplied, {
|
||||
entity: InfraMonitoringEvents.HostEntity,
|
||||
view: InfraMonitoringEvents.LogsView,
|
||||
page: InfraMonitoringEvents.DetailedPage,
|
||||
});
|
||||
}
|
||||
|
||||
const updatedFilters = {
|
||||
op: 'AND',
|
||||
items: [
|
||||
hostNameFilter,
|
||||
...(newFilters || []),
|
||||
...(paginationFilter ? [paginationFilter] : []),
|
||||
].filter((item): item is TagFilterItem => item !== undefined),
|
||||
};
|
||||
|
||||
setSearchParams({
|
||||
...Object.fromEntries(searchParams.entries()),
|
||||
[INFRA_MONITORING_K8S_PARAMS_KEYS.LOG_FILTERS]: JSON.stringify(
|
||||
updatedFilters,
|
||||
),
|
||||
[INFRA_MONITORING_K8S_PARAMS_KEYS.VIEW]: view,
|
||||
});
|
||||
return updatedFilters;
|
||||
});
|
||||
},
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
[],
|
||||
const [hostMetricLogsExpr] = useQueryState(
|
||||
HOST_METRICS_LOGS_EXPR_QUERY_KEY,
|
||||
parseAsString,
|
||||
);
|
||||
|
||||
const handleChangeTracesFilters = useCallback(
|
||||
@@ -308,11 +288,6 @@ function HostMetricsDetails({
|
||||
});
|
||||
|
||||
if (selectedView === VIEW_TYPES.LOGS) {
|
||||
const filtersWithoutPagination = {
|
||||
...logFilters,
|
||||
items: logFilters?.items?.filter((item) => item.key?.key !== 'id') || [],
|
||||
};
|
||||
|
||||
const compositeQuery = {
|
||||
...initialQueryState,
|
||||
queryType: 'builder',
|
||||
@@ -322,7 +297,11 @@ function HostMetricsDetails({
|
||||
{
|
||||
...initialQueryBuilderFormValuesMap.logs,
|
||||
aggregateOperator: LogsAggregatorOperator.NOOP,
|
||||
filters: filtersWithoutPagination,
|
||||
filter: { expression: hostMetricLogsExpr },
|
||||
expression: hostMetricLogsExpr,
|
||||
having: {
|
||||
expression: '',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -564,12 +543,11 @@ function HostMetricsDetails({
|
||||
/>
|
||||
)}
|
||||
{selectedView === VIEW_TYPES.LOGS && (
|
||||
<HostMetricLogsDetailedView
|
||||
<HostMetricsLogs
|
||||
timeRange={modalTimeRange}
|
||||
isModalTimeSelection={isModalTimeSelection}
|
||||
handleTimeChange={handleTimeChange}
|
||||
handleChangeLogFilters={handleChangeLogFilters}
|
||||
logFilters={logFilters}
|
||||
initialExpression={initialLogsExpression}
|
||||
selectedInterval={selectedInterval}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
.header {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
padding: var(--spacing-4) 0px;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.logs {
|
||||
border: 1px solid var(--border);
|
||||
margin-top: var(--spacing-4);
|
||||
}
|
||||
|
||||
.listContainer {
|
||||
flex: 1;
|
||||
height: calc(100vh - 278px) !important;
|
||||
display: flex;
|
||||
height: 100%;
|
||||
|
||||
:global(.raw-log-content) {
|
||||
width: 100%;
|
||||
text-wrap: inherit;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
}
|
||||
|
||||
.listCard {
|
||||
width: 100%;
|
||||
margin-top: 12px;
|
||||
|
||||
:global(.ant-card-body) {
|
||||
padding: 0;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.logsLoadingSkeleton {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
padding: 8px 0;
|
||||
|
||||
:global(.ant-skeleton-input-sm) {
|
||||
height: 18px;
|
||||
}
|
||||
}
|
||||
|
||||
.noLogsFound {
|
||||
height: 50vh;
|
||||
width: 100%;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
padding: 24px;
|
||||
box-sizing: border-box;
|
||||
|
||||
p {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
}
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
.host-metrics-logs-container {
|
||||
margin-top: 1rem;
|
||||
|
||||
.filter-section {
|
||||
flex: 1;
|
||||
|
||||
.ant-select-selector {
|
||||
border-radius: 2px;
|
||||
border: 1px solid var(--bg-slate-400) !important;
|
||||
background-color: var(--bg-ink-300) !important;
|
||||
|
||||
input {
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.ant-tag .ant-typography {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.host-metrics-logs-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 8px;
|
||||
|
||||
padding: 12px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid var(--bg-slate-500);
|
||||
}
|
||||
|
||||
.host-metrics-logs {
|
||||
margin-top: 1rem;
|
||||
|
||||
.virtuoso-list {
|
||||
overflow-y: hidden !important;
|
||||
|
||||
&::-webkit-scrollbar {
|
||||
width: 0.3rem;
|
||||
height: 0.3rem;
|
||||
}
|
||||
|
||||
&::-webkit-scrollbar-track {
|
||||
background: transparent;
|
||||
}
|
||||
|
||||
&::-webkit-scrollbar-thumb {
|
||||
background: var(--bg-slate-300);
|
||||
}
|
||||
|
||||
&::-webkit-scrollbar-thumb:hover {
|
||||
background: var(--bg-slate-200);
|
||||
}
|
||||
|
||||
.ant-row {
|
||||
width: fit-content;
|
||||
}
|
||||
}
|
||||
|
||||
.skeleton-container {
|
||||
height: 100%;
|
||||
padding: 16px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.host-metrics-logs-list-container {
|
||||
flex: 1;
|
||||
height: calc(100vh - 272px) !important;
|
||||
display: flex;
|
||||
height: 100%;
|
||||
|
||||
.raw-log-content {
|
||||
width: 100%;
|
||||
text-wrap: inherit;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
}
|
||||
|
||||
.host-metrics-logs-list-card {
|
||||
width: 100%;
|
||||
margin-top: 12px;
|
||||
|
||||
.ant-card-body {
|
||||
padding: 0;
|
||||
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
.logs-loading-skeleton {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 8px;
|
||||
padding: 8px 0;
|
||||
|
||||
.ant-skeleton-input-sm {
|
||||
height: 18px;
|
||||
}
|
||||
}
|
||||
|
||||
.no-logs-found {
|
||||
height: 50vh;
|
||||
width: 100%;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
|
||||
padding: 24px;
|
||||
box-sizing: border-box;
|
||||
|
||||
.ant-typography {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
}
|
||||
}
|
||||
|
||||
.lightMode {
|
||||
.filter-section {
|
||||
border-top: 1px solid var(--bg-vanilla-300);
|
||||
border-bottom: 1px solid var(--bg-vanilla-300);
|
||||
|
||||
.ant-select-selector {
|
||||
border-color: var(--bg-vanilla-300) !important;
|
||||
background-color: var(--bg-vanilla-100) !important;
|
||||
color: var(--bg-ink-200);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
import { useMemo } from 'react';
|
||||
import QueryBuilderSearch from 'container/QueryBuilder/filters/QueryBuilderSearch';
|
||||
import DateTimeSelectionV2 from 'container/TopNav/DateTimeSelectionV2';
|
||||
import {
|
||||
CustomTimeType,
|
||||
Time,
|
||||
} from 'container/TopNav/DateTimeSelectionV2/types';
|
||||
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
|
||||
import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
|
||||
import { VIEWS } from '../constants';
|
||||
import HostMetricsLogs from './HostMetricsLogs';
|
||||
|
||||
import './HostMetricLogs.styles.scss';
|
||||
|
||||
interface Props {
|
||||
timeRange: {
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
};
|
||||
isModalTimeSelection: boolean;
|
||||
handleTimeChange: (
|
||||
interval: Time | CustomTimeType,
|
||||
dateTimeRange?: [number, number],
|
||||
) => void;
|
||||
handleChangeLogFilters: (value: IBuilderQuery['filters'], view: VIEWS) => void;
|
||||
logFilters: IBuilderQuery['filters'];
|
||||
selectedInterval: Time;
|
||||
}
|
||||
|
||||
function HostMetricLogsDetailedView({
|
||||
timeRange,
|
||||
isModalTimeSelection,
|
||||
handleTimeChange,
|
||||
handleChangeLogFilters,
|
||||
logFilters,
|
||||
selectedInterval,
|
||||
}: Props): JSX.Element {
|
||||
const { currentQuery } = useQueryBuilder();
|
||||
const updatedCurrentQuery = useMemo(
|
||||
() => ({
|
||||
...currentQuery,
|
||||
builder: {
|
||||
...currentQuery.builder,
|
||||
queryData: [
|
||||
{
|
||||
...currentQuery.builder.queryData[0],
|
||||
dataSource: DataSource.LOGS,
|
||||
aggregateOperator: 'noop',
|
||||
aggregateAttribute: {
|
||||
...currentQuery.builder.queryData[0].aggregateAttribute,
|
||||
},
|
||||
filters: {
|
||||
items:
|
||||
logFilters?.items?.filter((item) => item.key?.key !== 'host.name') ||
|
||||
[],
|
||||
op: 'AND',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
[currentQuery, logFilters?.items],
|
||||
);
|
||||
|
||||
const query = updatedCurrentQuery?.builder?.queryData[0] || null;
|
||||
|
||||
return (
|
||||
<div className="host-metrics-logs-container">
|
||||
<div className="host-metrics-logs-header">
|
||||
<div className="filter-section">
|
||||
{query && (
|
||||
<QueryBuilderSearch
|
||||
query={query as IBuilderQuery}
|
||||
onChange={(value): void => handleChangeLogFilters(value, VIEWS.LOGS)}
|
||||
disableNavigationShortcuts
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
<div className="datetime-section">
|
||||
<DateTimeSelectionV2
|
||||
showAutoRefresh
|
||||
showRefreshText={false}
|
||||
hideShareModal
|
||||
isModalTimeSelection={isModalTimeSelection}
|
||||
onTimeChange={handleTimeChange}
|
||||
defaultRelativeTime="5m"
|
||||
modalSelectedInterval={selectedInterval}
|
||||
modalInitialStartTime={timeRange.startTime * 1000}
|
||||
modalInitialEndTime={timeRange.endTime * 1000}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<HostMetricsLogs timeRange={timeRange} filters={logFilters} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default HostMetricLogsDetailedView;
|
||||
@@ -1,88 +1,161 @@
|
||||
import { useCallback, useEffect, useMemo, useRef } from 'react';
|
||||
import { useQuery } from 'react-query';
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { Virtuoso, VirtuosoHandle } from 'react-virtuoso';
|
||||
import { Card } from 'antd';
|
||||
import logEvent from 'api/common/logEvent';
|
||||
import LogDetail from 'components/LogDetail';
|
||||
import RawLogView from 'components/Logs/RawLogView';
|
||||
import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
|
||||
import { DEFAULT_ENTITY_VERSION } from 'constants/app';
|
||||
import QuerySearch from 'components/QueryBuilderV2/QueryV2/QuerySearch/QuerySearch';
|
||||
import { InfraMonitoringEvents } from 'constants/events';
|
||||
import LogsError from 'container/LogsError/LogsError';
|
||||
import { LogsLoading } from 'container/LogsLoading/LogsLoading';
|
||||
import { FontSize } from 'container/OptionsMenu/types';
|
||||
import { useHandleLogsPagination } from 'hooks/infraMonitoring/useHandleLogsPagination';
|
||||
import DateTimeSelectionV2 from 'container/TopNav/DateTimeSelectionV2';
|
||||
import {
|
||||
CustomTimeType,
|
||||
Time,
|
||||
} from 'container/TopNav/DateTimeSelectionV2/types';
|
||||
import { getOldLogsOperatorFromNew } from 'hooks/logs/useActiveLog';
|
||||
import useLogDetailHandlers from 'hooks/logs/useLogDetailHandlers';
|
||||
import useScrollToLog from 'hooks/logs/useScrollToLog';
|
||||
import { GetMetricQueryRange } from 'lib/dashboard/getQueryResults';
|
||||
import useDebounce from 'hooks/useDebounce';
|
||||
import { generateFilterQuery } from 'lib/logs/generateFilterQuery';
|
||||
import { parseAsString, useQueryState } from 'nuqs';
|
||||
import { ILog } from 'types/api/logs/log';
|
||||
import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { DataSource } from 'types/common/queryBuilder';
|
||||
import { validateQuery } from 'utils/queryValidationUtils';
|
||||
|
||||
import { getHostLogsQueryPayload } from './constants';
|
||||
import {
|
||||
getHostLogsQueryPayload,
|
||||
HOST_METRICS_LOGS_EXPR_QUERY_KEY,
|
||||
} from './constants';
|
||||
import { useInfiniteHostMetricLogs } from './hooks';
|
||||
import NoLogsContainer from './NoLogsContainer';
|
||||
|
||||
import './HostMetricLogs.styles.scss';
|
||||
import styles from './HostMetricLogs.module.scss';
|
||||
|
||||
interface Props {
|
||||
initialExpression: string;
|
||||
timeRange: {
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
};
|
||||
filters: IBuilderQuery['filters'];
|
||||
isModalTimeSelection: boolean;
|
||||
handleTimeChange: (
|
||||
interval: Time | CustomTimeType,
|
||||
dateTimeRange?: [number, number],
|
||||
) => void;
|
||||
selectedInterval: Time;
|
||||
}
|
||||
|
||||
function HostMetricsLogs({ timeRange, filters }: Props): JSX.Element {
|
||||
const EXPRESSION_DEBOUNCE_TIME_MS = 300;
|
||||
|
||||
function HostMetricsLogs({
|
||||
initialExpression,
|
||||
timeRange,
|
||||
isModalTimeSelection,
|
||||
handleTimeChange,
|
||||
selectedInterval,
|
||||
}: Props): JSX.Element {
|
||||
const virtuosoRef = useRef<VirtuosoHandle>(null);
|
||||
|
||||
const [filterExpression, setFilterExpression] = useQueryState(
|
||||
HOST_METRICS_LOGS_EXPR_QUERY_KEY,
|
||||
parseAsString,
|
||||
);
|
||||
|
||||
const [inputExpression, setInputExpression] = useState(
|
||||
filterExpression || initialExpression,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
// If expression is present in the URL, prefer it and don't override it.
|
||||
// Otherwise, initialize URL state from the host's default expression.
|
||||
if (filterExpression) {
|
||||
setInputExpression(filterExpression);
|
||||
return;
|
||||
}
|
||||
|
||||
setInputExpression(initialExpression);
|
||||
setFilterExpression(initialExpression);
|
||||
}, [filterExpression, initialExpression, setFilterExpression]);
|
||||
|
||||
const debouncedFilterExpression = useDebounce(
|
||||
filterExpression?.trim() || initialExpression,
|
||||
EXPRESSION_DEBOUNCE_TIME_MS,
|
||||
);
|
||||
|
||||
const {
|
||||
activeLog,
|
||||
onAddToQuery,
|
||||
selectedTab,
|
||||
handleSetActiveLog,
|
||||
handleCloseLogDetail,
|
||||
} = useLogDetailHandlers();
|
||||
|
||||
const basePayload = getHostLogsQueryPayload(
|
||||
timeRange.startTime,
|
||||
timeRange.endTime,
|
||||
filters,
|
||||
const onAddToQuery = useCallback(
|
||||
(fieldKey: string, fieldValue: string, operator: string): void => {
|
||||
handleCloseLogDetail();
|
||||
|
||||
const partExpression = generateFilterQuery({
|
||||
fieldKey,
|
||||
fieldValue,
|
||||
type: getOldLogsOperatorFromNew(operator),
|
||||
});
|
||||
|
||||
const newExpression = inputExpression.trim()
|
||||
? `${inputExpression} AND ${partExpression}`
|
||||
: partExpression;
|
||||
|
||||
setInputExpression(newExpression);
|
||||
setFilterExpression(newExpression);
|
||||
},
|
||||
[inputExpression, setFilterExpression, handleCloseLogDetail],
|
||||
);
|
||||
|
||||
const handleFilterChange = useCallback(
|
||||
(expression: string): void => {
|
||||
setInputExpression(expression);
|
||||
|
||||
const validation = validateQuery(expression);
|
||||
if (validation.isValid) {
|
||||
setFilterExpression(expression);
|
||||
|
||||
logEvent(InfraMonitoringEvents.FilterApplied, {
|
||||
entity: InfraMonitoringEvents.HostEntity,
|
||||
view: InfraMonitoringEvents.LogsView,
|
||||
page: InfraMonitoringEvents.DetailedPage,
|
||||
});
|
||||
}
|
||||
},
|
||||
[setFilterExpression],
|
||||
);
|
||||
|
||||
const queryData = useMemo(
|
||||
() =>
|
||||
getHostLogsQueryPayload({
|
||||
start: timeRange.startTime,
|
||||
end: timeRange.endTime,
|
||||
// this should use inputExpression to show suggestions correctly
|
||||
// while we don't accept the final expression yet
|
||||
expression: inputExpression,
|
||||
}).queryData,
|
||||
[timeRange.startTime, timeRange.endTime, inputExpression],
|
||||
);
|
||||
|
||||
const {
|
||||
logs,
|
||||
hasReachedEndOfLogs,
|
||||
isPaginating,
|
||||
currentPage,
|
||||
setIsPaginating,
|
||||
handleNewData,
|
||||
loadMoreLogs,
|
||||
queryPayload,
|
||||
} = useHandleLogsPagination({
|
||||
timeRange,
|
||||
filters,
|
||||
excludeFilterKeys: ['host.name'],
|
||||
basePayload,
|
||||
hasNextPage,
|
||||
isFetchingNextPage,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isError,
|
||||
} = useInfiniteHostMetricLogs({
|
||||
expression: debouncedFilterExpression,
|
||||
startTime: timeRange.startTime,
|
||||
endTime: timeRange.endTime,
|
||||
});
|
||||
|
||||
const { data, isLoading, isFetching, isError } = useQuery({
|
||||
queryKey: [
|
||||
'hostMetricsLogs',
|
||||
timeRange.startTime,
|
||||
timeRange.endTime,
|
||||
filters,
|
||||
currentPage,
|
||||
],
|
||||
queryFn: () => GetMetricQueryRange(queryPayload, DEFAULT_ENTITY_VERSION),
|
||||
enabled: !!queryPayload,
|
||||
keepPreviousData: isPaginating,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (data?.payload?.data?.newResult?.data?.result) {
|
||||
handleNewData(data.payload.data.newResult.data.result);
|
||||
}
|
||||
}, [data, handleNewData]);
|
||||
|
||||
useEffect(() => {
|
||||
setIsPaginating(false);
|
||||
}, [data, setIsPaginating]);
|
||||
|
||||
const handleScrollToLog = useScrollToLog({
|
||||
logs,
|
||||
virtuosoRef,
|
||||
@@ -122,22 +195,21 @@ function HostMetricsLogs({ timeRange, filters }: Props): JSX.Element {
|
||||
const renderFooter = useCallback(
|
||||
(): JSX.Element | null => (
|
||||
<>
|
||||
{isFetching ? (
|
||||
<div className="logs-loading-skeleton"> Loading more logs ... </div>
|
||||
) : hasReachedEndOfLogs ? (
|
||||
<div className="logs-loading-skeleton"> *** End *** </div>
|
||||
{isFetchingNextPage ? (
|
||||
<div className={styles.logsLoadingSkeleton}> Loading more logs ... </div>
|
||||
) : !hasNextPage && logs.length > 0 ? (
|
||||
<div className={styles.logsLoadingSkeleton}> *** End *** </div>
|
||||
) : null}
|
||||
</>
|
||||
),
|
||||
[isFetching, hasReachedEndOfLogs],
|
||||
[isFetchingNextPage, hasNextPage, logs.length],
|
||||
);
|
||||
|
||||
const renderContent = useMemo(
|
||||
() => (
|
||||
<Card bordered={false} className="host-metrics-logs-list-card">
|
||||
<Card bordered={false} className={styles.listCard}>
|
||||
<OverlayScrollbar isVirtuoso>
|
||||
<Virtuoso
|
||||
className="host-metrics-logs-virtuoso"
|
||||
key="host-metrics-logs-virtuoso"
|
||||
ref={virtuosoRef}
|
||||
data={logs}
|
||||
@@ -155,32 +227,55 @@ function HostMetricsLogs({ timeRange, filters }: Props): JSX.Element {
|
||||
[logs, loadMoreLogs, getItemContent, renderFooter],
|
||||
);
|
||||
|
||||
const showInitialLoading = isLoading || (isFetching && logs.length === 0);
|
||||
|
||||
return (
|
||||
<div className="host-metrics-logs">
|
||||
{isLoading && <LogsLoading />}
|
||||
{!isLoading && !isError && logs.length === 0 && <NoLogsContainer />}
|
||||
{isError && !isLoading && <LogsError />}
|
||||
{!isLoading && !isError && logs.length > 0 && (
|
||||
<div
|
||||
className="host-metrics-logs-list-container"
|
||||
data-log-detail-ignore="true"
|
||||
>
|
||||
{renderContent}
|
||||
</div>
|
||||
)}
|
||||
{selectedTab && activeLog && (
|
||||
<LogDetail
|
||||
log={activeLog}
|
||||
onClose={handleCloseLogDetail}
|
||||
logs={logs}
|
||||
onNavigateLog={handleSetActiveLog}
|
||||
selectedTab={selectedTab}
|
||||
onAddToQuery={onAddToQuery}
|
||||
onClickActionItem={onAddToQuery}
|
||||
onScrollToLog={handleScrollToLog}
|
||||
<>
|
||||
<div className={styles.header}>
|
||||
<DateTimeSelectionV2
|
||||
showAutoRefresh
|
||||
showRefreshText={false}
|
||||
hideShareModal
|
||||
isModalTimeSelection={isModalTimeSelection}
|
||||
onTimeChange={handleTimeChange}
|
||||
defaultRelativeTime="5m"
|
||||
modalSelectedInterval={selectedInterval}
|
||||
modalInitialStartTime={timeRange.startTime * 1000}
|
||||
modalInitialEndTime={timeRange.endTime * 1000}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<QuerySearch
|
||||
queryData={queryData}
|
||||
onChange={handleFilterChange}
|
||||
dataSource={DataSource.LOGS}
|
||||
/>
|
||||
|
||||
<div className={styles.logs}>
|
||||
{showInitialLoading && <LogsLoading />}
|
||||
{!showInitialLoading && !isError && logs.length === 0 && (
|
||||
<NoLogsContainer />
|
||||
)}
|
||||
{isError && !showInitialLoading && <LogsError />}
|
||||
{!showInitialLoading && !isError && logs.length > 0 && (
|
||||
<div className={styles.listContainer} data-log-detail-ignore="true">
|
||||
{renderContent}
|
||||
</div>
|
||||
)}
|
||||
{selectedTab && activeLog && (
|
||||
<LogDetail
|
||||
log={activeLog}
|
||||
onClose={handleCloseLogDetail}
|
||||
logs={logs}
|
||||
onNavigateLog={handleSetActiveLog}
|
||||
selectedTab={selectedTab}
|
||||
onAddToQuery={onAddToQuery}
|
||||
onClickActionItem={onAddToQuery}
|
||||
onScrollToLog={handleScrollToLog}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
import { Color } from '@signozhq/design-tokens';
|
||||
import { Typography } from 'antd';
|
||||
import { Ghost } from 'lucide-react';
|
||||
|
||||
const { Text } = Typography;
|
||||
import styles from './HostMetricLogs.module.scss';
|
||||
|
||||
export default function NoLogsContainer(): React.ReactElement {
|
||||
return (
|
||||
<div className="no-logs-found">
|
||||
<Text type="secondary">
|
||||
<div className={styles.noLogsFound}>
|
||||
<p>
|
||||
<Ghost size={24} color={Color.BG_AMBER_500} /> No logs found for this host
|
||||
in the selected time range.
|
||||
</Text>
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,975 @@
|
||||
import { VirtuosoMockContext } from 'react-virtuoso';
|
||||
import { ENVIRONMENT } from 'constants/env';
|
||||
import { server } from 'mocks-server/server';
|
||||
import { rest } from 'msw';
|
||||
import { NuqsTestingAdapter } from 'nuqs/adapters/testing';
|
||||
import { act, render, screen, userEvent, waitFor } from 'tests/test-utils';
|
||||
|
||||
import HostMetricsLogs from '../HostMetricsLogs';
|
||||
|
||||
jest.mock('react-virtuoso', () => {
|
||||
const actual = jest.requireActual('react-virtuoso');
|
||||
return {
|
||||
...actual,
|
||||
Virtuoso: ({
|
||||
data,
|
||||
itemContent,
|
||||
endReached,
|
||||
components,
|
||||
className,
|
||||
}: {
|
||||
data?: any[];
|
||||
itemContent?: (index: number, item: any) => React.ReactNode;
|
||||
endReached?: (index: number) => void;
|
||||
components?: { Footer?: React.ComponentType };
|
||||
className?: string;
|
||||
}): JSX.Element => (
|
||||
<div data-testid="virtuoso-mock" className={className}>
|
||||
{Array.isArray(data) &&
|
||||
data.map((item, index) => (
|
||||
<div key={item?.id ?? index} data-testid={`virtuoso-item-${index}`}>
|
||||
{itemContent?.(index, item)}
|
||||
</div>
|
||||
))}
|
||||
<button
|
||||
type="button"
|
||||
data-testid="virtuoso-end-reached"
|
||||
onClick={(): void => endReached?.((data?.length || 0) - 1)}
|
||||
>
|
||||
endReached
|
||||
</button>
|
||||
{components?.Footer ? <components.Footer /> : null}
|
||||
</div>
|
||||
),
|
||||
};
|
||||
});
|
||||
|
||||
const QUERY_RANGE_URL = `${ENVIRONMENT.baseURL}/api/v5/query_range`;
|
||||
const FIELDS_KEYS_URL = `${ENVIRONMENT.baseURL}/api/v1/fields/keys`;
|
||||
const FIELDS_VALUES_URL = `${ENVIRONMENT.baseURL}/api/v1/fields/values`;
|
||||
|
||||
// Creates a V5 API response structure for raw logs data
|
||||
// The API response is wrapped in { data: { type: '...', data: { results: [...] } } }
|
||||
const createLogsResponse = ({
|
||||
offset = 0,
|
||||
pageSize = 100,
|
||||
hasMore = true,
|
||||
}: {
|
||||
offset?: number;
|
||||
pageSize?: number;
|
||||
hasMore?: boolean;
|
||||
}): any => {
|
||||
const itemsForThisPage = hasMore ? pageSize : Math.min(pageSize / 2, 10);
|
||||
|
||||
return {
|
||||
data: {
|
||||
type: 'raw',
|
||||
data: {
|
||||
results: [
|
||||
{
|
||||
queryName: 'A',
|
||||
rows: Array.from({ length: itemsForThisPage }, (_, index) => {
|
||||
const cumulativeIndex = offset + index;
|
||||
const baseTimestamp = new Date('2024-02-15T21:20:22Z').getTime();
|
||||
const currentTimestamp = new Date(
|
||||
baseTimestamp - cumulativeIndex * 1000,
|
||||
);
|
||||
const timestampString = currentTimestamp.toISOString();
|
||||
const id = `log-id-${cumulativeIndex}`;
|
||||
const logLevel = ['INFO', 'WARN', 'ERROR'][cumulativeIndex % 3];
|
||||
const service = ['frontend', 'backend', 'database'][cumulativeIndex % 3];
|
||||
|
||||
return {
|
||||
timestamp: timestampString,
|
||||
data: {
|
||||
attributes_bool: {},
|
||||
attributes_float64: {},
|
||||
attributes_int64: {},
|
||||
attributes_string: {
|
||||
host_name: 'test-host',
|
||||
log_level: logLevel,
|
||||
service,
|
||||
},
|
||||
body: `${timestampString} ${logLevel} ${service} Log message ${cumulativeIndex}`,
|
||||
id,
|
||||
resources_string: {
|
||||
'host.name': 'test-host',
|
||||
},
|
||||
severity_number: [9, 13, 17][cumulativeIndex % 3],
|
||||
severity_text: logLevel,
|
||||
span_id: `span-${cumulativeIndex}`,
|
||||
trace_flags: 0,
|
||||
trace_id: `trace-${cumulativeIndex}`,
|
||||
},
|
||||
};
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const createEmptyLogsResponse = (): any => ({
|
||||
data: {
|
||||
type: 'raw',
|
||||
data: {
|
||||
results: [
|
||||
{
|
||||
queryName: 'A',
|
||||
rows: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const defaultProps = {
|
||||
initialExpression: 'host_name = "test-host"',
|
||||
timeRange: {
|
||||
startTime: 1708000000,
|
||||
endTime: 1708003600,
|
||||
},
|
||||
isModalTimeSelection: false,
|
||||
handleTimeChange: jest.fn(),
|
||||
selectedInterval: '15m' as const,
|
||||
};
|
||||
|
||||
// Mock OverlayScrollbar to avoid scroll behavior issues in tests
|
||||
jest.mock('components/OverlayScrollbar/OverlayScrollbar', () => ({
|
||||
__esModule: true,
|
||||
default: ({ children }: { children: React.ReactNode }): JSX.Element => (
|
||||
<div>{children}</div>
|
||||
),
|
||||
}));
|
||||
|
||||
jest.mock('container/TopNav/DateTimeSelectionV2/index.tsx', () => ({
|
||||
__esModule: true,
|
||||
default: ({
|
||||
onTimeChange,
|
||||
}: {
|
||||
onTimeChange?: (interval: string, dateTimeRange?: [number, number]) => void;
|
||||
}): JSX.Element => {
|
||||
return (
|
||||
<div className="datetime-section" data-testid="datetime-selection">
|
||||
<button
|
||||
data-testid="time-picker-btn"
|
||||
onClick={(): void => {
|
||||
onTimeChange?.('5m');
|
||||
}}
|
||||
>
|
||||
Select Time
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
}));
|
||||
|
||||
const createFieldKeysResponse = (): any => ({
|
||||
status: 'success',
|
||||
data: {
|
||||
complete: true,
|
||||
keys: {},
|
||||
},
|
||||
});
|
||||
|
||||
const createFieldValuesResponse = (): any => ({
|
||||
status: 'success',
|
||||
data: {
|
||||
values: {
|
||||
stringValues: [],
|
||||
numberValues: [],
|
||||
boolValues: [],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const renderComponent = (
|
||||
props = defaultProps,
|
||||
searchParams?: Record<string, string>,
|
||||
): ReturnType<typeof render> =>
|
||||
render(
|
||||
<NuqsTestingAdapter searchParams={searchParams} hasMemory>
|
||||
<VirtuosoMockContext.Provider
|
||||
value={{ viewportHeight: 600, itemHeight: 50 }}
|
||||
>
|
||||
<HostMetricsLogs {...props} />
|
||||
</VirtuosoMockContext.Provider>
|
||||
</NuqsTestingAdapter>,
|
||||
);
|
||||
|
||||
describe('HostMetricsLogs', () => {
|
||||
beforeEach(() => {
|
||||
window.history.pushState({}, 'Test', '/');
|
||||
server.use(
|
||||
rest.get(FIELDS_KEYS_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createFieldKeysResponse())),
|
||||
),
|
||||
rest.get(FIELDS_VALUES_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createFieldValuesResponse())),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
describe('loading state', () => {
|
||||
it('should show loading state while fetching logs', async () => {
|
||||
let resolveRequest: (value: any) => void;
|
||||
const pendingPromise = new Promise((resolve) => {
|
||||
resolveRequest = resolve;
|
||||
});
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (_, res, ctx) => {
|
||||
await pendingPromise;
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({})));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
expect(screen.getByText('pending_data_placeholder')).toBeInTheDocument();
|
||||
|
||||
act(() => {
|
||||
resolveRequest!(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('empty state', () => {
|
||||
it('should show no logs message when no logs are returned', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createEmptyLogsResponse())),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(
|
||||
screen.getByText(/No logs found for this host/i),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('error state', () => {
|
||||
it('should show error state when API returns error', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Something went wrong/i)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('success state', () => {
|
||||
it('should render logs when API returns data', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('should render initial expression in QuerySearch editor', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
const editorText =
|
||||
document.querySelector('.query-where-clause-editor')?.textContent || '';
|
||||
expect(editorText).toContain('host_name');
|
||||
expect(editorText).toContain('test-host');
|
||||
});
|
||||
});
|
||||
|
||||
it('should render the filter section', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(
|
||||
document.querySelector('.code-mirror-where-clause'),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('should render date time selection component', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
// DateTimeSelectionV2 renders a time picker button
|
||||
expect(document.querySelector('.datetime-section')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('pagination', () => {
|
||||
it('should send correct offset for pagination', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
const querySpec = payload.compositeQuery?.queries?.[0]?.spec;
|
||||
const offset = querySpec?.offset ?? 0;
|
||||
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(
|
||||
createLogsResponse({
|
||||
offset,
|
||||
pageSize: 100,
|
||||
hasMore: offset === 0,
|
||||
}),
|
||||
),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstPayload = requestPayloads[0];
|
||||
const querySpec = firstPayload.compositeQuery?.queries?.[0]?.spec;
|
||||
expect(querySpec?.offset).toBe(0);
|
||||
});
|
||||
|
||||
it('should fetch next page when virtuoso endReached is triggered', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
const querySpec = payload.compositeQuery?.queries?.[0]?.spec;
|
||||
const offset = querySpec?.offset ?? 0;
|
||||
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(
|
||||
createLogsResponse({
|
||||
offset,
|
||||
pageSize: 100,
|
||||
hasMore: offset === 0,
|
||||
}),
|
||||
),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
expect(requestPayloads[0]?.compositeQuery?.queries?.[0]?.spec?.offset).toBe(
|
||||
0,
|
||||
);
|
||||
|
||||
await userEvent.click(screen.getByTestId('virtuoso-end-reached'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
expect(requestPayloads[1]?.compositeQuery?.queries?.[0]?.spec?.offset).toBe(
|
||||
100,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('filter expression', () => {
|
||||
it('should include initial expression in the query', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstPayload = requestPayloads[0];
|
||||
const querySpec = firstPayload.compositeQuery?.queries?.[0]?.spec;
|
||||
|
||||
expect(querySpec?.filter?.expression).toContain('host_name = "test-host"');
|
||||
});
|
||||
|
||||
it('should load expression from URL and persist it in the query', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
const querySpec = payload.compositeQuery?.queries?.[0]?.spec;
|
||||
const offset = querySpec?.offset ?? 0;
|
||||
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(
|
||||
createLogsResponse({
|
||||
offset,
|
||||
pageSize: 100,
|
||||
hasMore: offset === 0,
|
||||
}),
|
||||
),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const urlExpression = 'service = "from-url"';
|
||||
|
||||
renderComponent(defaultProps, { hostMetricsLogsExpr: urlExpression });
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
expect(
|
||||
requestPayloads[0]?.compositeQuery?.queries?.[0]?.spec?.filter?.expression,
|
||||
).toContain(urlExpression);
|
||||
|
||||
await userEvent.click(screen.getByTestId('virtuoso-end-reached'));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
expect(
|
||||
requestPayloads[1]?.compositeQuery?.queries?.[0]?.spec?.filter?.expression,
|
||||
).toContain(urlExpression);
|
||||
});
|
||||
|
||||
it('should use custom expression when provided', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
const customExpression = 'service = "custom-service"';
|
||||
|
||||
renderComponent({
|
||||
...defaultProps,
|
||||
initialExpression: customExpression,
|
||||
});
|
||||
|
||||
// Wait for debounce and potential re-renders to settle
|
||||
await waitFor(
|
||||
() => {
|
||||
const hasCustomExpression = requestPayloads.some((payload) => {
|
||||
const querySpec = payload.compositeQuery?.queries?.[0]?.spec;
|
||||
return querySpec?.filter?.expression?.includes('custom-service');
|
||||
});
|
||||
expect(hasCustomExpression).toBe(true);
|
||||
},
|
||||
{ timeout: 2000 },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('time range', () => {
|
||||
it('should include correct time range in the query', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
const customTimeRange = {
|
||||
startTime: 1700000000,
|
||||
endTime: 1700003600,
|
||||
};
|
||||
|
||||
renderComponent({
|
||||
...defaultProps,
|
||||
timeRange: customTimeRange,
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstPayload = requestPayloads[0];
|
||||
|
||||
// V5 API expects milliseconds (seconds * 1000)
|
||||
expect(firstPayload.start).toBe(customTimeRange.startTime * 1000);
|
||||
expect(firstPayload.end).toBe(customTimeRange.endTime * 1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('query structure', () => {
|
||||
it('should send correct query structure to the API', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstPayload = requestPayloads[0];
|
||||
const querySpec = firstPayload.compositeQuery?.queries?.[0]?.spec;
|
||||
|
||||
expect(querySpec?.signal).toBe('logs');
|
||||
expect(querySpec?.order).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
key: expect.objectContaining({ name: 'timestamp' }),
|
||||
direction: 'desc',
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('should send request type as raw for logs list', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstPayload = requestPayloads[0];
|
||||
expect(firstPayload.requestType).toBe('raw');
|
||||
});
|
||||
|
||||
it('should include pageSize in the query', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstPayload = requestPayloads[0];
|
||||
const querySpec = firstPayload.compositeQuery?.queries?.[0]?.spec;
|
||||
|
||||
// Should have a limit set for pagination
|
||||
expect(querySpec?.limit).toBeDefined();
|
||||
expect(typeof querySpec?.limit).toBe('number');
|
||||
});
|
||||
});
|
||||
|
||||
describe('component props', () => {
|
||||
it('should render datetime section with isModalTimeSelection', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent({
|
||||
...defaultProps,
|
||||
isModalTimeSelection: true,
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(document.querySelector('.datetime-section')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('should render component with handleTimeChange', async () => {
|
||||
const mockHandleTimeChange = jest.fn();
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent({
|
||||
...defaultProps,
|
||||
handleTimeChange: mockHandleTimeChange,
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(document.querySelector('.datetime-section')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('log detail interactions', () => {
|
||||
it('should open log detail drawer when clicking on a log', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
// Wait for logs to render
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Click on the first log
|
||||
const logElement = screen.getByText(/Log message 0/);
|
||||
await userEvent.click(logElement);
|
||||
|
||||
// Log detail drawer should open - it contains "Log details" title
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Log details')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('should close log detail drawer when clicking on the same log again', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
// Wait for logs to render
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Click on the first log to open
|
||||
const logElement = screen.getByText(/Log message 0/);
|
||||
await userEvent.click(logElement);
|
||||
|
||||
// Wait for drawer to open
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Log details')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Click on the same log to close (through the close button)
|
||||
const closeButton = document.querySelector('.ant-drawer-close');
|
||||
if (closeButton) {
|
||||
await userEvent.click(closeButton);
|
||||
}
|
||||
|
||||
// Drawer should close
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('Log details')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
it('should display log body in detail drawer', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
// Wait for logs to render
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Click on the first log to open drawer
|
||||
const logElement = screen.getByText(/Log message 0/);
|
||||
await userEvent.click(logElement);
|
||||
|
||||
// Wait for drawer to open
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Log details')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Verify the drawer tabs are displayed
|
||||
// The drawer should show the Overview tab
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Overview')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Verify other tabs are present
|
||||
expect(screen.getByText('JSON')).toBeInTheDocument();
|
||||
expect(screen.getByText('Context')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('log detail filter actions', () => {
|
||||
it('should apply filter-in from log detail and close the drawer', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await userEvent.click(screen.getByText(/Log message 0/));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Log details')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
const serviceRow = await waitFor(() => {
|
||||
const attributeNameCells = Array.from(
|
||||
document.querySelectorAll('.attribute-name'),
|
||||
);
|
||||
const serviceCell = attributeNameCells.find((cell) =>
|
||||
(cell.textContent || '').toLowerCase().includes('service'),
|
||||
);
|
||||
const row = serviceCell?.closest('tr');
|
||||
if (!row) {
|
||||
throw new Error('Service attribute row not found');
|
||||
}
|
||||
return row;
|
||||
});
|
||||
|
||||
const filterButtons = serviceRow.querySelectorAll('button.filter-btn');
|
||||
expect(filterButtons?.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
await userEvent.click(filterButtons[0] as HTMLButtonElement);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('Log details')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
await waitFor(
|
||||
() => {
|
||||
const matched = requestPayloads.some((payload) => {
|
||||
const expression =
|
||||
payload.compositeQuery?.queries?.[0]?.spec?.filter?.expression || '';
|
||||
return (
|
||||
(expression.includes('attributes_string.service') ||
|
||||
expression.includes('service')) &&
|
||||
expression.includes("('frontend')") &&
|
||||
expression.includes('IN')
|
||||
);
|
||||
});
|
||||
expect(matched).toBe(true);
|
||||
},
|
||||
{ timeout: 2500 },
|
||||
);
|
||||
});
|
||||
|
||||
it('should apply filter-out from log detail and close the drawer', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
renderComponent();
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText(/Log message 0/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
await userEvent.click(screen.getByText(/Log message 0/));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.getByText('Log details')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
const serviceRow = await waitFor(() => {
|
||||
const attributeNameCells = Array.from(
|
||||
document.querySelectorAll('.attribute-name'),
|
||||
);
|
||||
const serviceCell = attributeNameCells.find((cell) =>
|
||||
(cell.textContent || '').toLowerCase().includes('service'),
|
||||
);
|
||||
const row = serviceCell?.closest('tr');
|
||||
if (!row) {
|
||||
throw new Error('Service attribute row not found');
|
||||
}
|
||||
return row;
|
||||
});
|
||||
|
||||
const filterButtons = serviceRow.querySelectorAll('button.filter-btn');
|
||||
expect(filterButtons?.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
// the second button that represents filter out
|
||||
await userEvent.click(filterButtons[1] as HTMLButtonElement);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(screen.queryByText('Log details')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
await waitFor(
|
||||
() => {
|
||||
const matched = requestPayloads.some((payload) => {
|
||||
const expression =
|
||||
payload.compositeQuery?.queries?.[0]?.spec?.filter?.expression || '';
|
||||
return (
|
||||
(expression.includes('attributes_string.service') ||
|
||||
expression.includes('service')) &&
|
||||
expression.includes("('frontend')") &&
|
||||
(expression.includes('NIN') || expression.includes('NOT_IN'))
|
||||
);
|
||||
});
|
||||
expect(matched).toBe(true);
|
||||
},
|
||||
{ timeout: 2500 },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('time range change', () => {
|
||||
it('should use different time ranges for different renders', async () => {
|
||||
const requestPayloads: any[] = [];
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, async (req, res, ctx) => {
|
||||
const payload = await req.json();
|
||||
requestPayloads.push(payload);
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 })));
|
||||
}),
|
||||
);
|
||||
|
||||
// First render with initial time range
|
||||
const { unmount } = renderComponent();
|
||||
|
||||
// Wait for initial fetch
|
||||
await waitFor(() => {
|
||||
expect(requestPayloads.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
const firstStartTime = requestPayloads[0].start;
|
||||
expect(firstStartTime).toBe(defaultProps.timeRange.startTime * 1000);
|
||||
|
||||
// Unmount and render again with different time range
|
||||
unmount();
|
||||
|
||||
const newTimeRange = {
|
||||
startTime: 1709000000,
|
||||
endTime: 1709003600,
|
||||
};
|
||||
|
||||
renderComponent({
|
||||
...defaultProps,
|
||||
timeRange: newTimeRange,
|
||||
});
|
||||
|
||||
// Wait for fetch with new time range
|
||||
await waitFor(() => {
|
||||
const hasNewTimeRange = requestPayloads.some(
|
||||
(p) => p.start === newTimeRange.startTime * 1000,
|
||||
);
|
||||
expect(hasNewTimeRange).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
it('should call handleTimeChange callback when time picker is clicked', async () => {
|
||||
const mockHandleTimeChange = jest.fn();
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
renderComponent({
|
||||
...defaultProps,
|
||||
handleTimeChange: mockHandleTimeChange,
|
||||
});
|
||||
|
||||
// Wait for component to render
|
||||
await waitFor(() => {
|
||||
expect(screen.getByTestId('time-picker-btn')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
// Click the time picker button (from mock)
|
||||
await userEvent.click(screen.getByTestId('time-picker-btn'));
|
||||
|
||||
// Verify the callback was called
|
||||
expect(mockHandleTimeChange).toHaveBeenCalledWith('5m');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
import { QueryClient, QueryClientProvider } from 'react-query';
|
||||
import { act, renderHook, waitFor } from '@testing-library/react';
|
||||
import { ENVIRONMENT } from 'constants/env';
|
||||
import { server } from 'mocks-server/server';
|
||||
import { rest } from 'msw';
|
||||
|
||||
import { useInfiniteHostMetricLogs } from '../hooks';
|
||||
|
||||
const QUERY_RANGE_URL = `${ENVIRONMENT.baseURL}/api/v5/query_range`;
|
||||
|
||||
const createLogsResponse = ({
|
||||
offset = 0,
|
||||
pageSize = 100,
|
||||
hasMore = true,
|
||||
}: {
|
||||
offset?: number;
|
||||
pageSize?: number;
|
||||
hasMore?: boolean;
|
||||
}): any => {
|
||||
const itemsForThisPage = hasMore ? pageSize : pageSize / 2;
|
||||
|
||||
return {
|
||||
data: {
|
||||
type: 'raw',
|
||||
data: {
|
||||
results: [
|
||||
{
|
||||
queryName: 'A',
|
||||
rows: Array.from({ length: itemsForThisPage }, (_, index) => {
|
||||
const cumulativeIndex = offset + index;
|
||||
return {
|
||||
timestamp: new Date(Date.now() - cumulativeIndex * 1000).toISOString(),
|
||||
data: {
|
||||
body: `Log message ${cumulativeIndex}`,
|
||||
id: `log-${cumulativeIndex}`,
|
||||
severity_text: 'INFO',
|
||||
},
|
||||
};
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const createEmptyResponse = (): any => ({
|
||||
data: {
|
||||
type: 'raw',
|
||||
data: {
|
||||
results: [
|
||||
{
|
||||
queryName: 'A',
|
||||
rows: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const createWrapper = (): React.FC<{ children: React.ReactNode }> => {
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
retry: false,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
return function Wrapper({
|
||||
children,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
}): JSX.Element {
|
||||
return (
|
||||
<QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
|
||||
);
|
||||
};
|
||||
};
|
||||
|
||||
describe('useInfiniteHostMetricLogs', () => {
|
||||
const defaultParams = {
|
||||
expression: 'host_name = "test-host"',
|
||||
startTime: 1708000000,
|
||||
endTime: 1708003600,
|
||||
};
|
||||
|
||||
describe('initial state', () => {
|
||||
it('should return initial loading state', () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.delay(100), ctx.status(200), ctx.json(createLogsResponse({}))),
|
||||
),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.current.isLoading).toBe(true);
|
||||
expect(result.current.logs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('successful data fetching', () => {
|
||||
it('should return logs after successful fetch', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createLogsResponse({ pageSize: 5 }))),
|
||||
),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
});
|
||||
|
||||
expect(result.current.logs.length).toBe(5);
|
||||
expect(result.current.isError).toBe(false);
|
||||
});
|
||||
|
||||
it('should set hasNextPage based on response size', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json(createLogsResponse({ pageSize: 100, hasMore: true })),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
});
|
||||
|
||||
expect(result.current.hasNextPage).toBe(true);
|
||||
});
|
||||
|
||||
it('should not have next page when response is smaller than page size', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json(createLogsResponse({ pageSize: 100, hasMore: false })),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
});
|
||||
|
||||
expect(result.current.hasNextPage).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('empty state', () => {
|
||||
it('should return empty logs array when no data', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(createEmptyResponse())),
|
||||
),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
});
|
||||
|
||||
expect(result.current.logs).toEqual([]);
|
||||
expect(result.current.hasNextPage).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
it('should set isError on API failure', async () => {
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) =>
|
||||
res(ctx.status(500), ctx.json({ error: 'Internal Server Error' })),
|
||||
),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isError).toBe(true);
|
||||
});
|
||||
|
||||
expect(result.current.logs).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('query disabled state', () => {
|
||||
it('should not fetch when expression is empty', async () => {
|
||||
const requestCount = { count: 0 };
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) => {
|
||||
requestCount.count += 1;
|
||||
return res(ctx.status(200), ctx.json(createLogsResponse({})));
|
||||
}),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() =>
|
||||
useInfiniteHostMetricLogs({
|
||||
...defaultParams,
|
||||
expression: '',
|
||||
}),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
// Wait a bit to ensure no request is made
|
||||
await new Promise((resolve) => {
|
||||
setTimeout(resolve, 300);
|
||||
});
|
||||
|
||||
expect(requestCount.count).toBe(0);
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('load more functionality', () => {
|
||||
it('should fetch next page when loadMoreLogs is called', async () => {
|
||||
const requestCount = { count: 0 };
|
||||
|
||||
server.use(
|
||||
rest.post(QUERY_RANGE_URL, (_, res, ctx) => {
|
||||
requestCount.count += 1;
|
||||
if (requestCount.count === 1) {
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(
|
||||
createLogsResponse({ offset: 0, pageSize: 100, hasMore: true }),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json(
|
||||
createLogsResponse({ offset: 100, pageSize: 100, hasMore: false }),
|
||||
),
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
const { result } = renderHook(
|
||||
() => useInfiniteHostMetricLogs(defaultParams),
|
||||
{
|
||||
wrapper: createWrapper(),
|
||||
},
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.isLoading).toBe(false);
|
||||
});
|
||||
|
||||
expect(result.current.logs.length).toBe(100);
|
||||
expect(result.current.hasNextPage).toBe(true);
|
||||
expect(requestCount.count).toBe(1);
|
||||
|
||||
act(() => {
|
||||
result.current.loadMoreLogs();
|
||||
});
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.logs.length).toBe(150);
|
||||
});
|
||||
|
||||
expect(result.current.hasNextPage).toBe(false);
|
||||
expect(requestCount.count).toBe(2);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import { DEFAULT_PER_PAGE_VALUE } from 'container/Controls/config';
|
||||
import { GetQueryResultsProps } from 'lib/dashboard/getQueryResults';
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
|
||||
@@ -6,56 +7,82 @@ import { EQueryType } from 'types/common/dashboard';
|
||||
import { DataSource, ReduceOperators } from 'types/common/queryBuilder';
|
||||
import { v4 as uuidv4 } from 'uuid';
|
||||
|
||||
export const getHostLogsQueryPayload = (
|
||||
start: number,
|
||||
end: number,
|
||||
filters: IBuilderQuery['filters'],
|
||||
): GetQueryResultsProps => ({
|
||||
graphType: PANEL_TYPES.LIST,
|
||||
selectedTime: 'GLOBAL_TIME',
|
||||
query: {
|
||||
clickhouse_sql: [],
|
||||
promql: [],
|
||||
builder: {
|
||||
queryData: [
|
||||
{
|
||||
dataSource: DataSource.LOGS,
|
||||
queryName: 'A',
|
||||
aggregateOperator: 'noop',
|
||||
aggregateAttribute: {
|
||||
id: '------false',
|
||||
dataType: DataTypes.String,
|
||||
key: '',
|
||||
type: '',
|
||||
},
|
||||
timeAggregation: 'rate',
|
||||
spaceAggregation: 'sum',
|
||||
functions: [],
|
||||
filters,
|
||||
expression: 'A',
|
||||
disabled: false,
|
||||
stepInterval: 60,
|
||||
having: [],
|
||||
limit: null,
|
||||
orderBy: [
|
||||
{
|
||||
columnName: 'timestamp',
|
||||
order: 'desc',
|
||||
},
|
||||
],
|
||||
groupBy: [],
|
||||
legend: '',
|
||||
reduceTo: ReduceOperators.AVG,
|
||||
offset: 0,
|
||||
pageSize: 100,
|
||||
},
|
||||
],
|
||||
queryFormulas: [],
|
||||
queryTraceOperator: [],
|
||||
},
|
||||
id: uuidv4(),
|
||||
queryType: EQueryType.QUERY_BUILDER,
|
||||
},
|
||||
export interface HostLogsQueryParams {
|
||||
start: number;
|
||||
end: number;
|
||||
expression: string;
|
||||
offset?: number;
|
||||
pageSize?: number;
|
||||
}
|
||||
|
||||
export const getHostLogsQueryPayload = ({
|
||||
start,
|
||||
end,
|
||||
});
|
||||
expression,
|
||||
offset = 0,
|
||||
pageSize = DEFAULT_PER_PAGE_VALUE,
|
||||
}: HostLogsQueryParams): {
|
||||
query: GetQueryResultsProps;
|
||||
queryData: IBuilderQuery;
|
||||
} => {
|
||||
const queryData: IBuilderQuery = {
|
||||
dataSource: DataSource.LOGS,
|
||||
queryName: 'A',
|
||||
aggregateOperator: 'noop',
|
||||
aggregateAttribute: {
|
||||
id: '------false',
|
||||
dataType: DataTypes.String,
|
||||
key: '',
|
||||
type: '',
|
||||
},
|
||||
timeAggregation: 'rate',
|
||||
spaceAggregation: 'sum',
|
||||
functions: [],
|
||||
filter: { expression },
|
||||
expression,
|
||||
having: {
|
||||
expression: '',
|
||||
},
|
||||
disabled: false,
|
||||
stepInterval: 60,
|
||||
limit: null,
|
||||
orderBy: [
|
||||
{
|
||||
columnName: 'timestamp',
|
||||
order: 'desc',
|
||||
},
|
||||
{
|
||||
columnName: 'id',
|
||||
order: 'desc',
|
||||
},
|
||||
],
|
||||
groupBy: [],
|
||||
legend: '',
|
||||
reduceTo: ReduceOperators.AVG,
|
||||
offset,
|
||||
pageSize,
|
||||
};
|
||||
|
||||
return {
|
||||
query: {
|
||||
graphType: PANEL_TYPES.LIST,
|
||||
selectedTime: 'GLOBAL_TIME',
|
||||
query: {
|
||||
clickhouse_sql: [],
|
||||
promql: [],
|
||||
builder: {
|
||||
queryData: [queryData],
|
||||
queryFormulas: [],
|
||||
queryTraceOperator: [],
|
||||
},
|
||||
id: uuidv4(),
|
||||
queryType: EQueryType.QUERY_BUILDER,
|
||||
},
|
||||
start,
|
||||
end,
|
||||
},
|
||||
queryData,
|
||||
};
|
||||
};
|
||||
|
||||
export const HOST_METRICS_LOGS_EXPR_QUERY_KEY = 'hostMetricsLogsExpr';
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import { useCallback, useMemo } from 'react';
|
||||
import { useInfiniteQuery } from 'react-query';
|
||||
import { ENTITY_VERSION_V5 } from 'constants/app';
|
||||
import { DEFAULT_PER_PAGE_VALUE } from 'container/Controls/config';
|
||||
import { GetMetricQueryRange } from 'lib/dashboard/getQueryResults';
|
||||
import { ILog } from 'types/api/logs/log';
|
||||
|
||||
import { getHostLogsQueryPayload } from './constants';
|
||||
|
||||
export function useInfiniteHostMetricLogs({
|
||||
expression,
|
||||
startTime,
|
||||
endTime,
|
||||
}: {
|
||||
expression: string;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
}): {
|
||||
logs: ILog[];
|
||||
isLoading: boolean;
|
||||
isFetching: boolean;
|
||||
isFetchingNextPage: boolean;
|
||||
isError: boolean;
|
||||
hasNextPage: boolean;
|
||||
loadMoreLogs: () => void;
|
||||
} {
|
||||
const {
|
||||
data,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isFetchingNextPage,
|
||||
isError,
|
||||
hasNextPage,
|
||||
fetchNextPage,
|
||||
} = useInfiniteQuery({
|
||||
queryKey: ['hostMetricsLogs', startTime, endTime, expression],
|
||||
queryFn: async ({ pageParam = 0 }) => {
|
||||
const { query } = getHostLogsQueryPayload({
|
||||
start: startTime,
|
||||
end: endTime,
|
||||
expression,
|
||||
offset: pageParam,
|
||||
pageSize: DEFAULT_PER_PAGE_VALUE,
|
||||
});
|
||||
return GetMetricQueryRange(query, ENTITY_VERSION_V5);
|
||||
},
|
||||
getNextPageParam: (lastPage, allPages) => {
|
||||
const list = lastPage?.payload?.data?.newResult?.data?.result?.[0]?.list;
|
||||
if (!list || list.length < DEFAULT_PER_PAGE_VALUE) {
|
||||
return undefined;
|
||||
}
|
||||
return allPages.length * DEFAULT_PER_PAGE_VALUE;
|
||||
},
|
||||
enabled: !!expression,
|
||||
});
|
||||
|
||||
const logs = useMemo<ILog[]>(() => {
|
||||
if (!data?.pages) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return data.pages.flatMap((page) => {
|
||||
const list = page?.payload?.data?.newResult?.data?.result?.[0]?.list;
|
||||
if (!list) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return list.map(
|
||||
(item) =>
|
||||
({
|
||||
...item.data,
|
||||
timestamp: item.timestamp,
|
||||
} as ILog),
|
||||
);
|
||||
});
|
||||
}, [data?.pages]);
|
||||
|
||||
const loadMoreLogs = useCallback(() => {
|
||||
if (hasNextPage && !isFetchingNextPage) {
|
||||
fetchNextPage();
|
||||
}
|
||||
}, [hasNextPage, isFetchingNextPage, fetchNextPage]);
|
||||
|
||||
return {
|
||||
logs,
|
||||
isLoading,
|
||||
isFetching,
|
||||
isFetchingNextPage,
|
||||
isError,
|
||||
hasNextPage: !!hasNextPage,
|
||||
loadMoreLogs,
|
||||
};
|
||||
}
|
||||
@@ -48,7 +48,6 @@ import DashboardEmptyState from './DashboardEmptyState/DashboardEmptyState';
|
||||
import GridCard from './GridCard';
|
||||
import { Card, CardContainer, ReactGridLayout } from './styles';
|
||||
import {
|
||||
applyRowCollapse,
|
||||
hasColumnWidthsChanged,
|
||||
removeUndefinedValuesFromLayout,
|
||||
} from './utils';
|
||||
@@ -269,10 +268,13 @@ function GraphLayout(props: GraphLayoutProps): JSX.Element {
|
||||
return;
|
||||
}
|
||||
|
||||
const updatedWidgets = selectedDashboard?.data?.widgets?.map((e) =>
|
||||
e.id === currentSelectRowId ? { ...e, title: newTitle } : e,
|
||||
currentWidget.title = newTitle;
|
||||
const updatedWidgets = selectedDashboard?.data?.widgets?.filter(
|
||||
(e) => e.id !== currentSelectRowId,
|
||||
);
|
||||
|
||||
updatedWidgets?.push(currentWidget);
|
||||
|
||||
const updatedSelectedDashboard: Props = {
|
||||
id: selectedDashboard.id,
|
||||
data: {
|
||||
@@ -314,13 +316,88 @@ function GraphLayout(props: GraphLayoutProps): JSX.Element {
|
||||
if (!selectedDashboard) {
|
||||
return;
|
||||
}
|
||||
const { updatedLayout, updatedPanelMap } = applyRowCollapse(
|
||||
id,
|
||||
dashboardLayout,
|
||||
currentPanelMap,
|
||||
);
|
||||
setCurrentPanelMap((prev) => ({ ...prev, ...updatedPanelMap }));
|
||||
setDashboardLayout(sortLayout(updatedLayout));
|
||||
const rowProperties = { ...currentPanelMap[id] };
|
||||
const updatedPanelMap = { ...currentPanelMap };
|
||||
|
||||
let updatedDashboardLayout = [...dashboardLayout];
|
||||
if (rowProperties.collapsed === true) {
|
||||
rowProperties.collapsed = false;
|
||||
const widgetsInsideTheRow = rowProperties.widgets;
|
||||
let maxY = 0;
|
||||
widgetsInsideTheRow.forEach((w) => {
|
||||
maxY = Math.max(maxY, w.y + w.h);
|
||||
});
|
||||
const currentRowWidget = dashboardLayout.find((w) => w.i === id);
|
||||
if (currentRowWidget && widgetsInsideTheRow.length) {
|
||||
maxY -= currentRowWidget.h + currentRowWidget.y;
|
||||
}
|
||||
|
||||
const idxCurrentRow = dashboardLayout.findIndex((w) => w.i === id);
|
||||
|
||||
for (let j = idxCurrentRow + 1; j < dashboardLayout.length; j++) {
|
||||
updatedDashboardLayout[j].y += maxY;
|
||||
if (updatedPanelMap[updatedDashboardLayout[j].i]) {
|
||||
updatedPanelMap[updatedDashboardLayout[j].i].widgets = updatedPanelMap[
|
||||
updatedDashboardLayout[j].i
|
||||
].widgets.map((w) => ({
|
||||
...w,
|
||||
y: w.y + maxY,
|
||||
}));
|
||||
}
|
||||
}
|
||||
updatedDashboardLayout = [...updatedDashboardLayout, ...widgetsInsideTheRow];
|
||||
} else {
|
||||
rowProperties.collapsed = true;
|
||||
const currentIdx = dashboardLayout.findIndex((w) => w.i === id);
|
||||
|
||||
let widgetsInsideTheRow: Layout[] = [];
|
||||
let isPanelMapUpdated = false;
|
||||
for (let j = currentIdx + 1; j < dashboardLayout.length; j++) {
|
||||
if (currentPanelMap[dashboardLayout[j].i]) {
|
||||
rowProperties.widgets = widgetsInsideTheRow;
|
||||
widgetsInsideTheRow = [];
|
||||
isPanelMapUpdated = true;
|
||||
break;
|
||||
} else {
|
||||
widgetsInsideTheRow.push(dashboardLayout[j]);
|
||||
}
|
||||
}
|
||||
if (!isPanelMapUpdated) {
|
||||
rowProperties.widgets = widgetsInsideTheRow;
|
||||
}
|
||||
let maxY = 0;
|
||||
widgetsInsideTheRow.forEach((w) => {
|
||||
maxY = Math.max(maxY, w.y + w.h);
|
||||
});
|
||||
const currentRowWidget = dashboardLayout[currentIdx];
|
||||
if (currentRowWidget && widgetsInsideTheRow.length) {
|
||||
maxY -= currentRowWidget.h + currentRowWidget.y;
|
||||
}
|
||||
for (let j = currentIdx + 1; j < updatedDashboardLayout.length; j++) {
|
||||
updatedDashboardLayout[j].y += maxY;
|
||||
if (updatedPanelMap[updatedDashboardLayout[j].i]) {
|
||||
updatedPanelMap[updatedDashboardLayout[j].i].widgets = updatedPanelMap[
|
||||
updatedDashboardLayout[j].i
|
||||
].widgets.map((w) => ({
|
||||
...w,
|
||||
y: w.y + maxY,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
updatedDashboardLayout = updatedDashboardLayout.filter(
|
||||
(widget) => !rowProperties.widgets.some((w: Layout) => w.i === widget.i),
|
||||
);
|
||||
}
|
||||
setCurrentPanelMap((prev) => ({
|
||||
...prev,
|
||||
...updatedPanelMap,
|
||||
[id]: {
|
||||
...rowProperties,
|
||||
},
|
||||
}));
|
||||
|
||||
setDashboardLayout(sortLayout(updatedDashboardLayout));
|
||||
};
|
||||
|
||||
const handleDragStop: ItemCallback = (_, oldItem, newItem): void => {
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
import { Layout } from 'react-grid-layout';
|
||||
|
||||
import { applyRowCollapse, PanelMap } from '../utils';
|
||||
|
||||
// Helper to produce deeply-frozen objects that mimic what zustand/immer returns.
|
||||
function freeze<T>(obj: T): T {
|
||||
return JSON.parse(JSON.stringify(obj), (_, v) =>
|
||||
typeof v === 'object' && v !== null ? Object.freeze(v) : v,
|
||||
) as T;
|
||||
}
|
||||
|
||||
// ─── fixtures ────────────────────────────────────────────────────────────────
|
||||
|
||||
const ROW_ID = 'row1';
|
||||
|
||||
/** A layout with one row followed by two widgets. */
|
||||
function makeLayout(): Layout[] {
|
||||
return [
|
||||
{ i: ROW_ID, x: 0, y: 0, w: 12, h: 1 },
|
||||
{ i: 'w1', x: 0, y: 1, w: 6, h: 4 },
|
||||
{ i: 'w2', x: 6, y: 1, w: 6, h: 4 },
|
||||
];
|
||||
}
|
||||
|
||||
/** panelMap where the row is expanded (collapsed = false, widgets = []). */
|
||||
function makeExpandedPanelMap(): PanelMap {
|
||||
return {
|
||||
[ROW_ID]: { collapsed: false, widgets: [] },
|
||||
};
|
||||
}
|
||||
|
||||
/** panelMap where the row is collapsed (widgets stored inside). */
|
||||
function makeCollapsedPanelMap(): PanelMap {
|
||||
return {
|
||||
[ROW_ID]: {
|
||||
collapsed: true,
|
||||
widgets: [
|
||||
{ i: 'w1', x: 0, y: 1, w: 6, h: 4 },
|
||||
{ i: 'w2', x: 6, y: 1, w: 6, h: 4 },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ─── frozen-input guard (regression for zustand/immer read-only bug) ──────────
|
||||
|
||||
describe('applyRowCollapse – does not mutate frozen inputs', () => {
|
||||
it('does not throw when collapsing a row with frozen layout + panelMap', () => {
|
||||
expect(() =>
|
||||
applyRowCollapse(
|
||||
ROW_ID,
|
||||
freeze(makeLayout()),
|
||||
freeze(makeExpandedPanelMap()),
|
||||
),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('does not throw when expanding a row with frozen layout + panelMap', () => {
|
||||
// Collapsed layout only has the row item; widgets live in panelMap.
|
||||
const collapsedLayout = freeze([{ i: ROW_ID, x: 0, y: 0, w: 12, h: 1 }]);
|
||||
expect(() =>
|
||||
applyRowCollapse(ROW_ID, collapsedLayout, freeze(makeCollapsedPanelMap())),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('leaves the original layout array untouched after collapse', () => {
|
||||
const layout = makeLayout();
|
||||
const originalY = layout[1].y; // w1.y before collapse
|
||||
applyRowCollapse(ROW_ID, layout, makeExpandedPanelMap());
|
||||
expect(layout[1].y).toBe(originalY);
|
||||
});
|
||||
|
||||
it('leaves the original panelMap untouched after collapse', () => {
|
||||
const panelMap = makeExpandedPanelMap();
|
||||
applyRowCollapse(ROW_ID, makeLayout(), panelMap);
|
||||
expect(panelMap[ROW_ID].collapsed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── collapse behaviour ───────────────────────────────────────────────────────
|
||||
|
||||
describe('applyRowCollapse – collapsing a row', () => {
|
||||
it('sets collapsed = true on the row entry', () => {
|
||||
const { updatedPanelMap } = applyRowCollapse(
|
||||
ROW_ID,
|
||||
makeLayout(),
|
||||
makeExpandedPanelMap(),
|
||||
);
|
||||
expect(updatedPanelMap[ROW_ID].collapsed).toBe(true);
|
||||
});
|
||||
|
||||
it('stores the child widgets inside the panelMap entry', () => {
|
||||
const { updatedPanelMap } = applyRowCollapse(
|
||||
ROW_ID,
|
||||
makeLayout(),
|
||||
makeExpandedPanelMap(),
|
||||
);
|
||||
const ids = updatedPanelMap[ROW_ID].widgets.map((w) => w.i);
|
||||
expect(ids).toContain('w1');
|
||||
expect(ids).toContain('w2');
|
||||
});
|
||||
|
||||
it('removes child widgets from the returned layout', () => {
|
||||
const { updatedLayout } = applyRowCollapse(
|
||||
ROW_ID,
|
||||
makeLayout(),
|
||||
makeExpandedPanelMap(),
|
||||
);
|
||||
const ids = updatedLayout.map((l) => l.i);
|
||||
expect(ids).not.toContain('w1');
|
||||
expect(ids).not.toContain('w2');
|
||||
expect(ids).toContain(ROW_ID);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── expand behaviour ─────────────────────────────────────────────────────────
|
||||
|
||||
describe('applyRowCollapse – expanding a row', () => {
|
||||
it('sets collapsed = false on the row entry', () => {
|
||||
const collapsedLayout: Layout[] = [{ i: ROW_ID, x: 0, y: 0, w: 12, h: 1 }];
|
||||
const { updatedPanelMap } = applyRowCollapse(
|
||||
ROW_ID,
|
||||
collapsedLayout,
|
||||
makeCollapsedPanelMap(),
|
||||
);
|
||||
expect(updatedPanelMap[ROW_ID].collapsed).toBe(false);
|
||||
});
|
||||
|
||||
it('restores child widgets to the returned layout', () => {
|
||||
const collapsedLayout: Layout[] = [{ i: ROW_ID, x: 0, y: 0, w: 12, h: 1 }];
|
||||
const { updatedLayout } = applyRowCollapse(
|
||||
ROW_ID,
|
||||
collapsedLayout,
|
||||
makeCollapsedPanelMap(),
|
||||
);
|
||||
const ids = updatedLayout.map((l) => l.i);
|
||||
expect(ids).toContain('w1');
|
||||
expect(ids).toContain('w2');
|
||||
});
|
||||
|
||||
it('restored child widgets appear in both the layout and the panelMap entry', () => {
|
||||
const collapsedLayout: Layout[] = [{ i: ROW_ID, x: 0, y: 0, w: 12, h: 1 }];
|
||||
const { updatedLayout, updatedPanelMap } = applyRowCollapse(
|
||||
ROW_ID,
|
||||
collapsedLayout,
|
||||
makeCollapsedPanelMap(),
|
||||
);
|
||||
// The previously-stored widgets should now be back in the live layout.
|
||||
expect(updatedLayout.map((l) => l.i)).toContain('w1');
|
||||
// The panelMap entry still holds a reference to them (stale until next collapse).
|
||||
expect(updatedPanelMap[ROW_ID].widgets.map((w) => w.i)).toContain('w1');
|
||||
});
|
||||
});
|
||||
|
||||
// ─── y-offset adjustment ──────────────────────────────────────────────────────
|
||||
|
||||
describe('applyRowCollapse – y-offset adjustments for rows below', () => {
|
||||
it('shifts items below a second row down when the first row expands', () => {
|
||||
const ROW2 = 'row2';
|
||||
// Layout: row1 (y=0,h=1) | w1 (y=1,h=4) | row2 (y=5,h=1) | w3 (y=6,h=2)
|
||||
const layout: Layout[] = [
|
||||
{ i: ROW_ID, x: 0, y: 0, w: 12, h: 1 },
|
||||
{ i: 'w1', x: 0, y: 1, w: 12, h: 4 },
|
||||
{ i: ROW2, x: 0, y: 5, w: 12, h: 1 },
|
||||
{ i: 'w3', x: 0, y: 6, w: 12, h: 2 },
|
||||
];
|
||||
const panelMap: PanelMap = {
|
||||
[ROW_ID]: {
|
||||
collapsed: true,
|
||||
widgets: [{ i: 'w1', x: 0, y: 1, w: 12, h: 4 }],
|
||||
},
|
||||
[ROW2]: { collapsed: false, widgets: [] },
|
||||
};
|
||||
// Expanding row1 should push row2 and w3 down by the height of w1 (4).
|
||||
const collapsedLayout = layout.filter((l) => l.i !== 'w1');
|
||||
const { updatedLayout } = applyRowCollapse(ROW_ID, collapsedLayout, panelMap);
|
||||
|
||||
const row2Item = updatedLayout.find((l) => l.i === ROW2);
|
||||
expect(row2Item?.y).toBe(5 + 4); // shifted by maxY = 4
|
||||
});
|
||||
});
|
||||
@@ -4,122 +4,6 @@ import { isEmpty, isEqual } from 'lodash-es';
|
||||
import { Dashboard, Widgets } from 'types/api/dashboard/getAll';
|
||||
import { IBuilderQuery, Query } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
export type PanelMap = Record<
|
||||
string,
|
||||
{ widgets: Layout[]; collapsed: boolean }
|
||||
>;
|
||||
|
||||
export interface RowCollapseResult {
|
||||
updatedLayout: Layout[];
|
||||
updatedPanelMap: PanelMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure function that computes the new layout and panelMap after toggling a
|
||||
* row's collapsed state. All inputs are treated as immutable — no input object
|
||||
* is mutated, so it is safe to pass frozen objects from the zustand store.
|
||||
*/
|
||||
// eslint-disable-next-line sonarjs/cognitive-complexity
|
||||
export function applyRowCollapse(
|
||||
id: string,
|
||||
dashboardLayout: Layout[],
|
||||
currentPanelMap: PanelMap,
|
||||
): RowCollapseResult {
|
||||
// Deep-copy the row's own properties so we can mutate our local copy.
|
||||
const rowProperties = {
|
||||
...currentPanelMap[id],
|
||||
widgets: [...(currentPanelMap[id]?.widgets ?? [])],
|
||||
};
|
||||
|
||||
// Shallow-copy each entry's widgets array so inner .map() calls are safe.
|
||||
const updatedPanelMap: PanelMap = Object.fromEntries(
|
||||
Object.entries(currentPanelMap).map(([k, v]) => [
|
||||
k,
|
||||
{ ...v, widgets: [...v.widgets] },
|
||||
]),
|
||||
);
|
||||
|
||||
let updatedDashboardLayout = [...dashboardLayout];
|
||||
|
||||
if (rowProperties.collapsed === true) {
|
||||
// ── EXPAND ──────────────────────────────────────────────────────────────
|
||||
rowProperties.collapsed = false;
|
||||
const widgetsInsideTheRow = rowProperties.widgets;
|
||||
|
||||
let maxY = 0;
|
||||
widgetsInsideTheRow.forEach((w) => {
|
||||
maxY = Math.max(maxY, w.y + w.h);
|
||||
});
|
||||
const currentRowWidget = dashboardLayout.find((w) => w.i === id);
|
||||
if (currentRowWidget && widgetsInsideTheRow.length) {
|
||||
maxY -= currentRowWidget.h + currentRowWidget.y;
|
||||
}
|
||||
|
||||
const idxCurrentRow = dashboardLayout.findIndex((w) => w.i === id);
|
||||
for (let j = idxCurrentRow + 1; j < dashboardLayout.length; j++) {
|
||||
updatedDashboardLayout[j] = {
|
||||
...updatedDashboardLayout[j],
|
||||
y: updatedDashboardLayout[j].y + maxY,
|
||||
};
|
||||
if (updatedPanelMap[updatedDashboardLayout[j].i]) {
|
||||
updatedPanelMap[updatedDashboardLayout[j].i].widgets = updatedPanelMap[
|
||||
updatedDashboardLayout[j].i
|
||||
].widgets.map((w) => ({ ...w, y: w.y + maxY }));
|
||||
}
|
||||
}
|
||||
updatedDashboardLayout = [...updatedDashboardLayout, ...widgetsInsideTheRow];
|
||||
} else {
|
||||
// ── COLLAPSE ─────────────────────────────────────────────────────────────
|
||||
rowProperties.collapsed = true;
|
||||
const currentIdx = dashboardLayout.findIndex((w) => w.i === id);
|
||||
|
||||
let widgetsInsideTheRow: Layout[] = [];
|
||||
let isPanelMapUpdated = false;
|
||||
for (let j = currentIdx + 1; j < dashboardLayout.length; j++) {
|
||||
if (currentPanelMap[dashboardLayout[j].i]) {
|
||||
rowProperties.widgets = widgetsInsideTheRow;
|
||||
widgetsInsideTheRow = [];
|
||||
isPanelMapUpdated = true;
|
||||
break;
|
||||
} else {
|
||||
widgetsInsideTheRow.push(dashboardLayout[j]);
|
||||
}
|
||||
}
|
||||
if (!isPanelMapUpdated) {
|
||||
rowProperties.widgets = widgetsInsideTheRow;
|
||||
}
|
||||
|
||||
let maxY = 0;
|
||||
widgetsInsideTheRow.forEach((w) => {
|
||||
maxY = Math.max(maxY, w.y + w.h);
|
||||
});
|
||||
const currentRowWidget = dashboardLayout[currentIdx];
|
||||
if (currentRowWidget && widgetsInsideTheRow.length) {
|
||||
maxY -= currentRowWidget.h + currentRowWidget.y;
|
||||
}
|
||||
|
||||
for (let j = currentIdx + 1; j < updatedDashboardLayout.length; j++) {
|
||||
updatedDashboardLayout[j] = {
|
||||
...updatedDashboardLayout[j],
|
||||
y: updatedDashboardLayout[j].y + maxY,
|
||||
};
|
||||
if (updatedPanelMap[updatedDashboardLayout[j].i]) {
|
||||
updatedPanelMap[updatedDashboardLayout[j].i].widgets = updatedPanelMap[
|
||||
updatedDashboardLayout[j].i
|
||||
].widgets.map((w) => ({ ...w, y: w.y + maxY }));
|
||||
}
|
||||
}
|
||||
|
||||
updatedDashboardLayout = updatedDashboardLayout.filter(
|
||||
(widget) => !rowProperties.widgets.some((w: Layout) => w.i === widget.i),
|
||||
);
|
||||
}
|
||||
|
||||
updatedPanelMap[id] = { ...rowProperties };
|
||||
|
||||
return { updatedLayout: updatedDashboardLayout, updatedPanelMap };
|
||||
}
|
||||
|
||||
export const removeUndefinedValuesFromLayout = (layout: Layout[]): Layout[] =>
|
||||
layout.map((obj) =>
|
||||
Object.fromEntries(
|
||||
|
||||
@@ -89,7 +89,7 @@ function HostsList(): JSX.Element {
|
||||
...baseQuery,
|
||||
limit: pageSize,
|
||||
offset: (currentPage - 1) * pageSize,
|
||||
filters,
|
||||
filters: filters?.items?.length ? filters : undefined,
|
||||
start: Math.floor(minTime / 1000000),
|
||||
end: Math.floor(maxTime / 1000000),
|
||||
orderBy,
|
||||
@@ -97,15 +97,6 @@ function HostsList(): JSX.Element {
|
||||
}, [pageSize, currentPage, filters, minTime, maxTime, orderBy]);
|
||||
|
||||
const queryKey = useMemo(() => {
|
||||
if (selectedHostName) {
|
||||
return [
|
||||
'hostList',
|
||||
String(pageSize),
|
||||
String(currentPage),
|
||||
JSON.stringify(filters),
|
||||
JSON.stringify(orderBy),
|
||||
];
|
||||
}
|
||||
return [
|
||||
'hostList',
|
||||
String(pageSize),
|
||||
@@ -115,15 +106,7 @@ function HostsList(): JSX.Element {
|
||||
String(minTime),
|
||||
String(maxTime),
|
||||
];
|
||||
}, [
|
||||
pageSize,
|
||||
currentPage,
|
||||
filters,
|
||||
orderBy,
|
||||
selectedHostName,
|
||||
minTime,
|
||||
maxTime,
|
||||
]);
|
||||
}, [pageSize, currentPage, filters, orderBy, minTime, maxTime]);
|
||||
|
||||
const { data, isFetching, isLoading, isError } = useGetHostList(
|
||||
query as HostListPayload,
|
||||
|
||||
@@ -4,6 +4,7 @@ import { Provider } from 'react-redux';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { render } from '@testing-library/react';
|
||||
import * as useGetHostListHooks from 'hooks/infraMonitoring/useGetHostList';
|
||||
import { NuqsTestingAdapter } from 'nuqs/adapters/testing';
|
||||
import * as appContextHooks from 'providers/App/App';
|
||||
import * as timezoneHooks from 'providers/Timezone';
|
||||
import store from 'store';
|
||||
@@ -130,26 +131,30 @@ jest.spyOn(appContextHooks, 'useAppContext').mockReturnValue({
|
||||
describe('HostsList', () => {
|
||||
it('renders hosts list table', () => {
|
||||
const { container } = render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<MemoryRouter>
|
||||
<Provider store={store}>
|
||||
<HostsList />
|
||||
</Provider>
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>,
|
||||
<NuqsTestingAdapter>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<MemoryRouter>
|
||||
<Provider store={store}>
|
||||
<HostsList />
|
||||
</Provider>
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
</NuqsTestingAdapter>,
|
||||
);
|
||||
expect(container.querySelector('.hosts-list-table')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('renders filters', () => {
|
||||
const { container } = render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<MemoryRouter>
|
||||
<Provider store={store}>
|
||||
<HostsList />
|
||||
</Provider>
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>,
|
||||
<NuqsTestingAdapter>
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<MemoryRouter>
|
||||
<Provider store={store}>
|
||||
<HostsList />
|
||||
</Provider>
|
||||
</MemoryRouter>
|
||||
</QueryClientProvider>
|
||||
</NuqsTestingAdapter>,
|
||||
);
|
||||
expect(container.querySelector('.filters')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
@@ -92,10 +92,7 @@ function LogsActionsContainer({
|
||||
/>
|
||||
</div>
|
||||
<div className="download-options-container">
|
||||
<DownloadOptionsMenu
|
||||
dataSource={DataSource.LOGS}
|
||||
selectedColumns={options?.selectColumns}
|
||||
/>
|
||||
<DownloadOptionsMenu dataSource={DataSource.LOGS} />
|
||||
</div>
|
||||
<div className="format-options-container">
|
||||
<LogsFormatOptionsMenu
|
||||
|
||||
@@ -42,15 +42,8 @@ function LogsPanelComponent({
|
||||
setPageSize(value);
|
||||
setOffset(0);
|
||||
setRequestData((prev) => {
|
||||
const newQueryData = {
|
||||
...prev.query,
|
||||
builder: {
|
||||
...prev.query.builder,
|
||||
queryData: prev.query.builder.queryData.map((qd, i) =>
|
||||
i === 0 ? { ...qd, pageSize: value } : qd,
|
||||
),
|
||||
},
|
||||
};
|
||||
const newQueryData = { ...prev.query };
|
||||
newQueryData.builder.queryData[0].pageSize = value;
|
||||
return {
|
||||
...prev,
|
||||
query: newQueryData,
|
||||
|
||||
@@ -42,19 +42,11 @@ function Panel({
|
||||
};
|
||||
}
|
||||
|
||||
updatedQuery.builder.queryData[0].pageSize = 10;
|
||||
const initialDataSource = updatedQuery.builder.queryData[0].dataSource;
|
||||
const updatedQueryForList = {
|
||||
...updatedQuery,
|
||||
builder: {
|
||||
...updatedQuery.builder,
|
||||
queryData: updatedQuery.builder.queryData.map((qd, i) =>
|
||||
i === 0 ? { ...qd, pageSize: 10 } : qd,
|
||||
),
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
query: updatedQueryForList,
|
||||
query: updatedQuery,
|
||||
graphType: PANEL_TYPES.LIST,
|
||||
selectedTime: widget.timePreferance || 'GLOBAL_TIME',
|
||||
tableParams: {
|
||||
|
||||
@@ -239,10 +239,7 @@ function ListView({
|
||||
/>
|
||||
</div>
|
||||
|
||||
<DownloadOptionsMenu
|
||||
dataSource={DataSource.TRACES}
|
||||
selectedColumns={options?.selectColumns}
|
||||
/>
|
||||
<DownloadOptionsMenu dataSource={DataSource.TRACES} />
|
||||
|
||||
<TraceExplorerControls
|
||||
isLoading={isFetching}
|
||||
|
||||
@@ -52,44 +52,37 @@ export const useGetQueryRange: UseGetQueryRange = (
|
||||
!firstQueryData?.filters?.items.some((filter) => filter.key?.key === 'id') &&
|
||||
firstQueryData?.orderBy[0].columnName === 'timestamp';
|
||||
|
||||
if (
|
||||
isListWithSingleTimestampOrder &&
|
||||
firstQueryData?.dataSource === DataSource.LOGS
|
||||
) {
|
||||
return {
|
||||
...requestData,
|
||||
graphType:
|
||||
requestData.graphType === PANEL_TYPES.BAR
|
||||
? PANEL_TYPES.TIME_SERIES
|
||||
: requestData.graphType,
|
||||
query: {
|
||||
...requestData.query,
|
||||
builder: {
|
||||
...requestData.query.builder,
|
||||
queryData: [
|
||||
{
|
||||
...firstQueryData,
|
||||
orderBy: [
|
||||
...(firstQueryData?.orderBy || []),
|
||||
{
|
||||
columnName: 'id',
|
||||
order: firstQueryData?.orderBy[0]?.order,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
const modifiedRequestData = {
|
||||
...requestData,
|
||||
graphType:
|
||||
requestData.graphType === PANEL_TYPES.BAR
|
||||
? PANEL_TYPES.TIME_SERIES
|
||||
: requestData.graphType,
|
||||
};
|
||||
|
||||
// If the query is a list with a single timestamp order, we need to add the id column to the order by clause
|
||||
if (
|
||||
isListWithSingleTimestampOrder &&
|
||||
firstQueryData?.dataSource === DataSource.LOGS
|
||||
) {
|
||||
modifiedRequestData.query.builder = {
|
||||
...requestData.query.builder,
|
||||
queryData: [
|
||||
{
|
||||
...firstQueryData,
|
||||
orderBy: [
|
||||
...(firstQueryData?.orderBy || []),
|
||||
{
|
||||
columnName: 'id',
|
||||
order: firstQueryData?.orderBy[0]?.order,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
return modifiedRequestData;
|
||||
}, [requestData]);
|
||||
|
||||
const queryKey = useMemo(() => {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { useCallback, useState } from 'react';
|
||||
import { useSelector } from 'react-redux';
|
||||
import { message } from 'antd';
|
||||
import { downloadExportData } from 'api/v1/download/downloadExportData';
|
||||
import { prepareQueryRangePayloadV5, TelemetryFieldKey } from 'api/v5/v5';
|
||||
import { prepareQueryRangePayloadV5 } from 'api/v5/v5';
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
|
||||
import { AppState } from 'store/reducers';
|
||||
@@ -14,7 +14,6 @@ interface ExportOptions {
|
||||
format: string;
|
||||
rowLimit: number;
|
||||
clearSelectColumns: boolean;
|
||||
selectedColumns?: TelemetryFieldKey[];
|
||||
}
|
||||
|
||||
interface UseExportRawDataProps {
|
||||
@@ -43,7 +42,6 @@ export function useExportRawData({
|
||||
format,
|
||||
rowLimit,
|
||||
clearSelectColumns,
|
||||
selectedColumns,
|
||||
}: ExportOptions): Promise<void> => {
|
||||
if (!stagedQuery) {
|
||||
return;
|
||||
@@ -52,12 +50,6 @@ export function useExportRawData({
|
||||
try {
|
||||
setIsDownloading(true);
|
||||
|
||||
const selectColumnsOverride = clearSelectColumns
|
||||
? {}
|
||||
: selectedColumns?.length
|
||||
? { selectColumns: selectedColumns }
|
||||
: {};
|
||||
|
||||
const exportQuery = {
|
||||
...stagedQuery,
|
||||
builder: {
|
||||
@@ -67,7 +59,7 @@ export function useExportRawData({
|
||||
groupBy: [],
|
||||
having: { expression: '' },
|
||||
limit: rowLimit,
|
||||
...selectColumnsOverride,
|
||||
...(clearSelectColumns && { selectColumns: [] }),
|
||||
})),
|
||||
queryTraceOperator: (stagedQuery.builder.queryTraceOperator || []).map(
|
||||
(traceOp) => ({
|
||||
@@ -75,7 +67,7 @@ export function useExportRawData({
|
||||
groupBy: [],
|
||||
having: { expression: '' },
|
||||
limit: rowLimit,
|
||||
...selectColumnsOverride,
|
||||
...(clearSelectColumns && { selectColumns: [] }),
|
||||
}),
|
||||
),
|
||||
},
|
||||
|
||||
@@ -797,17 +797,17 @@ func (m *module) buildFilterClause(ctx context.Context, filter *qbtypes.Filter,
|
||||
}
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: m.logger,
|
||||
FieldMapper: m.fieldMapper,
|
||||
ConditionBuilder: m.condBuilder,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "metric_name", FieldContext: telemetrytypes.FieldContextMetric},
|
||||
FieldKeys: keys,
|
||||
StartNs: querybuilder.ToNanoSecs(uint64(startMillis)),
|
||||
EndNs: querybuilder.ToNanoSecs(uint64(endMillis)),
|
||||
}
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(expression, opts)
|
||||
startNs := querybuilder.ToNanoSecs(uint64(startMillis))
|
||||
endNs := querybuilder.ToNanoSecs(uint64(endMillis))
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(expression, opts, startNs, endNs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -66,7 +66,6 @@ func newProvider(
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataLocalTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
|
||||
// Create trace statement builder
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package tracedetail
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
@@ -64,22 +63,26 @@ func findIndexForSelectedSpanFromPreOrder(spans []*model.Span, selectedSpanId st
|
||||
return selectedSpanIndex
|
||||
}
|
||||
|
||||
func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string) (bool, []string) {
|
||||
func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, uncollapsedSpans []string, isSelectedSpanIDUnCollapsed bool) (bool, []string) {
|
||||
spansFromRootToNode := []string{}
|
||||
|
||||
spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
|
||||
if node.SpanID == selectedSpanId {
|
||||
if isSelectedSpanIDUnCollapsed && !slices.Contains(uncollapsedSpans, node.SpanID) {
|
||||
spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
|
||||
}
|
||||
return true, spansFromRootToNode
|
||||
}
|
||||
|
||||
isPresentInSubtreeForTheNode := false
|
||||
for _, child := range node.Children {
|
||||
isPresentInThisSubtree, _spansFromRootToNode := getPathFromRootToSelectedSpanId(child, selectedSpanId)
|
||||
isPresentInThisSubtree, _spansFromRootToNode := getPathFromRootToSelectedSpanId(child, selectedSpanId, uncollapsedSpans, isSelectedSpanIDUnCollapsed)
|
||||
// if the interested node is present in the given subtree then add the span node to uncollapsed node list
|
||||
if isPresentInThisSubtree {
|
||||
if !slices.Contains(uncollapsedSpans, node.SpanID) {
|
||||
spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
|
||||
}
|
||||
isPresentInSubtreeForTheNode = true
|
||||
spansFromRootToNode = append(spansFromRootToNode, _spansFromRootToNode...)
|
||||
break
|
||||
}
|
||||
}
|
||||
return isPresentInSubtreeForTheNode, spansFromRootToNode
|
||||
@@ -89,7 +92,7 @@ func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string) (b
|
||||
// throughout the recursion. Per-call state (level, isPartOfPreOrder, etc.)
|
||||
// is passed as direct arguments.
|
||||
type traverseOpts struct {
|
||||
uncollapsedSpans map[string]struct{}
|
||||
uncollapsedSpans []string
|
||||
selectedSpanID string
|
||||
}
|
||||
|
||||
@@ -131,7 +134,7 @@ func traverseTrace(span *model.Span, opts traverseOpts, level uint64, isPartOfPr
|
||||
preOrderTraversal = append(preOrderTraversal, &nodeWithoutChildren)
|
||||
}
|
||||
|
||||
_, isAlreadyUncollapsed := opts.uncollapsedSpans[span.SpanID]
|
||||
isAlreadyUncollapsed := slices.Contains(opts.uncollapsedSpans, span.SpanID)
|
||||
for index, child := range span.Children {
|
||||
_childTraversal := traverseTrace(child, opts, level+1, isPartOfPreOrder && isAlreadyUncollapsed, index != (len(span.Children)-1))
|
||||
preOrderTraversal = append(preOrderTraversal, _childTraversal...)
|
||||
@@ -166,28 +169,16 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
|
||||
|
||||
var preOrderTraversal = make([]*model.Span, 0)
|
||||
var rootServiceName, rootServiceEntryPoint string
|
||||
|
||||
// create a map of uncollapsed spans for quick lookup
|
||||
uncollapsedSpanMap := make(map[string]struct{})
|
||||
for _, spanID := range uncollapsedSpans {
|
||||
uncollapsedSpanMap[spanID] = struct{}{}
|
||||
}
|
||||
updatedUncollapsedSpans := uncollapsedSpans
|
||||
|
||||
selectedSpanIndex := -1
|
||||
for _, rootSpanID := range traceRoots {
|
||||
if rootNode, exists := spanIdToSpanNodeMap[rootSpanID.SpanID]; exists {
|
||||
present, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID)
|
||||
if present {
|
||||
for _, spanID := range spansFromRootToNode {
|
||||
if selectedSpanID == spanID && !isSelectedSpanIDUnCollapsed {
|
||||
continue
|
||||
}
|
||||
uncollapsedSpanMap[spanID] = struct{}{}
|
||||
}
|
||||
}
|
||||
_, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID, updatedUncollapsedSpans, isSelectedSpanIDUnCollapsed)
|
||||
updatedUncollapsedSpans = append(updatedUncollapsedSpans, spansFromRootToNode...)
|
||||
|
||||
opts := traverseOpts{
|
||||
uncollapsedSpans: uncollapsedSpanMap,
|
||||
uncollapsedSpans: updatedUncollapsedSpans,
|
||||
selectedSpanID: selectedSpanID,
|
||||
}
|
||||
_preOrderTraversal := traverseTrace(rootNode, opts, 0, true, false)
|
||||
@@ -232,5 +223,5 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
|
||||
startIndex = 0
|
||||
}
|
||||
|
||||
return preOrderTraversal[startIndex:endIndex], slices.Collect(maps.Keys(uncollapsedSpanMap)), rootServiceName, rootServiceEntryPoint
|
||||
return preOrderTraversal[startIndex:endIndex], updatedUncollapsedSpans, rootServiceName, rootServiceEntryPoint
|
||||
}
|
||||
|
||||
@@ -48,8 +48,6 @@ func NewAggExprRewriter(
|
||||
// and the args if the parametric aggregation function is used.
|
||||
func (r *aggExprRewriter) Rewrite(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
expr string,
|
||||
rateInterval uint64,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
@@ -76,12 +74,7 @@ func (r *aggExprRewriter) Rewrite(
|
||||
return "", nil, errors.NewInternalf(errors.CodeInternal, "no SELECT items for %q", expr)
|
||||
}
|
||||
|
||||
visitor := newExprVisitor(
|
||||
ctx,
|
||||
startNs,
|
||||
endNs,
|
||||
r.logger,
|
||||
keys,
|
||||
visitor := newExprVisitor(r.logger, keys,
|
||||
r.fullTextColumn,
|
||||
r.fieldMapper,
|
||||
r.conditionBuilder,
|
||||
@@ -101,8 +94,6 @@ func (r *aggExprRewriter) Rewrite(
|
||||
// RewriteMulti rewrites a slice of expressions.
|
||||
func (r *aggExprRewriter) RewriteMulti(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
exprs []string,
|
||||
rateInterval uint64,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
@@ -111,7 +102,7 @@ func (r *aggExprRewriter) RewriteMulti(
|
||||
var errs []error
|
||||
var chArgsList [][]any
|
||||
for i, e := range exprs {
|
||||
w, chArgs, err := r.Rewrite(ctx, startNs, endNs, e, rateInterval, keys)
|
||||
w, chArgs, err := r.Rewrite(ctx, e, rateInterval, keys)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
out[i] = e
|
||||
@@ -128,9 +119,6 @@ func (r *aggExprRewriter) RewriteMulti(
|
||||
|
||||
// exprVisitor walks FunctionExpr nodes and applies the mappers.
|
||||
type exprVisitor struct {
|
||||
ctx context.Context
|
||||
startNs uint64
|
||||
endNs uint64
|
||||
chparser.DefaultASTVisitor
|
||||
logger *slog.Logger
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
@@ -144,9 +132,6 @@ type exprVisitor struct {
|
||||
}
|
||||
|
||||
func newExprVisitor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
logger *slog.Logger,
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
@@ -155,9 +140,6 @@ func newExprVisitor(
|
||||
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
|
||||
) *exprVisitor {
|
||||
return &exprVisitor{
|
||||
ctx: ctx,
|
||||
startNs: startNs,
|
||||
endNs: endNs,
|
||||
logger: logger,
|
||||
fieldKeys: fieldKeys,
|
||||
fullTextColumn: fullTextColumn,
|
||||
@@ -204,16 +186,13 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
whereClause, err := PrepareWhereClause(
|
||||
origPred,
|
||||
FilterExprVisitorOpts{
|
||||
Context: v.ctx,
|
||||
Logger: v.logger,
|
||||
FieldKeys: v.fieldKeys,
|
||||
FieldMapper: v.fieldMapper,
|
||||
ConditionBuilder: v.conditionBuilder,
|
||||
FullTextColumn: v.fullTextColumn,
|
||||
JsonKeyToKey: v.jsonKeyToKey,
|
||||
StartNs: v.startNs,
|
||||
EndNs: v.endNs,
|
||||
},
|
||||
}, 0, 0,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -233,7 +212,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
origVal := args[i].String()
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(v.ctx, v.startNs, v.endNs, &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to get table field name for %q", origVal)
|
||||
}
|
||||
@@ -251,7 +230,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
for i, arg := range args {
|
||||
orig := arg.String()
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(v.ctx, v.startNs, v.endNs, &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -147,7 +147,12 @@ func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemet
|
||||
// So we can safely override the context and data type
|
||||
|
||||
actions = append(actions, fmt.Sprintf("Overriding key: %s to %s", key, intrinsicOrCalculatedField))
|
||||
key.OverrideMetadataFrom(intrinsicOrCalculatedField)
|
||||
key.FieldContext = intrinsicOrCalculatedField.FieldContext
|
||||
key.FieldDataType = intrinsicOrCalculatedField.FieldDataType
|
||||
key.JSONDataType = intrinsicOrCalculatedField.JSONDataType
|
||||
key.Indexes = intrinsicOrCalculatedField.Indexes
|
||||
key.Materialized = intrinsicOrCalculatedField.Materialized
|
||||
key.JSONPlan = intrinsicOrCalculatedField.JSONPlan
|
||||
return actions
|
||||
|
||||
}
|
||||
@@ -193,9 +198,13 @@ func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemet
|
||||
if !key.Equal(matchingKey) {
|
||||
actions = append(actions, fmt.Sprintf("Adjusting key %s to %s", key, matchingKey))
|
||||
}
|
||||
|
||||
key.Name = matchingKey.Name
|
||||
key.OverrideMetadataFrom(matchingKey)
|
||||
key.FieldContext = matchingKey.FieldContext
|
||||
key.FieldDataType = matchingKey.FieldDataType
|
||||
key.JSONDataType = matchingKey.JSONDataType
|
||||
key.Indexes = matchingKey.Indexes
|
||||
key.Materialized = matchingKey.Materialized
|
||||
key.JSONPlan = matchingKey.JSONPlan
|
||||
|
||||
return actions
|
||||
} else {
|
||||
|
||||
@@ -19,8 +19,6 @@ import (
|
||||
|
||||
func CollisionHandledFinalExpr(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
fm qbtypes.FieldMapper,
|
||||
cb qbtypes.ConditionBuilder,
|
||||
@@ -46,7 +44,7 @@ func CollisionHandledFinalExpr(
|
||||
|
||||
addCondition := func(key *telemetrytypes.TelemetryFieldKey) error {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
condition, err := cb.ConditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
condition, err := cb.ConditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb, 0, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -59,7 +57,7 @@ func CollisionHandledFinalExpr(
|
||||
return nil
|
||||
}
|
||||
|
||||
fieldExpression, fieldForErr := fm.FieldFor(ctx, startNs, endNs, field)
|
||||
colName, fieldForErr := fm.FieldFor(ctx, field)
|
||||
if errors.Is(fieldForErr, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -94,9 +92,9 @@ func CollisionHandledFinalExpr(
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
fieldExpression, _ = fm.FieldFor(ctx, startNs, endNs, key)
|
||||
fieldExpression, _ = DataTypeCollisionHandledFieldName(key, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown)
|
||||
stmts = append(stmts, fieldExpression)
|
||||
colName, _ = fm.FieldFor(ctx, key)
|
||||
colName, _ = DataTypeCollisionHandledFieldName(key, dummyValue, colName, qbtypes.FilterOperatorUnknown)
|
||||
stmts = append(stmts, colName)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -111,10 +109,10 @@ func CollisionHandledFinalExpr(
|
||||
} else if strings.Contains(field.Name, telemetrytypes.ArraySep) || strings.Contains(field.Name, telemetrytypes.ArrayAnyIndex) {
|
||||
return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the Array Paths: %s", field.Name)
|
||||
} else {
|
||||
fieldExpression, _ = DataTypeCollisionHandledFieldName(field, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown)
|
||||
colName, _ = DataTypeCollisionHandledFieldName(field, dummyValue, colName, qbtypes.FilterOperatorUnknown)
|
||||
}
|
||||
|
||||
stmts = append(stmts, fieldExpression)
|
||||
stmts = append(stmts, colName)
|
||||
}
|
||||
|
||||
for idx := range stmts {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -45,12 +44,12 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
|
||||
|
||||
func (b *defaultConditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
op qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
@@ -61,23 +60,15 @@ func (b *defaultConditionBuilder) ConditionFor(
|
||||
// as we store resource values as string
|
||||
formattedValue := querybuilder.FormatValueForContains(value)
|
||||
|
||||
columns, err := b.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
column, err := b.fm.ColumnFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(columns) != 1 {
|
||||
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
|
||||
}
|
||||
|
||||
// resource evolution on main table doesn't affect this
|
||||
// as we have not changed the resource column in the resource fingerprint table.
|
||||
column := columns[0]
|
||||
|
||||
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
|
||||
valueForIndexFilter := valueForIndexFilter(op, key, value)
|
||||
|
||||
fieldName, err := b.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
fieldName, err := b.fm.FieldFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -206,7 +206,7 @@ func TestConditionBuilder(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(context.Background(), 0, 0, tc.key, tc.op, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(context.Background(), tc.key, tc.op, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedErr != nil {
|
||||
|
||||
@@ -27,50 +27,46 @@ func NewFieldMapper() *defaultFieldMapper {
|
||||
|
||||
func (m *defaultFieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) ([]*schema.Column, error) {
|
||||
) (*schema.Column, error) {
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
return []*schema.Column{resourceColumns["labels"]}, nil
|
||||
return resourceColumns["labels"], nil
|
||||
}
|
||||
if col, ok := resourceColumns[key.Name]; ok {
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) ColumnFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) FieldFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
column, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", column.Name, key.Name), nil
|
||||
}
|
||||
return columns[0].Name, nil
|
||||
return column.Name, nil
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
_ map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
fieldExpression, err := m.FieldFor(ctx, tsStart, tsEnd, key)
|
||||
colName, err := m.FieldFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s AS `%s`", fieldExpression, key.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", colName, key.Name), nil
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ func (b *resourceFilterStatementBuilder[T]) Build(
|
||||
|
||||
// addConditions adds both filter and time conditions to the query
|
||||
func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
ctx context.Context,
|
||||
_ context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[T],
|
||||
@@ -160,7 +160,6 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
|
||||
// warnings would be encountered as part of the main condition already
|
||||
filterWhereClause, err := querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fieldMapper,
|
||||
ConditionBuilder: b.conditionBuilder,
|
||||
@@ -172,9 +171,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
// there is no need for "key" not found error for resource filtering
|
||||
IgnoreNotFoundKeys: true,
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -23,7 +23,6 @@ const stringMatchingOperatorDocURL = "https://signoz.io/docs/userguide/operators
|
||||
// filterExpressionVisitor implements the FilterQueryVisitor interface
|
||||
// to convert the parsed filter expressions into ClickHouse WHERE clause
|
||||
type filterExpressionVisitor struct {
|
||||
context context.Context
|
||||
logger *slog.Logger
|
||||
fieldMapper qbtypes.FieldMapper
|
||||
conditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -47,7 +46,6 @@ type filterExpressionVisitor struct {
|
||||
}
|
||||
|
||||
type FilterExprVisitorOpts struct {
|
||||
Context context.Context
|
||||
Logger *slog.Logger
|
||||
FieldMapper qbtypes.FieldMapper
|
||||
ConditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -67,7 +65,6 @@ type FilterExprVisitorOpts struct {
|
||||
// newFilterExpressionVisitor creates a new filterExpressionVisitor
|
||||
func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVisitor {
|
||||
return &filterExpressionVisitor{
|
||||
context: opts.Context,
|
||||
logger: opts.Logger,
|
||||
fieldMapper: opts.FieldMapper,
|
||||
conditionBuilder: opts.ConditionBuilder,
|
||||
@@ -93,7 +90,7 @@ type PreparedWhereClause struct {
|
||||
}
|
||||
|
||||
// PrepareWhereClause generates a ClickHouse compatible WHERE clause from the filter query
|
||||
func PrepareWhereClause(query string, opts FilterExprVisitorOpts) (*PreparedWhereClause, error) {
|
||||
func PrepareWhereClause(query string, opts FilterExprVisitorOpts, startNs uint64, endNs uint64) (*PreparedWhereClause, error) {
|
||||
|
||||
// Setup the ANTLR parsing pipeline
|
||||
input := antlr.NewInputStream(query)
|
||||
@@ -127,6 +124,8 @@ func PrepareWhereClause(query string, opts FilterExprVisitorOpts) (*PreparedWher
|
||||
}
|
||||
tokens.Reset()
|
||||
|
||||
opts.StartNs = startNs
|
||||
opts.EndNs = endNs
|
||||
visitor := newFilterExpressionVisitor(opts)
|
||||
|
||||
// Handle syntax errors
|
||||
@@ -332,7 +331,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
return ""
|
||||
}
|
||||
}
|
||||
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.startNs, v.endNs, v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder)
|
||||
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder, v.startNs, v.endNs)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -375,7 +374,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
}
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, nil, v.builder)
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, nil, v.builder, v.startNs, v.endNs)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -448,7 +447,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
}
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, values, v.builder)
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, values, v.builder, v.startNs, v.endNs)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -496,7 +495,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, []any{value1, value2}, v.builder)
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, []any{value1, value2}, v.builder, v.startNs, v.endNs)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -581,7 +580,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, value, v.builder)
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, value, v.builder, v.startNs, v.endNs)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -659,7 +658,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
|
||||
v.errors = append(v.errors, "full text search is not supported")
|
||||
return ""
|
||||
}
|
||||
cond, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder)
|
||||
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder, v.startNs, v.endNs)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -745,13 +744,13 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
|
||||
if key.FieldContext == telemetrytypes.FieldContextBody {
|
||||
var err error
|
||||
if BodyJSONQueryEnabled {
|
||||
fieldName, err = v.fieldMapper.FieldFor(v.context, v.startNs, v.endNs, key)
|
||||
fieldName, err = v.fieldMapper.FieldFor(context.Background(), key)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to get field name for key %s: %s", key.Name, err.Error()))
|
||||
return ""
|
||||
}
|
||||
} else {
|
||||
fieldName, _ = v.jsonKeyToKey(v.context, key, qbtypes.FilterOperatorUnknown, value)
|
||||
fieldName, _ = v.jsonKeyToKey(context.Background(), key, qbtypes.FilterOperatorUnknown, value)
|
||||
}
|
||||
} else {
|
||||
// TODO(add docs for json body search)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -55,12 +54,11 @@ func TestPrepareWhereClause_EmptyVariableList(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
opts := FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
FieldKeys: keys,
|
||||
Variables: tt.variables,
|
||||
}
|
||||
|
||||
_, err := PrepareWhereClause(tt.expr, opts)
|
||||
_, err := PrepareWhereClause(tt.expr, opts, 0, 0)
|
||||
|
||||
if tt.expectError {
|
||||
if err == nil {
|
||||
@@ -469,7 +467,7 @@ func TestVisitKey(t *testing.T) {
|
||||
expectedWarnings: nil,
|
||||
expectedMainWrnURL: "",
|
||||
},
|
||||
{
|
||||
{
|
||||
name: "only attribute.custom_field is selected",
|
||||
keyText: "attribute.attribute.custom_field",
|
||||
fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
@@ -394,7 +394,6 @@ func New(
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataLocalTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
|
||||
global, err := factory.NewProviderFromNamedMap(
|
||||
|
||||
@@ -23,61 +23,56 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
column, err := c.fm.ColumnFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
|
||||
for _, column := range columns {
|
||||
if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled && key.Name != messageSubField {
|
||||
valueType, value := InferDataType(value, operator, key)
|
||||
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return cond, nil
|
||||
if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled && key.Name != messageSubField {
|
||||
valueType, value := InferDataType(value, operator, key)
|
||||
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return cond, nil
|
||||
}
|
||||
|
||||
if operator.IsStringSearchOperator() {
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Check if this is a body JSON search - either by FieldContext
|
||||
if key.FieldContext == telemetrytypes.FieldContextBody && !querybuilder.BodyJSONQueryEnabled {
|
||||
fieldExpression, value = GetBodyJSONKey(ctx, key, operator, value)
|
||||
tblFieldName, value = GetBodyJSONKey(ctx, key, operator, value)
|
||||
}
|
||||
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
|
||||
|
||||
// make use of case insensitive index for body
|
||||
if fieldExpression == "body" || fieldExpression == messageSubColumn {
|
||||
if tblFieldName == "body" || tblFieldName == messageSubColumn {
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,41 +80,40 @@ func (c *conditionBuilder) conditionFor(
|
||||
switch operator {
|
||||
// regular operators
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.G(fieldExpression, value), nil
|
||||
return sb.G(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThanOrEq:
|
||||
return sb.GE(fieldExpression, value), nil
|
||||
return sb.GE(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorLessThan:
|
||||
return sb.LT(fieldExpression, value), nil
|
||||
return sb.LT(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorLessThanOrEq:
|
||||
return sb.LE(fieldExpression, value), nil
|
||||
return sb.LE(tblFieldName, value), nil
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.Like(fieldExpression, value), nil
|
||||
return sb.Like(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotLike(fieldExpression, value), nil
|
||||
return sb.NotLike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorILike:
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
|
||||
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
// between and not between
|
||||
case qbtypes.FilterOperatorBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -129,7 +123,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
return sb.Between(tblFieldName, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -138,7 +132,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
return sb.NotBetween(tblFieldName, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -149,7 +143,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
conditions = append(conditions, sb.E(tblFieldName, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -160,7 +154,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
conditions = append(conditions, sb.NE(tblFieldName, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
@@ -177,61 +171,36 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
|
||||
var value any
|
||||
column := columns[0]
|
||||
if len(key.Evolutions) > 0 {
|
||||
// we will use the corresponding column and its evolution entry for the query
|
||||
newColumns, _, err := selectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(newColumns) == 0 {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name)
|
||||
}
|
||||
|
||||
// This mean tblFieldName is with multiIf, we just need to do a null check.
|
||||
if len(newColumns) > 1 {
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.IsNotNull(fieldExpression), nil
|
||||
} else {
|
||||
return sb.IsNull(fieldExpression), nil
|
||||
}
|
||||
}
|
||||
|
||||
// otherwise we have to find the correct exist operator based on the column type
|
||||
column = newColumns[0]
|
||||
}
|
||||
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.IsNotNull(fieldExpression), nil
|
||||
return sb.IsNotNull(tblFieldName), nil
|
||||
}
|
||||
return sb.IsNull(fieldExpression), nil
|
||||
return sb.IsNull(tblFieldName), nil
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
}
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
case schema.ColumnTypeEnumString:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
} else {
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
|
||||
value = 0
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
} else {
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
@@ -255,7 +224,6 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
|
||||
|
||||
}
|
||||
}
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
|
||||
@@ -263,15 +231,14 @@ func (c *conditionBuilder) conditionFor(
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
|
||||
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
|
||||
condition, err := c.conditionFor(ctx, key, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -294,7 +261,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
|
||||
if buildExistCondition {
|
||||
existsCondition, err := c.conditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
existsCondition, err := c.conditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@ package telemetrylogs
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -12,148 +11,14 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExistsConditionForWithEvolutions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
startTs uint64
|
||||
endTs uint64
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
operator qbtypes.FilterOperator
|
||||
value any
|
||||
expectedSQL string
|
||||
expectedArgs []any
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "New column",
|
||||
startTs: uint64(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
endTs: uint64(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Old column",
|
||||
startTs: uint64(time.Date(2023, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
endTs: uint64(time.Date(2023, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Both Old column and new - empty filter",
|
||||
startTs: uint64(time.Date(2023, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
endTs: uint64(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, tc.startTs, tc.endTs, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, tc.expectedSQL)
|
||||
assert.Equal(t, tc.expectedArgs, args)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionFor(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
mockEvolution := mockEvolutionData(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC))
|
||||
testCases := []struct {
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
operator qbtypes.FilterOperator
|
||||
value any
|
||||
evolutions []*telemetrytypes.EvolutionEntry
|
||||
expectedSQL string
|
||||
expectedArgs []any
|
||||
expectedError error
|
||||
@@ -376,11 +241,9 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "mapContains(resources_string, 'service.name') = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -390,11 +253,9 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorNotExists,
|
||||
value: nil,
|
||||
expectedSQL: "mapContains(resources_string, 'service.name') <> ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NULL",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -455,11 +316,10 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorRegexp,
|
||||
value: "frontend-.*",
|
||||
expectedSQL: "WHERE (match(`resource_string_service$$name`, ?) AND `resource_string_service$$name_exists` = ?)",
|
||||
expectedArgs: []any{"frontend-.*", true},
|
||||
expectedSQL: "(match(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL), ?) AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL) IS NOT NULL)",
|
||||
expectedArgs: []any{"frontend-.*"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -470,10 +330,9 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorNotRegexp,
|
||||
value: "test-.*",
|
||||
expectedSQL: "WHERE NOT match(`resource_string_service$$name`, ?)",
|
||||
expectedSQL: "WHERE NOT match(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL), ?)",
|
||||
expectedArgs: []any{"test-.*"},
|
||||
expectedError: nil,
|
||||
},
|
||||
@@ -513,13 +372,14 @@ func TestConditionFor(t *testing.T) {
|
||||
expectedError: qbtypes.ErrColumnNotFound,
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.key.Evolutions = tc.evolutions
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
@@ -574,7 +434,7 @@ func TestConditionForMultipleKeys(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var err error
|
||||
for _, key := range tc.keys {
|
||||
cond, err := conditionBuilder.conditionFor(ctx, 0, 0, &key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &key, tc.operator, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting condition for key %s: %v", key.Name, err)
|
||||
@@ -831,7 +691,7 @@ func TestConditionForJSONBodySearch(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.conditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -3,11 +3,7 @@ package telemetrylogs
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz-otel-collector/utils"
|
||||
@@ -71,42 +67,40 @@ type fieldMapper struct{}
|
||||
func NewFieldMapper() qbtypes.FieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
columns := []*schema.Column{logsV2Columns["resources_string"], logsV2Columns["resource"]}
|
||||
return columns, nil
|
||||
return logsV2Columns["resource"], nil
|
||||
case telemetrytypes.FieldContextScope:
|
||||
switch key.Name {
|
||||
case "name", "scope.name", "scope_name":
|
||||
return []*schema.Column{logsV2Columns["scope_name"]}, nil
|
||||
return logsV2Columns["scope_name"], nil
|
||||
case "version", "scope.version", "scope_version":
|
||||
return []*schema.Column{logsV2Columns["scope_version"]}, nil
|
||||
return logsV2Columns["scope_version"], nil
|
||||
}
|
||||
return []*schema.Column{logsV2Columns["scope_string"]}, nil
|
||||
return logsV2Columns["scope_string"], nil
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
switch key.FieldDataType {
|
||||
case telemetrytypes.FieldDataTypeString:
|
||||
return []*schema.Column{logsV2Columns["attributes_string"]}, nil
|
||||
return logsV2Columns["attributes_string"], nil
|
||||
case telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeNumber:
|
||||
return []*schema.Column{logsV2Columns["attributes_number"]}, nil
|
||||
return logsV2Columns["attributes_number"], nil
|
||||
case telemetrytypes.FieldDataTypeBool:
|
||||
return []*schema.Column{logsV2Columns["attributes_bool"]}, nil
|
||||
return logsV2Columns["attributes_bool"], nil
|
||||
}
|
||||
case telemetrytypes.FieldContextBody:
|
||||
// Body context is for JSON body fields. Use body_v2 if feature flag is enabled.
|
||||
if querybuilder.BodyJSONQueryEnabled {
|
||||
if key.Name == messageSubField {
|
||||
return []*schema.Column{logsV2Columns[messageSubColumn]}, nil
|
||||
return logsV2Columns[messageSubColumn], nil
|
||||
}
|
||||
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}, nil
|
||||
return logsV2Columns[LogsV2BodyV2Column], nil
|
||||
}
|
||||
// Fall back to legacy body column
|
||||
return []*schema.Column{logsV2Columns["body"]}, nil
|
||||
return logsV2Columns["body"], nil
|
||||
case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextUnspecified:
|
||||
if key.Name == LogsV2BodyColumn && querybuilder.BodyJSONQueryEnabled {
|
||||
return []*schema.Column{logsV2Columns[messageSubColumn]}, nil
|
||||
return logsV2Columns[messageSubColumn], nil
|
||||
}
|
||||
col, ok := logsV2Columns[key.Name]
|
||||
if !ok {
|
||||
@@ -114,248 +108,100 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry
|
||||
if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
|
||||
// Use body_v2 if feature flag is enabled and we have a body condition builder
|
||||
if querybuilder.BodyJSONQueryEnabled {
|
||||
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
|
||||
// i.e return both the body json and body json promoted and let the evolutions decide which one to use
|
||||
// based on the query range time.
|
||||
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}, nil
|
||||
return logsV2Columns[LogsV2BodyV2Column], nil
|
||||
}
|
||||
// Fall back to legacy body column
|
||||
return []*schema.Column{logsV2Columns["body"]}, nil
|
||||
return logsV2Columns["body"], nil
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
}
|
||||
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
// selectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
|
||||
// Logic:
|
||||
// - Finds the latest base evolution (<= tsStartTime) across ALL columns
|
||||
// - Rejects all evolutions before this latest base evolution
|
||||
// - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
|
||||
// - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
|
||||
// - Results are sorted by ReleaseTime descending (newest first)
|
||||
func selectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
|
||||
|
||||
sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
|
||||
copy(sortedEvolutions, evolutions)
|
||||
|
||||
// sort the evolutions by ReleaseTime ascending
|
||||
sort.Slice(sortedEvolutions, func(i, j int) bool {
|
||||
return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
|
||||
})
|
||||
|
||||
tsStartTime := time.Unix(0, int64(tsStart))
|
||||
tsEndTime := time.Unix(0, int64(tsEnd))
|
||||
|
||||
// Build evolution map: column name -> evolution
|
||||
evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
|
||||
for _, evolution := range sortedEvolutions {
|
||||
if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
|
||||
// since if there is duplicate we would just use the oldest one.
|
||||
continue
|
||||
}
|
||||
evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
|
||||
}
|
||||
|
||||
// Find the latest base evolution (<= tsStartTime) across ALL columns
|
||||
// Evolutions are sorted, so we can break early
|
||||
var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
|
||||
for _, evolution := range sortedEvolutions {
|
||||
if evolution.ReleaseTime.After(tsStartTime) {
|
||||
break
|
||||
}
|
||||
latestBaseEvolutionAcrossAll = evolution
|
||||
}
|
||||
|
||||
// We shouldn't reach this, it basically means there is something wrong with the evolutions data
|
||||
if latestBaseEvolutionAcrossAll == nil {
|
||||
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
|
||||
}
|
||||
|
||||
columnLookUpMap := make(map[string]*schema.Column)
|
||||
for _, column := range columns {
|
||||
columnLookUpMap[column.Name] = column
|
||||
}
|
||||
|
||||
// Collect column-evolution pairs
|
||||
type colEvoPair struct {
|
||||
column *schema.Column
|
||||
evolution *telemetrytypes.EvolutionEntry
|
||||
}
|
||||
pairs := []colEvoPair{}
|
||||
|
||||
for _, evolution := range evolutionMap {
|
||||
// Reject evolutions before the latest base evolution
|
||||
if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
|
||||
continue
|
||||
}
|
||||
// skip evolutions after tsEndTime
|
||||
if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
|
||||
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
|
||||
}
|
||||
|
||||
pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
|
||||
}
|
||||
|
||||
// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
|
||||
if len(pairs) == 0 {
|
||||
for _, column := range columns {
|
||||
// Use latestBaseEvolutionAcrossAll if this column name matches its column name
|
||||
if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
|
||||
pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by ReleaseTime descending (newest first)
|
||||
slices.SortFunc(pairs, func(a, b colEvoPair) int {
|
||||
// Sort by ReleaseTime descending (newest first)
|
||||
if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
|
||||
return -1
|
||||
}
|
||||
if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
|
||||
// Extract results
|
||||
newColumns := make([]*schema.Column, len(pairs))
|
||||
evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
|
||||
for i, pair := range pairs {
|
||||
newColumns[i] = pair.column
|
||||
evolutionsEntries[i] = pair.evolution
|
||||
}
|
||||
|
||||
return newColumns, evolutionsEntries, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, key)
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var newColumns []*schema.Column
|
||||
var evolutionsEntries []*telemetrytypes.EvolutionEntry
|
||||
if len(key.Evolutions) > 0 {
|
||||
// we will use the corresponding column and its evolution entry for the query
|
||||
newColumns, evolutionsEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
newColumns = columns
|
||||
}
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
// json is only supported for resource context as of now
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
oldColumn := logsV2Columns["resources_string"]
|
||||
oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
|
||||
|
||||
exprs := []string{}
|
||||
existExpr := []string{}
|
||||
for i, column := range newColumns {
|
||||
// Use evolution column name if available, otherwise use the column name
|
||||
columnName := column.Name
|
||||
if evolutionsEntries != nil && evolutionsEntries[i] != nil {
|
||||
columnName = evolutionsEntries[i].ColumnName
|
||||
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
|
||||
// once clickHouse dependency is updated, we need to check if we can remove it.
|
||||
if key.Materialized {
|
||||
oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
|
||||
oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
|
||||
}
|
||||
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if key.Name == messageSubField {
|
||||
return messageSubColumn, nil
|
||||
}
|
||||
|
||||
if key.JSONDataType == nil {
|
||||
return "", qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
if key.KeyNameContainsArray() && !key.JSONDataType.IsArray {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "FieldFor not supported for nested fields; only supported for flat paths (e.g. body.status.detail) and paths of Array type: %s(%s)", key.Name, key.FieldDataType)
|
||||
}
|
||||
|
||||
return m.buildFieldForJSON(key)
|
||||
default:
|
||||
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String)
|
||||
}
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
return column.Name, nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
|
||||
return column.Name, nil
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
|
||||
}
|
||||
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if key.Name == messageSubField {
|
||||
exprs = append(exprs, messageSubColumn)
|
||||
continue
|
||||
}
|
||||
|
||||
if key.JSONDataType == nil {
|
||||
return "", qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
if key.KeyNameContainsArray() && !key.JSONDataType.IsArray {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "FieldFor not supported for nested fields; only supported for flat paths (e.g. body.status.detail) and paths of Array type: %s(%s)", key.Name, key.FieldDataType)
|
||||
}
|
||||
|
||||
expr, err := m.buildFieldForJSON(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
exprs = append(exprs, expr)
|
||||
default:
|
||||
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String)
|
||||
}
|
||||
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
exprs = append(exprs, column.Name)
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
|
||||
exprs = append(exprs, column.Name)
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
|
||||
}
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)))
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
|
||||
}
|
||||
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
}
|
||||
}
|
||||
|
||||
if len(exprs) == 1 {
|
||||
return exprs[0], nil
|
||||
} else if len(exprs) > 1 {
|
||||
// Ensure existExpr has the same length as exprs
|
||||
if len(existExpr) != len(exprs) {
|
||||
return "", errors.New(errors.TypeInternal, errors.CodeInternal, "length of exist exprs doesn't match to that of exprs")
|
||||
}
|
||||
finalExprs := []string{}
|
||||
for i, expr := range exprs {
|
||||
finalExprs = append(finalExprs, fmt.Sprintf("%s, %s", existExpr[i], expr))
|
||||
}
|
||||
return "multiIf(" + strings.Join(finalExprs, ", ") + ", NULL)", nil
|
||||
}
|
||||
|
||||
// should not reach here
|
||||
return columns[0].Name, nil
|
||||
return column.Name, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
fieldExpression, err := m.FieldFor(ctx, tsStart, tsEnd, field)
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -365,7 +211,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
if _, ok := logsV2Columns[field.Name]; ok {
|
||||
// if it is, attach the column name directly
|
||||
field.FieldContext = telemetrytypes.FieldContextLog
|
||||
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, field)
|
||||
colName, _ = m.FieldFor(ctx, field)
|
||||
} else {
|
||||
// - the context is not provided
|
||||
// - there are not keys for the field
|
||||
@@ -383,19 +229,19 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
} else if len(keysForField) == 1 {
|
||||
// we have a single key for the field, use it
|
||||
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, keysForField[0])
|
||||
colName, _ = m.FieldFor(ctx, keysForField[0])
|
||||
} else {
|
||||
// select any non-empty value from the keys
|
||||
args := []string{}
|
||||
for _, key := range keysForField {
|
||||
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression))
|
||||
colName, _ = m.FieldFor(ctx, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", colName, colName))
|
||||
}
|
||||
fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
colName = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
}
|
||||
|
||||
// buildFieldForJSON builds the field expression for body JSON fields using arrayConcat pattern
|
||||
|
||||
@@ -3,7 +3,6 @@ package telemetrylogs
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -18,7 +17,7 @@ func TestGetColumn(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedCol []*schema.Column
|
||||
expectedCol *schema.Column
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
@@ -27,7 +26,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["resources_string"], logsV2Columns["resource"]},
|
||||
expectedCol: logsV2Columns["resource"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -36,7 +35,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "name",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_name"]},
|
||||
expectedCol: logsV2Columns["scope_name"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -45,7 +44,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "scope.name",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_name"]},
|
||||
expectedCol: logsV2Columns["scope_name"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -54,7 +53,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "scope_name",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_name"]},
|
||||
expectedCol: logsV2Columns["scope_name"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -63,7 +62,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "version",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_version"]},
|
||||
expectedCol: logsV2Columns["scope_version"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -72,7 +71,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "custom.scope.field",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_string"]},
|
||||
expectedCol: logsV2Columns["scope_string"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -82,7 +81,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_string"]},
|
||||
expectedCol: logsV2Columns["attributes_string"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -92,7 +91,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_number"]},
|
||||
expectedCol: logsV2Columns["attributes_number"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -102,7 +101,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeInt64,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_number"]},
|
||||
expectedCol: logsV2Columns["attributes_number"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -112,7 +111,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeFloat64,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_number"]},
|
||||
expectedCol: logsV2Columns["attributes_number"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -122,7 +121,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_bool"]},
|
||||
expectedCol: logsV2Columns["attributes_bool"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -131,7 +130,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "timestamp",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["timestamp"]},
|
||||
expectedCol: logsV2Columns["timestamp"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -140,7 +139,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "body",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["body"]},
|
||||
expectedCol: logsV2Columns["body"],
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -160,7 +159,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_bool"]},
|
||||
expectedCol: logsV2Columns["attributes_bool"],
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
@@ -169,7 +168,7 @@ func TestGetColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
col, err := fm.ColumnFor(ctx, 0, 0, &tc.key)
|
||||
col, err := fm.ColumnFor(ctx, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
@@ -184,14 +183,11 @@ func TestGetColumn(t *testing.T) {
|
||||
func TestGetFieldKeyName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
resourceEvolution := mockEvolutionData(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC))
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedResult string
|
||||
expectedError error
|
||||
addExistsFilter bool
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedResult string
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "Simple column type - timestamp",
|
||||
@@ -199,9 +195,8 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
Name: "timestamp",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
expectedResult: "timestamp",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
expectedResult: "timestamp",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Map column type - string attribute",
|
||||
@@ -210,9 +205,8 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
expectedResult: "attributes_string['user.id']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
expectedResult: "attributes_string['user.id']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Map column type - number attribute",
|
||||
@@ -221,9 +215,8 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber,
|
||||
},
|
||||
expectedResult: "attributes_number['request.size']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
expectedResult: "attributes_number['request.size']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Map column type - bool attribute",
|
||||
@@ -232,33 +225,28 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
expectedResult: "attributes_bool['request.success']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
expectedResult: "attributes_bool['request.success']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Map column type - resource attribute",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
Evolutions: resourceEvolution,
|
||||
},
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Map column type - resource attribute - Materialized - json",
|
||||
name: "Map column type - resource attribute - Materialized",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
Evolutions: resourceEvolution,
|
||||
},
|
||||
expectedResult: "`resource_string_service$$name`",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Non-existent column",
|
||||
@@ -274,7 +262,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
@@ -285,693 +273,3 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldForWithEvolutions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
evolutions []*telemetrytypes.EvolutionEntry
|
||||
key *telemetrytypes.TelemetryFieldKey
|
||||
tsStartTime time.Time
|
||||
tsEndTime time.Time
|
||||
expectedResult string
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "Single evolution before tsStartTime",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Single evolution exactly at tsStartTime",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Single evolution after tsStartTime",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
// TODO(piyush): to be added once integration with JSON is done.
|
||||
// {
|
||||
// name: "Single evolution after tsStartTime - JSON body",
|
||||
// evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
// {
|
||||
// Signal: telemetrytypes.SignalLogs,
|
||||
// ColumnName: LogsV2BodyV2Column,
|
||||
// ColumnType: "JSON(max_dynamic_paths=0)",
|
||||
// FieldContext: telemetrytypes.FieldContextBody,
|
||||
// FieldName: "__all__",
|
||||
// ReleaseTime: time.Unix(0, 0),
|
||||
// },
|
||||
// {
|
||||
// Signal: telemetrytypes.SignalLogs,
|
||||
// ColumnName: LogsV2BodyPromotedColumn,
|
||||
// ColumnType: "JSON()",
|
||||
// FieldContext: telemetrytypes.FieldContextBody,
|
||||
// FieldName: "user.name",
|
||||
// ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
// },
|
||||
// },
|
||||
// key: &telemetrytypes.TelemetryFieldKey{
|
||||
// Name: "user.name",
|
||||
// FieldContext: telemetrytypes.FieldContextBody,
|
||||
// JSONDataType: &telemetrytypes.String,
|
||||
// Materialized: true,
|
||||
// },
|
||||
// tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
// tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
// expectedResult: "coalesce(dynamicElement(body_json.`user.name`, 'String'), dynamicElement(body_promoted.`user.name`, 'String'))",
|
||||
// expectedError: nil,
|
||||
// },
|
||||
{
|
||||
name: "Multiple evolutions before tsStartTime - only latest should be included",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resource.`service.name`::String",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Multiple evolutions after tsStartTime - all should be included",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Unix(0, 0),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Duplicate evolutions after tsStartTime - all should be included",
|
||||
// Note: on production when this happens, we should go ahead and clean it up if required
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resource.`service.name`::String",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Evolution exactly at tsEndTime - should not be included",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
|
||||
tsStart := uint64(tc.tsStartTime.UnixNano())
|
||||
tsEnd := uint64(tc.tsEndTime.UnixNano())
|
||||
tc.key.Evolutions = tc.evolutions
|
||||
|
||||
result, err := fm.FieldFor(ctx, tsStart, tsEnd, tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedResult, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectEvolutionsForColumns(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
columns []*schema.Column
|
||||
evolutions []*telemetrytypes.EvolutionEntry
|
||||
tsStart uint64
|
||||
tsEnd uint64
|
||||
expectedColumns []string // column names
|
||||
expectedEvols []string // evolution column names
|
||||
expectedError bool
|
||||
errorStr string
|
||||
}{
|
||||
{
|
||||
name: "New evolutions at tsStartTime - should include latest evolution",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resource"},
|
||||
expectedEvols: []string{"resource"},
|
||||
},
|
||||
{
|
||||
name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
|
||||
expectedEvols: []string{"resource", "resources_string"},
|
||||
},
|
||||
{
|
||||
name: "Columns without matching evolutions - should exclude them",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"], // no evolution for this
|
||||
logsV2Columns["attributes_string"], // no evolution for this
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"},
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "New evolutions at tsEndTime - should not include new evolution",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"},
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "New evolutions after tsEndTime - should exclude new",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"},
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "Empty columns array",
|
||||
columns: []*schema.Column{},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{},
|
||||
expectedEvols: []string{},
|
||||
expectedError: true,
|
||||
errorStr: "column resources_string not found",
|
||||
},
|
||||
{
|
||||
name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resource"},
|
||||
expectedEvols: []string{"resource"}, // should use first one (older)
|
||||
},
|
||||
{
|
||||
name: "Genuine Duplicate evolutions with new version- should consider both",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 2,
|
||||
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string", "resource"},
|
||||
expectedEvols: []string{"resources_string", "resource"}, // should use first one (older)
|
||||
},
|
||||
{
|
||||
name: "Evolution exactly at tsEndTime",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "Single evolution after tsStartTime - JSON body",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns[LogsV2BodyV2Column],
|
||||
logsV2Columns[LogsV2BodyPromotedColumn],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyV2Column,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyPromotedColumn,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "user.name",
|
||||
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
|
||||
expectedEvols: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
|
||||
},
|
||||
{
|
||||
name: "No evolution after tsStartTime - JSON body",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns[LogsV2BodyV2Column],
|
||||
logsV2Columns[LogsV2BodyPromotedColumn],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyV2Column,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyPromotedColumn,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "user.name",
|
||||
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{LogsV2BodyPromotedColumn},
|
||||
expectedEvols: []string{LogsV2BodyPromotedColumn},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resultColumns, resultEvols, err := selectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
|
||||
|
||||
if tc.expectedError {
|
||||
assert.Contains(t, err.Error(), tc.errorStr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
|
||||
assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
|
||||
|
||||
resultColumnNames := make([]string, len(resultColumns))
|
||||
for i, col := range resultColumns {
|
||||
resultColumnNames[i] = col.Name
|
||||
}
|
||||
resultEvolNames := make([]string, len(resultEvols))
|
||||
for i, evol := range resultEvols {
|
||||
resultEvolNames[i] = evol.ColumnName
|
||||
}
|
||||
|
||||
for i := range tc.expectedColumns {
|
||||
assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
|
||||
}
|
||||
for i := range tc.expectedEvols {
|
||||
assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
|
||||
}
|
||||
// Verify sorting: should be descending by ReleaseTime
|
||||
for i := 0; i < len(resultEvols)-1; i++ {
|
||||
assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
|
||||
"evolutions should be sorted descending by ReleaseTime")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldForWithMaterialized(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
materializedKey := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
start, end time.Time
|
||||
expectedResult string
|
||||
}{
|
||||
{
|
||||
name: "Map column in use (pre-evolution to JSON)",
|
||||
start: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "`resource_string_service$$name`",
|
||||
},
|
||||
{
|
||||
name: "Multi evolution - both columns (JSON + materialized)",
|
||||
start: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2024, 4, 2, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL)",
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
start := uint64(tc.start.UnixNano())
|
||||
end := uint64(tc.end.UnixNano())
|
||||
result, err := fm.FieldFor(ctx, start, end, materializedKey)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedResult, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -12,15 +10,12 @@ import (
|
||||
|
||||
// TestLikeAndILikeWithoutWildcards_Warns Tests that LIKE/ILIKE without wildcards add warnings and include docs URL
|
||||
func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -38,7 +33,7 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
|
||||
|
||||
for _, expr := range tests {
|
||||
t.Run(expr, func(t *testing.T) {
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts)
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts, 0, 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, clause)
|
||||
|
||||
@@ -54,11 +49,9 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -76,7 +69,7 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
|
||||
|
||||
for _, expr := range tests {
|
||||
t.Run(expr, func(t *testing.T) {
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts)
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts, 0, 0)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, clause)
|
||||
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -18,11 +16,9 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -165,7 +161,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s: %s", tc.category, limitString(tc.query, 50)), func(t *testing.T) {
|
||||
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts)
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts, 0, 0)
|
||||
|
||||
if tc.shouldPass {
|
||||
if err != nil {
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
@@ -17,24 +15,19 @@ import (
|
||||
|
||||
// TestFilterExprLogs tests a comprehensive set of query patterns for logs search
|
||||
func TestFilterExprLogs(t *testing.T) {
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: DefaultFullTextColumn,
|
||||
JsonKeyToKey: GetBodyJSONKey,
|
||||
StartNs: uint64(releaseTime.Add(-5 * time.Minute).UnixNano()),
|
||||
EndNs: uint64(releaseTime.Add(5 * time.Minute).UnixNano()),
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
@@ -473,7 +466,7 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
|
||||
//fulltext with parenthesized expression
|
||||
// fulltext with parenthesized expression
|
||||
{
|
||||
category: "FREETEXT with parentheses",
|
||||
query: "error (status.code=500 OR status.code=503)",
|
||||
@@ -2393,7 +2386,7 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s: %s", tc.category, limitString(tc.query, 50)), func(t *testing.T) {
|
||||
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts)
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts, 0, 0)
|
||||
|
||||
if tc.shouldPass {
|
||||
if err != nil {
|
||||
@@ -2433,8 +2426,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
|
||||
keys["body"] = []*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
@@ -2450,7 +2442,6 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
}
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -2513,7 +2504,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s: %s", tc.category, limitString(tc.query, 50)), func(t *testing.T) {
|
||||
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts)
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts, 0, 0)
|
||||
|
||||
if tc.shouldPass {
|
||||
if err != nil {
|
||||
|
||||
@@ -283,7 +283,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
// get column expression for the field - use array index directly to avoid pointer to loop variable
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &query.SelectFields[index], keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &query.SelectFields[index], keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -292,6 +292,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
|
||||
|
||||
@@ -301,8 +302,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
|
||||
// Add order by
|
||||
for _, orderBy := range query.Order {
|
||||
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -368,7 +368,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
// Keep original column expressions so we can build the tuple
|
||||
fieldNames := make([]string, 0, len(query.GroupBy))
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -383,7 +383,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
allAggChArgs := make([]any, 0)
|
||||
for i, agg := range query.Aggregations {
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, start, end, agg.Expression,
|
||||
ctx, agg.Expression,
|
||||
uint64(query.StepInterval.Seconds()),
|
||||
keys,
|
||||
)
|
||||
@@ -515,7 +515,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
var allGroupByArgs []any
|
||||
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -533,7 +533,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
for idx := range query.Aggregations {
|
||||
aggExpr := query.Aggregations[idx]
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, start, end, aggExpr.Expression,
|
||||
ctx, aggExpr.Expression,
|
||||
rateInterval,
|
||||
keys,
|
||||
)
|
||||
@@ -605,7 +605,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
|
||||
// buildFilterCondition builds SQL condition from filter expression
|
||||
func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
ctx context.Context,
|
||||
_ context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
|
||||
@@ -619,7 +619,6 @@ func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
// add filter expression
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
@@ -628,9 +627,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
FullTextColumn: b.fullTextColumn,
|
||||
JsonKeyToKey: b.jsonKeyToKey,
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -19,7 +19,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation
|
||||
fm := resourcefilter.NewFieldMapper()
|
||||
cb := resourcefilter.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keysMap := buildCompleteFieldKeyMap(time.Now())
|
||||
keysMap := buildCompleteFieldKeyMap()
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
key.Signal = telemetrytypes.SignalLogs
|
||||
@@ -38,14 +38,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation
|
||||
}
|
||||
|
||||
func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
releaseTimeNano := uint64(releaseTime.UnixNano())
|
||||
|
||||
cases := []struct {
|
||||
startTs uint64
|
||||
endTs uint64
|
||||
name string
|
||||
requestType qbtypes.RequestType
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
||||
@@ -53,16 +46,14 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
expectedErr error
|
||||
}{
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with limit and count distinct on service.name",
|
||||
name: "Time series with limit",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.LogAggregation{
|
||||
{
|
||||
Expression: "count_distinct(service.name)",
|
||||
Expression: "count()",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
@@ -78,22 +69,20 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano - uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with OR b/w resource attr and attribute filter and count distinct on service.name",
|
||||
name: "Time series with OR b/w resource attr and attribute filter",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.LogAggregation{
|
||||
{
|
||||
Expression: "count_distinct(service.name)",
|
||||
Expression: "count()",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
@@ -109,14 +98,12 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1705224600), uint64(1705485600), "redis-manual", "GET", true, "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600), 10, "redis-manual", "GET", true, "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600)},
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "redis-manual", "GET", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "redis-manual", "GET", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with limit + custom order by",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -150,14 +137,12 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name` ORDER BY `service.name` desc, ts desc",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name` ORDER BY `service.name` desc, ts desc",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with group by on materialized column",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -184,12 +169,10 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(`attribute_string_materialized$$key$$name_exists` = ?, `attribute_string_materialized$$key$$name`, NULL)) AS `materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `materialized.key.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(`attribute_string_materialized$$key$$name_exists` = ?, `attribute_string_materialized$$key$$name`, NULL)) AS `materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`materialized.key.name`) GLOBAL IN (SELECT `materialized.key.name` FROM __limit_cte) GROUP BY ts, `materialized.key.name`",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
},
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with materialised column using or with regex operator",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -207,19 +190,14 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (true OR true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts",
|
||||
Args: []any{uint64(1705397400), uint64(1705485600), "redis.*", true, "memcached", true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
Args: []any{uint64(1747945619), uint64(1747983448), "redis.*", true, "memcached", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keysMap := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
mockMetadataStore.KeysMap = keysMap
|
||||
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
@@ -241,7 +219,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(ctx, c.startTs, c.endTs, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -338,13 +316,9 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -365,7 +339,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -482,12 +456,9 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -505,10 +476,12 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
GetBodyJSONKey,
|
||||
)
|
||||
|
||||
//
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -559,12 +532,9 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -585,7 +555,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErrContains != "" {
|
||||
require.Error(t, err)
|
||||
@@ -657,10 +627,9 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
fm := NewFieldMapper()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -681,7 +650,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -699,9 +668,6 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAdjustKey(t *testing.T) {
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
inputKey telemetrytypes.TelemetryFieldKey
|
||||
@@ -715,7 +681,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: IntrinsicFields["severity_text"],
|
||||
},
|
||||
{
|
||||
@@ -752,7 +718,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "severity_number",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
@@ -766,8 +732,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "single matching key with incorrect context specified - no override",
|
||||
@@ -776,7 +742,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -790,8 +756,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "multiple matching keys - all materialized",
|
||||
@@ -800,7 +766,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "multi.mat.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -814,7 +780,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "mixed.materialization.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -828,8 +794,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["mixed.materialization.key"][0],
|
||||
},
|
||||
{
|
||||
name: "no matching keys - unknown field",
|
||||
@@ -838,7 +804,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
@@ -853,7 +819,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -868,8 +834,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["mat.key"][0],
|
||||
},
|
||||
{
|
||||
name: "non-materialized field",
|
||||
@@ -878,8 +844,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["user.id"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["user.id"][0],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ package telemetrylogs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
@@ -19,7 +18,7 @@ func limitString(s string, maxLen int) string {
|
||||
}
|
||||
|
||||
// Function to build a complete field key map for testing all scenarios
|
||||
func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"service.name": {
|
||||
{
|
||||
@@ -944,9 +943,6 @@ func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
key.Signal = telemetrytypes.SignalLogs
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
key.Evolutions = mockEvolutionData(releaseTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1012,24 +1008,3 @@ func buildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryF
|
||||
}
|
||||
return keysMap
|
||||
}
|
||||
|
||||
func mockEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {
|
||||
return []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: releaseTime,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,11 +21,12 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
|
||||
switch operator {
|
||||
@@ -38,13 +39,13 @@ func (c *conditionBuilder) ConditionFor(
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
columns, err := c.fm.ColumnFor(ctx, tsStart, tsEnd, key)
|
||||
column, err := c.fm.ColumnFor(ctx, key)
|
||||
if err != nil {
|
||||
// if we don't have a column, we can't build a condition for related values
|
||||
return "", nil
|
||||
}
|
||||
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, tsStart, tsEnd, key)
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
if err != nil {
|
||||
// if we don't have a table field name, we can't build a condition for related values
|
||||
return "", nil
|
||||
@@ -56,7 +57,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return "", nil
|
||||
}
|
||||
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
|
||||
|
||||
// key must exists to apply main filter
|
||||
expr := `if(mapContains(%s, %s), %s, true)`
|
||||
@@ -67,29 +68,29 @@ func (c *conditionBuilder) ConditionFor(
|
||||
switch operator {
|
||||
// regular operators
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
cond = sb.E(fieldExpression, value)
|
||||
cond = sb.E(tblFieldName, value)
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
cond = sb.NE(fieldExpression, value)
|
||||
cond = sb.NE(tblFieldName, value)
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
cond = sb.Like(fieldExpression, value)
|
||||
cond = sb.Like(tblFieldName, value)
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
cond = sb.NotLike(fieldExpression, value)
|
||||
cond = sb.NotLike(tblFieldName, value)
|
||||
case qbtypes.FilterOperatorILike:
|
||||
cond = sb.ILike(fieldExpression, value)
|
||||
cond = sb.ILike(tblFieldName, value)
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
cond = sb.NotILike(fieldExpression, value)
|
||||
cond = sb.NotILike(tblFieldName, value)
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
cond = sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value))
|
||||
cond = sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value))
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
cond = sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value))
|
||||
cond = sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value))
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
cond = fmt.Sprintf(`match(%s, %s)`, fieldExpression, sb.Var(value))
|
||||
cond = fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value))
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
cond = fmt.Sprintf(`NOT match(%s, %s)`, fieldExpression, sb.Var(value))
|
||||
cond = fmt.Sprintf(`NOT match(%s, %s)`, tblFieldName, sb.Var(value))
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -100,7 +101,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
conditions = append(conditions, sb.E(tblFieldName, value))
|
||||
}
|
||||
cond = sb.Or(conditions...)
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -111,7 +112,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
conditions = append(conditions, sb.NE(tblFieldName, value))
|
||||
}
|
||||
cond = sb.And(conditions...)
|
||||
|
||||
@@ -119,12 +120,12 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// in the query builder, `exists` and `not exists` are used for
|
||||
// key membership checks, so depending on the column type, the condition changes
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
switch columns[0].Type {
|
||||
switch column.Type {
|
||||
case schema.MapColumnType{
|
||||
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
|
||||
ValueType: schema.ColumnTypeString,
|
||||
}:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
cond = sb.E(leftOperand, true)
|
||||
} else {
|
||||
@@ -133,5 +134,5 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf(expr, columns[0].Name, sb.Var(key.Name), cond), nil
|
||||
return fmt.Sprintf(expr, column.Name, sb.Var(key.Name), cond), nil
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestConditionFor(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -33,48 +33,47 @@ func NewFieldMapper() qbtypes.FieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(_ context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return []*schema.Column{attributeMetadataColumns["resource_attributes"]}, nil
|
||||
return attributeMetadataColumns["resource_attributes"], nil
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
return []*schema.Column{attributeMetadataColumns["attributes"]}, nil
|
||||
return attributeMetadataColumns["attributes"], nil
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return columns, nil
|
||||
return column, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch columns[0].Type {
|
||||
switch column.Type {
|
||||
case schema.MapColumnType{
|
||||
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
|
||||
ValueType: schema.ColumnTypeString,
|
||||
}:
|
||||
return fmt.Sprintf("%s['%s']", columns[0].Name, key.Name), nil
|
||||
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
|
||||
}
|
||||
return columns[0].Name, nil
|
||||
return column.Name, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
fieldExpression, err := m.FieldFor(ctx, startNs, endNs, field)
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -84,7 +83,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
if _, ok := attributeMetadataColumns[field.Name]; ok {
|
||||
// if it is, attach the column name directly
|
||||
field.FieldContext = telemetrytypes.FieldContextSpan
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, field)
|
||||
colName, _ = m.FieldFor(ctx, field)
|
||||
} else {
|
||||
// - the context is not provided
|
||||
// - there are not keys for the field
|
||||
@@ -102,17 +101,17 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
} else if len(keysForField) == 1 {
|
||||
// we have a single key for the field, use it
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, keysForField[0])
|
||||
colName, _ = m.FieldFor(ctx, keysForField[0])
|
||||
} else {
|
||||
// select any non-empty value from the keys
|
||||
args := []string{}
|
||||
for _, key := range keysForField {
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression))
|
||||
colName, _ = m.FieldFor(ctx, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", colName, colName))
|
||||
}
|
||||
fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
colName = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
}
|
||||
|
||||
@@ -128,13 +128,13 @@ func TestGetColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
col, err := fm.ColumnFor(context.Background(), 0, 0, &tc.key)
|
||||
col, err := fm.ColumnFor(context.Background(), &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedCol, col[0])
|
||||
assert.Equal(t, tc.expectedCol, col)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -145,8 +145,6 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
tsStart uint64
|
||||
tsEnd uint64
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedResult string
|
||||
expectedError error
|
||||
@@ -205,7 +203,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result, err := fm.FieldFor(ctx, tc.tsStart, tc.tsEnd, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"golang.org/x/exp/maps"
|
||||
@@ -35,24 +34,23 @@ var (
|
||||
)
|
||||
|
||||
type telemetryMetaStore struct {
|
||||
logger *slog.Logger
|
||||
telemetrystore telemetrystore.TelemetryStore
|
||||
tracesDBName string
|
||||
tracesFieldsTblName string
|
||||
spanAttributesKeysTblName string
|
||||
indexV3TblName string
|
||||
metricsDBName string
|
||||
metricsFieldsTblName string
|
||||
meterDBName string
|
||||
meterFieldsTblName string
|
||||
logsDBName string
|
||||
logsFieldsTblName string
|
||||
logAttributeKeysTblName string
|
||||
logResourceKeysTblName string
|
||||
logsV2TblName string
|
||||
relatedMetadataDBName string
|
||||
relatedMetadataTblName string
|
||||
columnEvolutionMetadataTblName string
|
||||
logger *slog.Logger
|
||||
telemetrystore telemetrystore.TelemetryStore
|
||||
tracesDBName string
|
||||
tracesFieldsTblName string
|
||||
spanAttributesKeysTblName string
|
||||
indexV3TblName string
|
||||
metricsDBName string
|
||||
metricsFieldsTblName string
|
||||
meterDBName string
|
||||
meterFieldsTblName string
|
||||
logsDBName string
|
||||
logsFieldsTblName string
|
||||
logAttributeKeysTblName string
|
||||
logResourceKeysTblName string
|
||||
logsV2TblName string
|
||||
relatedMetadataDBName string
|
||||
relatedMetadataTblName string
|
||||
|
||||
fm qbtypes.FieldMapper
|
||||
conditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -81,29 +79,27 @@ func NewTelemetryMetaStore(
|
||||
logResourceKeysTblName string,
|
||||
relatedMetadataDBName string,
|
||||
relatedMetadataTblName string,
|
||||
columnEvolutionMetadataTblName string,
|
||||
) telemetrytypes.MetadataStore {
|
||||
metadataSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetadata")
|
||||
|
||||
t := &telemetryMetaStore{
|
||||
logger: metadataSettings.Logger(),
|
||||
telemetrystore: telemetrystore,
|
||||
tracesDBName: tracesDBName,
|
||||
tracesFieldsTblName: tracesFieldsTblName,
|
||||
spanAttributesKeysTblName: spanAttributesKeysTblName,
|
||||
indexV3TblName: indexV3TblName,
|
||||
metricsDBName: metricsDBName,
|
||||
metricsFieldsTblName: metricsFieldsTblName,
|
||||
meterDBName: meterDBName,
|
||||
meterFieldsTblName: meterFieldsTblName,
|
||||
logsDBName: logsDBName,
|
||||
logsV2TblName: logsV2TblName,
|
||||
logsFieldsTblName: logsFieldsTblName,
|
||||
logAttributeKeysTblName: logAttributeKeysTblName,
|
||||
logResourceKeysTblName: logResourceKeysTblName,
|
||||
relatedMetadataDBName: relatedMetadataDBName,
|
||||
relatedMetadataTblName: relatedMetadataTblName,
|
||||
columnEvolutionMetadataTblName: columnEvolutionMetadataTblName,
|
||||
logger: metadataSettings.Logger(),
|
||||
telemetrystore: telemetrystore,
|
||||
tracesDBName: tracesDBName,
|
||||
tracesFieldsTblName: tracesFieldsTblName,
|
||||
spanAttributesKeysTblName: spanAttributesKeysTblName,
|
||||
indexV3TblName: indexV3TblName,
|
||||
metricsDBName: metricsDBName,
|
||||
metricsFieldsTblName: metricsFieldsTblName,
|
||||
meterDBName: meterDBName,
|
||||
meterFieldsTblName: meterFieldsTblName,
|
||||
logsDBName: logsDBName,
|
||||
logsV2TblName: logsV2TblName,
|
||||
logsFieldsTblName: logsFieldsTblName,
|
||||
logAttributeKeysTblName: logAttributeKeysTblName,
|
||||
logResourceKeysTblName: logResourceKeysTblName,
|
||||
relatedMetadataDBName: relatedMetadataDBName,
|
||||
relatedMetadataTblName: relatedMetadataTblName,
|
||||
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
|
||||
telemetrytypes.SignalLogs: {
|
||||
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
|
||||
@@ -592,11 +588,6 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
|
||||
keys = append(keys, bodyJSONPaths...)
|
||||
complete = complete && finished
|
||||
}
|
||||
|
||||
if _, err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
}
|
||||
|
||||
@@ -1036,18 +1027,18 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
|
||||
FieldDataType: fieldValueSelector.FieldDataType,
|
||||
}
|
||||
|
||||
selectColumn, err := t.fm.FieldFor(ctx, 0, 0, key)
|
||||
selectColumn, err := t.fm.FieldFor(ctx, key)
|
||||
|
||||
if err != nil {
|
||||
// we don't have a explicit column to select from the related metadata table
|
||||
// so we will select either from resource_attributes or attributes table
|
||||
// in that order
|
||||
resourceColumn, _ := t.fm.FieldFor(ctx, 0, 0, &telemetrytypes.TelemetryFieldKey{
|
||||
resourceColumn, _ := t.fm.FieldFor(ctx, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key.Name,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
})
|
||||
attributeColumn, _ := t.fm.FieldFor(ctx, 0, 0, &telemetrytypes.TelemetryFieldKey{
|
||||
attributeColumn, _ := t.fm.FieldFor(ctx, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key.Name,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -1068,12 +1059,11 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
|
||||
}
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(fieldValueSelector.ExistingQuery, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: t.logger,
|
||||
FieldMapper: t.fm,
|
||||
ConditionBuilder: t.conditionBuilder,
|
||||
FieldKeys: keys,
|
||||
})
|
||||
}, 0, 0)
|
||||
if err == nil {
|
||||
sb.AddWhereClause(whereClause.WhereClause)
|
||||
} else {
|
||||
@@ -1097,20 +1087,20 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
|
||||
|
||||
// search on attributes
|
||||
key.FieldContext = telemetrytypes.FieldContextAttribute
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, 0, 0, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb)
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
|
||||
if err == nil {
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
|
||||
// search on resource
|
||||
key.FieldContext = telemetrytypes.FieldContextResource
|
||||
cond, err = t.conditionBuilder.ConditionFor(ctx, 0, 0, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb)
|
||||
cond, err = t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
|
||||
if err == nil {
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
key.FieldContext = origContext
|
||||
} else {
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, 0, 0, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb)
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
|
||||
if err == nil {
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
@@ -1854,113 +1844,6 @@ func (t *telemetryMetaStore) fetchMeterSourceMetricsTemporalityAndType(ctx conte
|
||||
return temporalities, types, nil
|
||||
}
|
||||
|
||||
func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Context, selectors []*telemetrytypes.EvolutionSelector) ([]*telemetrytypes.EvolutionEntry, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("signal", "column_name", "column_type", "field_context", "field_name", "version", "release_time")
|
||||
sb.From(fmt.Sprintf("%s.%s", k.relatedMetadataDBName, k.columnEvolutionMetadataTblName))
|
||||
sb.OrderBy("release_time ASC")
|
||||
|
||||
var clauses []string
|
||||
for _, selector := range selectors {
|
||||
var clause string
|
||||
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
clause = sb.E("field_context", selector.FieldContext)
|
||||
}
|
||||
|
||||
clause = sb.And(clause,
|
||||
sb.Or(sb.E("field_name", selector.FieldName), sb.E("field_name", "__all__")),
|
||||
)
|
||||
|
||||
clauses = append(clauses, sb.And(sb.E("signal", selector.Signal), clause))
|
||||
}
|
||||
sb.Where(sb.Or(clauses...))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var entries []*telemetrytypes.EvolutionEntry
|
||||
rows, err := k.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var entry telemetrytypes.EvolutionEntry
|
||||
var releaseTimeNs float64
|
||||
if err := rows.Scan(
|
||||
&entry.Signal,
|
||||
&entry.ColumnName,
|
||||
&entry.ColumnType,
|
||||
&entry.FieldContext,
|
||||
&entry.FieldName,
|
||||
&entry.Version,
|
||||
&releaseTimeNs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Convert nanoseconds to time.Time
|
||||
releaseTime := time.Unix(0, int64(releaseTimeNs))
|
||||
entry.ReleaseTime = releaseTime
|
||||
entries = append(entries, &entry)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Get retrieves all evolutions for the given selectors from DB.
|
||||
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) (map[string][]*telemetrytypes.EvolutionEntry, error) {
|
||||
|
||||
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
|
||||
for _, keySelector := range keysToUpdate {
|
||||
selector := &telemetrytypes.EvolutionSelector{
|
||||
Signal: keySelector.Signal,
|
||||
FieldContext: keySelector.FieldContext,
|
||||
FieldName: keySelector.Name,
|
||||
}
|
||||
metadataKeySelectors = append(metadataKeySelectors, selector)
|
||||
}
|
||||
|
||||
evolutions, err := k.fetchEvolutionEntryFromClickHouse(ctx, metadataKeySelectors)
|
||||
if err != nil {
|
||||
return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
|
||||
}
|
||||
|
||||
evolutionsByUniqueKey := make(map[string][]*telemetrytypes.EvolutionEntry)
|
||||
for _, evolution := range evolutions {
|
||||
key := &telemetrytypes.EvolutionSelector{
|
||||
Signal: evolution.Signal,
|
||||
FieldContext: evolution.FieldContext,
|
||||
FieldName: evolution.FieldName,
|
||||
}
|
||||
evolutionsByUniqueKey[key.QualifiedName()] = append(evolutionsByUniqueKey[key.QualifiedName()], evolution)
|
||||
}
|
||||
|
||||
if len(keysToUpdate) > 0 {
|
||||
for i, key := range keysToUpdate {
|
||||
selector := &telemetrytypes.EvolutionSelector{
|
||||
Signal: key.Signal,
|
||||
FieldContext: key.FieldContext,
|
||||
FieldName: "__all__",
|
||||
}
|
||||
// first check if there is evolutions that with field name as __all__
|
||||
if keyEvolutions, ok := evolutionsByUniqueKey[selector.QualifiedName()]; ok {
|
||||
keysToUpdate[i].Evolutions = keyEvolutions
|
||||
}
|
||||
// then check for specific field name
|
||||
selector.FieldName = key.Name
|
||||
if keyEvolutions, ok := evolutionsByUniqueKey[selector.QualifiedName()]; ok {
|
||||
keysToUpdate[i].Evolutions = keyEvolutions
|
||||
}
|
||||
}
|
||||
}
|
||||
return evolutionsByUniqueKey, nil
|
||||
}
|
||||
|
||||
// chunkSizeFirstSeenMetricMetadata limits the number of tuples per SQL query to avoid hitting the max_query_size limit.
|
||||
//
|
||||
// Calculation Logic:
|
||||
|
||||
@@ -39,7 +39,6 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataLocalTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
|
||||
lookupKeys := []telemetrytypes.MetricMetadataLookupKey{
|
||||
|
||||
@@ -38,7 +38,6 @@ func newTestTelemetryMetaStoreTestHelper(store telemetrystore.TelemetryStore) te
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataLocalTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ const (
|
||||
DBName = "signoz_metadata"
|
||||
AttributesMetadataTableName = "distributed_attributes_metadata"
|
||||
AttributesMetadataLocalTableName = "attributes_metadata"
|
||||
ColumnEvolutionMetadataTableName = "distributed_column_evolution_metadata"
|
||||
PathTypesTableName = otelcollectorconst.DistributedPathTypesTable
|
||||
// Column Evolution table stores promoted paths as (signal, column_name, field_context, field_name); see signoz-otel-collector metadata_migrations.
|
||||
PromotedPathsTableName = "distributed_column_evolution_metadata"
|
||||
|
||||
@@ -122,7 +122,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
stepSec,
|
||||
))
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -147,16 +147,13 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
)
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -208,7 +205,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
))
|
||||
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -236,16 +233,13 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -284,7 +278,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
stepSec,
|
||||
))
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -306,16 +300,13 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
)
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
@@ -23,8 +23,6 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
@@ -35,7 +33,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -43,52 +41,52 @@ func (c *conditionBuilder) conditionFor(
|
||||
// TODO(srikanthccv): use the same data type collision handling when metrics schemas are updated
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
tblFieldName = fmt.Sprintf("toFloat64OrNull(%s)", tblFieldName)
|
||||
case []any:
|
||||
if len(v) > 0 && (operator == qbtypes.FilterOperatorBetween || operator == qbtypes.FilterOperatorNotBetween) {
|
||||
if _, ok := v[0].(float64); ok {
|
||||
fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
tblFieldName = fmt.Sprintf("toFloat64OrNull(%s)", tblFieldName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.G(fieldExpression, value), nil
|
||||
return sb.G(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThanOrEq:
|
||||
return sb.GE(fieldExpression, value), nil
|
||||
return sb.GE(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorLessThan:
|
||||
return sb.LT(fieldExpression, value), nil
|
||||
return sb.LT(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorLessThanOrEq:
|
||||
return sb.LE(fieldExpression, value), nil
|
||||
return sb.LE(tblFieldName, value), nil
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.Like(fieldExpression, value), nil
|
||||
return sb.Like(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotLike(fieldExpression, value), nil
|
||||
return sb.NotLike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorILike:
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
// between and not between
|
||||
case qbtypes.FilterOperatorBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -98,7 +96,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
return sb.Between(tblFieldName, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -107,7 +105,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
return sb.NotBetween(tblFieldName, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -115,13 +113,13 @@ func (c *conditionBuilder) conditionFor(
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
return sb.In(fieldExpression, values), nil
|
||||
return sb.In(tblFieldName, values), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
return sb.NotIn(fieldExpression, values), nil
|
||||
return sb.NotIn(tblFieldName, values), nil
|
||||
|
||||
// exists and not exists
|
||||
// in the UI based query builder, `exists` and `not exists` are used for
|
||||
@@ -143,14 +141,14 @@ func (c *conditionBuilder) conditionFor(
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
|
||||
condition, err := c.conditionFor(ctx, key, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -234,7 +234,7 @@ func TestConditionFor(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
@@ -289,7 +289,7 @@ func TestConditionForMultipleKeys(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var err error
|
||||
for _, key := range tc.keys {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &key, tc.operator, tc.value, sb, 0, 0)
|
||||
sb.Where(cond)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting condition for key %s: %v", key.Name, err)
|
||||
|
||||
@@ -41,66 +41,65 @@ func NewFieldMapper() qbtypes.FieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(_ context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
|
||||
return []*schema.Column{timeSeriesV4Columns["labels"]}, nil
|
||||
return timeSeriesV4Columns["labels"], nil
|
||||
case telemetrytypes.FieldContextMetric:
|
||||
col, ok := timeSeriesV4Columns[key.Name]
|
||||
if !ok {
|
||||
return []*schema.Column{}, qbtypes.ErrColumnNotFound
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
col, ok := timeSeriesV4Columns[key.Name]
|
||||
if !ok {
|
||||
// if nothing is found, return labels column
|
||||
// as we keep all the labels in the labels column
|
||||
return []*schema.Column{timeSeriesV4Columns["labels"]}, nil
|
||||
return timeSeriesV4Columns["labels"], nil
|
||||
}
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
}
|
||||
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", column.Name, key.Name), nil
|
||||
case telemetrytypes.FieldContextMetric:
|
||||
return columns[0].Name, nil
|
||||
return column.Name, nil
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
if slices.Contains(IntrinsicFields, key.Name) {
|
||||
return columns[0].Name, nil
|
||||
return column.Name, nil
|
||||
}
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", column.Name, key.Name), nil
|
||||
}
|
||||
|
||||
return columns[0].Name, nil
|
||||
return column.Name, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
fieldExpression, err := m.FieldFor(ctx, startNs, endNs, field)
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
}
|
||||
|
||||
@@ -123,13 +123,13 @@ func TestGetColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
col, err := fm.ColumnFor(ctx, 0, 0, &tc.key)
|
||||
col, err := fm.ColumnFor(ctx, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedCol, col[0])
|
||||
assert.Equal(t, tc.expectedCol, col)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -207,7 +207,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
|
||||
@@ -269,16 +269,13 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -289,7 +286,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
|
||||
sb.Select("fingerprint")
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
@@ -29,8 +29,6 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
@@ -42,13 +40,13 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
|
||||
// first, locate the raw column type (so we can choose the right EXISTS logic)
|
||||
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
column, err := c.fm.ColumnFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// then ask the mapper for the actual SQL reference
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -69,48 +67,48 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
|
||||
}
|
||||
|
||||
// regular operators
|
||||
switch operator {
|
||||
// regular operators
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.G(fieldExpression, value), nil
|
||||
return sb.G(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThanOrEq:
|
||||
return sb.GE(fieldExpression, value), nil
|
||||
return sb.GE(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorLessThan:
|
||||
return sb.LT(fieldExpression, value), nil
|
||||
return sb.LT(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorLessThanOrEq:
|
||||
return sb.LE(fieldExpression, value), nil
|
||||
return sb.LE(tblFieldName, value), nil
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.Like(fieldExpression, value), nil
|
||||
return sb.Like(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotLike(fieldExpression, value), nil
|
||||
return sb.NotLike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorILike:
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
// between and not between
|
||||
case qbtypes.FilterOperatorBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -120,7 +118,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
return sb.Between(tblFieldName, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -129,7 +127,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
return sb.NotBetween(tblFieldName, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -140,7 +138,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
conditions = append(conditions, sb.E(tblFieldName, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -151,7 +149,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
conditions = append(conditions, sb.NE(tblFieldName, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
@@ -161,30 +159,30 @@ func (c *conditionBuilder) conditionFor(
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
|
||||
var value any
|
||||
switch columns[0].Type.GetType() {
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.IsNotNull(fieldExpression), nil
|
||||
return sb.IsNotNull(tblFieldName), nil
|
||||
} else {
|
||||
return sb.IsNull(fieldExpression), nil
|
||||
return sb.IsNull(tblFieldName), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumFixedString,
|
||||
schema.ColumnTypeEnumDateTime64:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
} else {
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := columns[0].Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
}
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
@@ -197,19 +195,19 @@ func (c *conditionBuilder) conditionFor(
|
||||
schema.ColumnTypeEnumBool:
|
||||
value = 0
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
} else {
|
||||
return sb.E(fieldExpression, value), nil
|
||||
return sb.E(tblFieldName, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := columns[0].Type.(schema.MapColumnType).KeyType
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, columns[0].Type)
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
|
||||
}
|
||||
|
||||
switch valueType := columns[0].Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
if key.Materialized {
|
||||
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
}
|
||||
@@ -222,7 +220,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", columns[0].Type)
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
@@ -230,25 +228,25 @@ func (c *conditionBuilder) conditionFor(
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
startNs uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
if c.isSpanScopeField(key.Name) {
|
||||
return c.buildSpanScopeCondition(key, operator, value, startNs)
|
||||
}
|
||||
|
||||
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
|
||||
condition, err := c.conditionFor(ctx, key, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if operator.AddDefaultExistsFilter() {
|
||||
// skip adding exists filter for intrinsic fields
|
||||
field, _ := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
field, _ := c.fm.FieldFor(ctx, key)
|
||||
if slices.Contains(maps.Keys(IntrinsicFields), field) ||
|
||||
slices.Contains(maps.Keys(IntrinsicFieldsDeprecated), field) ||
|
||||
slices.Contains(maps.Keys(CalculatedFields), field) ||
|
||||
@@ -256,7 +254,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
existsCondition, err := c.conditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
existsCondition, err := c.conditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -289,7 +289,7 @@ func TestConditionFor(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 1761437108000000000, 1761458708000000000, &tc.key, tc.operator, tc.value, sb)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 1761437108000000000, 1761458708000000000)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -169,24 +169,23 @@ func NewFieldMapper() *defaultFieldMapper {
|
||||
|
||||
func (m *defaultFieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) ([]*schema.Column, error) {
|
||||
) (*schema.Column, error) {
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return []*schema.Column{indexV3Columns["resource"]}, nil
|
||||
return indexV3Columns["resource"], nil
|
||||
case telemetrytypes.FieldContextScope:
|
||||
return []*schema.Column{}, qbtypes.ErrColumnNotFound
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
switch key.FieldDataType {
|
||||
case telemetrytypes.FieldDataTypeString:
|
||||
return []*schema.Column{indexV3Columns["attributes_string"]}, nil
|
||||
return indexV3Columns["attributes_string"], nil
|
||||
case telemetrytypes.FieldDataTypeInt64,
|
||||
telemetrytypes.FieldDataTypeFloat64,
|
||||
telemetrytypes.FieldDataTypeNumber:
|
||||
return []*schema.Column{indexV3Columns["attributes_number"]}, nil
|
||||
return indexV3Columns["attributes_number"], nil
|
||||
case telemetrytypes.FieldDataTypeBool:
|
||||
return []*schema.Column{indexV3Columns["attributes_bool"]}, nil
|
||||
return indexV3Columns["attributes_bool"], nil
|
||||
}
|
||||
case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextUnspecified:
|
||||
/*
|
||||
@@ -197,7 +196,7 @@ func (m *defaultFieldMapper) getColumn(
|
||||
// Check if this is a span scope field
|
||||
if strings.ToLower(key.Name) == SpanSearchScopeRoot || strings.ToLower(key.Name) == SpanSearchScopeEntryPoint {
|
||||
// The actual SQL will be generated in the condition builder
|
||||
return []*schema.Column{{Name: key.Name, Type: schema.ColumnTypeBool}}, nil
|
||||
return &schema.Column{Name: key.Name, Type: schema.ColumnTypeBool}, nil
|
||||
}
|
||||
|
||||
// TODO(srikanthccv): remove this when it's safe to remove
|
||||
@@ -211,18 +210,18 @@ func (m *defaultFieldMapper) getColumn(
|
||||
if _, ok := CalculatedFieldsDeprecated[key.Name]; ok {
|
||||
// Check if we have a mapping for the deprecated calculated field
|
||||
if col, ok := indexV3Columns[oldToNew[key.Name]]; ok {
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
}
|
||||
}
|
||||
if _, ok := IntrinsicFieldsDeprecated[key.Name]; ok {
|
||||
// Check if we have a mapping for the deprecated intrinsic field
|
||||
if col, ok := indexV3Columns[oldToNew[key.Name]]; ok {
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
}
|
||||
}
|
||||
|
||||
if col, ok := indexV3Columns[key.Name]; ok {
|
||||
return []*schema.Column{col}, nil
|
||||
return col, nil
|
||||
}
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
@@ -230,17 +229,15 @@ func (m *defaultFieldMapper) getColumn(
|
||||
|
||||
func (m *defaultFieldMapper) ColumnFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, startNs, endNs, key)
|
||||
) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
}
|
||||
|
||||
// FieldFor returns the table field name for the given key if it exists
|
||||
// otherwise it returns qbtypes.ErrColumnNotFound
|
||||
func (m *defaultFieldMapper) FieldFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
// Special handling for span scope fields
|
||||
@@ -250,14 +247,10 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
return key.Name, nil
|
||||
}
|
||||
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
column, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(columns) != 1 {
|
||||
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
|
||||
}
|
||||
column := columns[0]
|
||||
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
@@ -317,12 +310,11 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
// if it exists otherwise it returns qbtypes.ErrColumnNotFound
|
||||
func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
fieldExpression, err := m.FieldFor(ctx, startNs, endNs, field)
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -332,7 +324,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
if _, ok := indexV3Columns[field.Name]; ok {
|
||||
// if it is, attach the column name directly
|
||||
field.FieldContext = telemetrytypes.FieldContextSpan
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, field)
|
||||
colName, _ = m.FieldFor(ctx, field)
|
||||
} else {
|
||||
// - the context is not provided
|
||||
// - there are not keys for the field
|
||||
@@ -350,17 +342,17 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
} else if len(keysForField) == 1 {
|
||||
// we have a single key for the field, use it
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, keysForField[0])
|
||||
colName, _ = m.FieldFor(ctx, keysForField[0])
|
||||
} else {
|
||||
// select any non-empty value from the keys
|
||||
args := []string{}
|
||||
for _, key := range keysForField {
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression))
|
||||
colName, _ = m.FieldFor(ctx, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", colName, colName))
|
||||
}
|
||||
fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
colName = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package telemetrytraces
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
@@ -76,16 +75,13 @@ func TestSpanScopeFilterExpression(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
}}
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
whereClause, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
FieldKeys: fieldKeys,
|
||||
Builder: sb,
|
||||
StartNs: tt.startNs,
|
||||
EndNs: 1761458708000000000,
|
||||
})
|
||||
}, tt.startNs, 1761458708000000000)
|
||||
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
@@ -146,16 +142,13 @@ func TestSpanScopeWithResourceFilter(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}}
|
||||
|
||||
_, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
_, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
FieldKeys: fieldKeys,
|
||||
SkipResourceFilter: false, // This would be set by the statement builder
|
||||
StartNs: 1761437108000000000,
|
||||
EndNs: 1761458708000000000,
|
||||
})
|
||||
}, 1761437108000000000, 1761458708000000000)
|
||||
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
@@ -296,7 +296,7 @@ func (b *traceQueryStatementBuilder) buildListQuery(
|
||||
|
||||
// TODO: should we deprecate `SelectFields` and return everything from a span like we do for logs?
|
||||
for _, field := range query.SelectFields {
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &field, keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &field, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -314,7 +314,7 @@ func (b *traceQueryStatementBuilder) buildListQuery(
|
||||
|
||||
// Add order by
|
||||
for _, orderBy := range query.Order {
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -498,7 +498,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
// Keep original column expressions so we can build the tuple
|
||||
fieldNames := make([]string, 0, len(query.GroupBy))
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -512,7 +512,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
allAggChArgs := make([]any, 0)
|
||||
for i, agg := range query.Aggregations {
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, start, end, agg.Expression,
|
||||
ctx, agg.Expression,
|
||||
uint64(query.StepInterval.Seconds()),
|
||||
keys,
|
||||
)
|
||||
@@ -640,7 +640,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
|
||||
var allGroupByArgs []any
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -657,7 +657,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
for idx := range query.Aggregations {
|
||||
aggExpr := query.Aggregations[idx]
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, start, end, aggExpr.Expression,
|
||||
ctx, aggExpr.Expression,
|
||||
rateInterval,
|
||||
keys,
|
||||
)
|
||||
@@ -729,7 +729,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
|
||||
// buildFilterCondition builds SQL condition from filter expression
|
||||
func (b *traceQueryStatementBuilder) addFilterCondition(
|
||||
ctx context.Context,
|
||||
_ context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
@@ -743,16 +743,13 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
// add filter expression
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
SkipResourceFilter: true,
|
||||
Variables: variables,
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
}, start, end)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -234,15 +234,12 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
|
||||
filterWhereClause, err := querybuilder.PrepareWhereClause(
|
||||
query.Filter.Expression,
|
||||
querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.stmtBuilder.logger,
|
||||
FieldMapper: b.stmtBuilder.fm,
|
||||
ConditionBuilder: b.stmtBuilder.cb,
|
||||
FieldKeys: keys,
|
||||
SkipResourceFilter: true,
|
||||
StartNs: b.start,
|
||||
EndNs: b.end,
|
||||
},
|
||||
}, b.start, b.end,
|
||||
)
|
||||
if err != nil {
|
||||
b.stmtBuilder.logger.ErrorContext(ctx, "Failed to prepare where clause", errors.Attr(err), slog.String("filter", query.Filter.Expression))
|
||||
@@ -455,7 +452,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
|
||||
if selectedFields[field.Name] {
|
||||
continue
|
||||
}
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.start, b.end, &field, keys)
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, &field, keys)
|
||||
if err != nil {
|
||||
b.stmtBuilder.logger.WarnContext(ctx, "failed to map select field",
|
||||
slog.String("field", field.Name), errors.Attr(err))
|
||||
@@ -470,7 +467,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
|
||||
// Add order by support using ColumnExpressionFor
|
||||
orderApplied := false
|
||||
for _, orderBy := range b.operator.Order {
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.start, b.end, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -552,8 +549,6 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
|
||||
for _, gb := range b.operator.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
&gb.TelemetryFieldKey,
|
||||
b.stmtBuilder.fm,
|
||||
b.stmtBuilder.cb,
|
||||
@@ -578,8 +573,6 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
|
||||
for i, agg := range b.operator.Aggregations {
|
||||
rewritten, chArgs, err := b.stmtBuilder.aggExprRewriter.Rewrite(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
agg.Expression,
|
||||
uint64(b.operator.StepInterval.Seconds()),
|
||||
keys,
|
||||
@@ -665,8 +658,6 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
|
||||
for _, gb := range b.operator.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
&gb.TelemetryFieldKey,
|
||||
b.stmtBuilder.fm,
|
||||
b.stmtBuilder.cb,
|
||||
@@ -693,8 +684,6 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
|
||||
for i, agg := range b.operator.Aggregations {
|
||||
rewritten, chArgs, err := b.stmtBuilder.aggExprRewriter.Rewrite(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
agg.Expression,
|
||||
rateInterval,
|
||||
keys,
|
||||
@@ -808,8 +797,6 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
|
||||
for _, gb := range b.operator.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
&gb.TelemetryFieldKey,
|
||||
b.stmtBuilder.fm,
|
||||
b.stmtBuilder.cb,
|
||||
@@ -834,8 +821,6 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
|
||||
for i, agg := range b.operator.Aggregations {
|
||||
rewritten, chArgs, err := b.stmtBuilder.aggExprRewriter.Rewrite(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
agg.Expression,
|
||||
uint64((b.end-b.start)/querybuilder.NsToSeconds),
|
||||
keys,
|
||||
|
||||
@@ -2,13 +2,12 @@ package telemetrytraces
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type traceOperatorStatementBuilder struct {
|
||||
|
||||
@@ -22,23 +22,24 @@ type JsonKeyToFieldFunc func(context.Context, *telemetrytypes.TelemetryFieldKey,
|
||||
// FieldMapper maps the telemetry field key to the table field name.
|
||||
type FieldMapper interface {
|
||||
// FieldFor returns the field name for the given key.
|
||||
FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error)
|
||||
FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error)
|
||||
// ColumnFor returns the column for the given key.
|
||||
ColumnFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error)
|
||||
ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error)
|
||||
// ColumnExpressionFor returns the column expression for the given key.
|
||||
ColumnExpressionFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, error)
|
||||
ColumnExpressionFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, error)
|
||||
}
|
||||
|
||||
// ConditionBuilder builds the condition for the filter.
|
||||
type ConditionBuilder interface {
|
||||
// ConditionFor returns the condition for the given key, operator and value.
|
||||
ConditionFor(ctx context.Context, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error)
|
||||
// TODO(srikanthccv,nikhilmantri0902): remove startNs, endNs when top_level_operations can be replaced with `is_remote`
|
||||
ConditionFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder, startNs uint64, endNs uint64) (string, error)
|
||||
}
|
||||
|
||||
type AggExprRewriter interface {
|
||||
// Rewrite rewrites the aggregation expression to be used in the query.
|
||||
Rewrite(ctx context.Context, startNs, endNs uint64, expr string, rateInterval uint64, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, []any, error)
|
||||
RewriteMulti(ctx context.Context, startNs, endNs uint64, exprs []string, rateInterval uint64, keys map[string][]*telemetrytypes.TelemetryFieldKey) ([]string, [][]any, error)
|
||||
Rewrite(ctx context.Context, expr string, rateInterval uint64, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, []any, error)
|
||||
RewriteMulti(ctx context.Context, exprs []string, rateInterval uint64, keys map[string][]*telemetrytypes.TelemetryFieldKey) ([]string, [][]any, error)
|
||||
}
|
||||
|
||||
type Statement struct {
|
||||
|
||||
@@ -206,11 +206,8 @@ func (q *QueryBuilderQuery[T]) validateAggregations(cfg validationConfig) error
|
||||
return nil
|
||||
}
|
||||
|
||||
// At least one aggregation required for aggregation queries, even if
|
||||
// they are disabled, usually because they are used in formula
|
||||
// regardless of use in formula, it's invalid to have empty Aggregations
|
||||
// for aggregation request
|
||||
if len(q.Aggregations) == 0 {
|
||||
// At least one aggregation required for non-disabled queries
|
||||
if len(q.Aggregations) == 0 && !q.Disabled {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"at least one aggregation is required",
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
@@ -32,14 +31,7 @@ func TestQueryRangeRequest_ValidateAllQueriesNotDisabled(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[MetricAggregation]{
|
||||
Name: "A",
|
||||
Disabled: true,
|
||||
Aggregations: []MetricAggregation{
|
||||
{
|
||||
MetricName: "test",
|
||||
TimeAggregation: metrictypes.TimeAggregationAvg,
|
||||
SpaceAggregation: metrictypes.SpaceAggregationMax,
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -47,12 +39,7 @@ func TestQueryRangeRequest_ValidateAllQueriesNotDisabled(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[LogAggregation]{
|
||||
Name: "B",
|
||||
Disabled: true,
|
||||
Aggregations: []LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -74,14 +61,7 @@ func TestQueryRangeRequest_ValidateAllQueriesNotDisabled(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[MetricAggregation]{
|
||||
Name: "A",
|
||||
Disabled: true,
|
||||
Aggregations: []MetricAggregation{
|
||||
{
|
||||
MetricName: "test",
|
||||
TimeAggregation: metrictypes.TimeAggregationAvg,
|
||||
SpaceAggregation: metrictypes.SpaceAggregationMax,
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -214,14 +194,7 @@ func TestQueryRangeRequest_ValidateAllQueriesNotDisabled(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[MetricAggregation]{
|
||||
Name: "A",
|
||||
Disabled: true,
|
||||
Aggregations: []MetricAggregation{
|
||||
{
|
||||
MetricName: "test",
|
||||
TimeAggregation: metrictypes.TimeAggregationAvg,
|
||||
SpaceAggregation: metrictypes.SpaceAggregationMax,
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -259,12 +232,7 @@ func TestQueryRangeRequest_ValidateAllQueriesNotDisabled(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[LogAggregation]{
|
||||
Name: "A",
|
||||
Disabled: true,
|
||||
Aggregations: []LogAggregation{
|
||||
{
|
||||
Expression: "sum(duration)",
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -398,12 +366,7 @@ func TestQueryRangeRequest_ValidateCompositeQuery(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[LogAggregation]{
|
||||
Name: "A",
|
||||
Disabled: true,
|
||||
Aggregations: []LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -411,12 +374,7 @@ func TestQueryRangeRequest_ValidateCompositeQuery(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[TraceAggregation]{
|
||||
Name: "A",
|
||||
Disabled: true,
|
||||
Aggregations: []TraceAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -438,12 +396,7 @@ func TestQueryRangeRequest_ValidateCompositeQuery(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[LogAggregation]{
|
||||
Name: "X",
|
||||
Disabled: true,
|
||||
Aggregations: []LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -451,14 +404,7 @@ func TestQueryRangeRequest_ValidateCompositeQuery(t *testing.T) {
|
||||
Spec: QueryBuilderQuery[MetricAggregation]{
|
||||
Name: "X",
|
||||
Disabled: true,
|
||||
Aggregations: []MetricAggregation{
|
||||
{
|
||||
MetricName: "test",
|
||||
TimeAggregation: metrictypes.TimeAggregationAvg,
|
||||
SpaceAggregation: metrictypes.SpaceAggregationMax,
|
||||
},
|
||||
},
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -481,9 +427,7 @@ func TestQueryRangeRequest_ValidateCompositeQuery(t *testing.T) {
|
||||
Name: "A",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Aggregations: []LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
},
|
||||
{Expression: "count()"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -637,9 +581,7 @@ func TestQueryRangeRequest_ValidateCompositeQuery(t *testing.T) {
|
||||
Name: "A",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Aggregations: []LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
},
|
||||
{Expression: "count()"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package telemetrytypes
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type EvolutionEntry struct {
|
||||
Signal Signal `json:"signal"`
|
||||
ColumnName string `json:"column_name"`
|
||||
ColumnType string `json:"column_type"`
|
||||
FieldContext FieldContext `json:"field_context"`
|
||||
FieldName string `json:"field_name"`
|
||||
ReleaseTime time.Time `json:"release_time"`
|
||||
Version uint32 `json:"version"`
|
||||
}
|
||||
|
||||
type EvolutionSelector struct {
|
||||
Signal Signal
|
||||
FieldContext FieldContext
|
||||
FieldName string
|
||||
}
|
||||
|
||||
func (e *EvolutionSelector) QualifiedName() string {
|
||||
return e.Signal.StringValue() + ":" + e.FieldContext.StringValue() + ":" + e.FieldName
|
||||
}
|
||||
@@ -41,8 +41,6 @@ type TelemetryFieldKey struct {
|
||||
JSONPlan JSONAccessPlan `json:"-"`
|
||||
Indexes []JSONDataTypeIndex `json:"-"`
|
||||
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
|
||||
|
||||
Evolutions []*EvolutionEntry `json:"-"`
|
||||
}
|
||||
|
||||
func (f *TelemetryFieldKey) KeyNameContainsArray() bool {
|
||||
@@ -121,7 +119,6 @@ func (f *TelemetryFieldKey) OverrideMetadataFrom(src *TelemetryFieldKey) {
|
||||
f.Indexes = src.Indexes
|
||||
f.Materialized = src.Materialized
|
||||
f.JSONPlan = src.JSONPlan
|
||||
f.Evolutions = src.Evolutions
|
||||
}
|
||||
|
||||
func (f *TelemetryFieldKey) Equal(key *TelemetryFieldKey) bool {
|
||||
|
||||
@@ -12,15 +12,14 @@ import (
|
||||
// MockMetadataStore implements the MetadataStore interface for testing purposes
|
||||
type MockMetadataStore struct {
|
||||
// Maps to store test data
|
||||
KeysMap map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
RelatedValuesMap map[string][]string
|
||||
AllValuesMap map[string]*telemetrytypes.TelemetryFieldValues
|
||||
TemporalityMap map[string]metrictypes.Temporality
|
||||
TypeMap map[string]metrictypes.Type
|
||||
PromotedPathsMap map[string]bool
|
||||
LogsJSONIndexesMap map[string][]schemamigrator.Index
|
||||
ColumnEvolutionMetadataMap map[string][]*telemetrytypes.EvolutionEntry
|
||||
LookupKeysMap map[telemetrytypes.MetricMetadataLookupKey]int64
|
||||
KeysMap map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
RelatedValuesMap map[string][]string
|
||||
AllValuesMap map[string]*telemetrytypes.TelemetryFieldValues
|
||||
TemporalityMap map[string]metrictypes.Temporality
|
||||
TypeMap map[string]metrictypes.Type
|
||||
PromotedPathsMap map[string]bool
|
||||
LogsJSONIndexesMap map[string][]schemamigrator.Index
|
||||
LookupKeysMap map[telemetrytypes.MetricMetadataLookupKey]int64
|
||||
// StaticFields holds signal-specific intrinsic field definitions (e.g. telemetrylogs.IntrinsicFields).
|
||||
StaticFields map[string]telemetrytypes.TelemetryFieldKey
|
||||
}
|
||||
@@ -28,16 +27,15 @@ type MockMetadataStore struct {
|
||||
// NewMockMetadataStore creates a new instance of MockMetadataStore with initialized maps.
|
||||
func NewMockMetadataStore() *MockMetadataStore {
|
||||
return &MockMetadataStore{
|
||||
KeysMap: make(map[string][]*telemetrytypes.TelemetryFieldKey),
|
||||
RelatedValuesMap: make(map[string][]string),
|
||||
AllValuesMap: make(map[string]*telemetrytypes.TelemetryFieldValues),
|
||||
TemporalityMap: make(map[string]metrictypes.Temporality),
|
||||
TypeMap: make(map[string]metrictypes.Type),
|
||||
PromotedPathsMap: make(map[string]bool),
|
||||
LogsJSONIndexesMap: make(map[string][]schemamigrator.Index),
|
||||
ColumnEvolutionMetadataMap: make(map[string][]*telemetrytypes.EvolutionEntry),
|
||||
LookupKeysMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
StaticFields: make(map[string]telemetrytypes.TelemetryFieldKey),
|
||||
KeysMap: make(map[string][]*telemetrytypes.TelemetryFieldKey),
|
||||
RelatedValuesMap: make(map[string][]string),
|
||||
AllValuesMap: make(map[string]*telemetrytypes.TelemetryFieldValues),
|
||||
TemporalityMap: make(map[string]metrictypes.Temporality),
|
||||
TypeMap: make(map[string]metrictypes.Type),
|
||||
PromotedPathsMap: make(map[string]bool),
|
||||
LogsJSONIndexesMap: make(map[string][]schemamigrator.Index),
|
||||
LookupKeysMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
StaticFields: make(map[string]telemetrytypes.TelemetryFieldKey),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,11 +119,6 @@ func (m *MockMetadataStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
|
||||
}
|
||||
}
|
||||
|
||||
// fetch and add evolutions
|
||||
for _, v := range result {
|
||||
m.updateColumnEvolutionMetadataForKeys(ctx, v)
|
||||
}
|
||||
|
||||
return result, true, nil
|
||||
}
|
||||
|
||||
@@ -373,37 +366,6 @@ func (m *MockMetadataStore) ListLogsJSONIndexes(ctx context.Context, filters ...
|
||||
return m.LogsJSONIndexesMap, nil
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) updateColumnEvolutionMetadataForKeys(_ context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) map[string][]*telemetrytypes.EvolutionEntry {
|
||||
|
||||
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
|
||||
for _, keySelector := range keysToUpdate {
|
||||
selector := &telemetrytypes.EvolutionSelector{
|
||||
Signal: keySelector.Signal,
|
||||
FieldContext: keySelector.FieldContext,
|
||||
FieldName: keySelector.Name,
|
||||
}
|
||||
metadataKeySelectors = append(metadataKeySelectors, selector)
|
||||
}
|
||||
result := make(map[string][]*telemetrytypes.EvolutionEntry)
|
||||
for i, selector := range metadataKeySelectors {
|
||||
sel := &telemetrytypes.EvolutionSelector{
|
||||
Signal: selector.Signal,
|
||||
FieldContext: selector.FieldContext,
|
||||
FieldName: "__all__",
|
||||
}
|
||||
key := sel.QualifiedName()
|
||||
if entries, exists := m.ColumnEvolutionMetadataMap[key]; exists {
|
||||
result[key] = entries
|
||||
}
|
||||
sel.FieldName = metadataKeySelectors[i].FieldName
|
||||
key = sel.QualifiedName()
|
||||
if entries, exists := m.ColumnEvolutionMetadataMap[key]; exists {
|
||||
result[key] = entries
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) GetFirstSeenFromMetricMetadata(ctx context.Context, lookupKeys []telemetrytypes.MetricMetadataLookupKey) (map[telemetrytypes.MetricMetadataLookupKey]int64, error) {
|
||||
return m.LookupKeysMap, nil
|
||||
}
|
||||
|
||||
@@ -71,6 +71,6 @@ def pytest_addoption(parser: pytest.Parser):
|
||||
parser.addoption(
|
||||
"--schema-migrator-version",
|
||||
action="store",
|
||||
default="v0.144.2",
|
||||
default="v0.129.7",
|
||||
help="schema migrator version",
|
||||
)
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
import datetime
|
||||
import json
|
||||
from abc import ABC
|
||||
from http import HTTPStatus
|
||||
from typing import Any, Callable, Generator, List, Literal, Optional
|
||||
from typing import Any, Callable, Generator, List, Optional
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
from ksuid import KsuidMs
|
||||
|
||||
from fixtures import types
|
||||
@@ -106,7 +104,6 @@ class Logs(ABC):
|
||||
attributes_number: dict[str, np.float64]
|
||||
attributes_bool: dict[str, bool]
|
||||
resources_string: dict[str, str]
|
||||
resource_json: dict[str, str]
|
||||
scope_name: str
|
||||
scope_version: str
|
||||
scope_string: dict[str, str]
|
||||
@@ -129,7 +126,6 @@ class Logs(ABC):
|
||||
scope_name: str = "",
|
||||
scope_version: str = "",
|
||||
scope_attributes: dict[str, str] = {},
|
||||
resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
|
||||
) -> None:
|
||||
if timestamp is None:
|
||||
timestamp = datetime.datetime.now()
|
||||
@@ -169,9 +165,6 @@ class Logs(ABC):
|
||||
|
||||
# Process resources and attributes
|
||||
self.resources_string = {k: str(v) for k, v in resources.items()}
|
||||
self.resource_json = (
|
||||
{} if resource_write_mode == "legacy_only" else dict(self.resources_string)
|
||||
)
|
||||
for k, v in self.resources_string.items():
|
||||
self.tag_attributes.append(
|
||||
LogsTagAttributes(
|
||||
@@ -333,7 +326,7 @@ class Logs(ABC):
|
||||
self.scope_name,
|
||||
self.scope_version,
|
||||
self.scope_string,
|
||||
self.resource_json,
|
||||
self.resources_string,
|
||||
]
|
||||
)
|
||||
|
||||
@@ -500,54 +493,6 @@ def insert_logs(
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="materialize_log_field", scope="function")
|
||||
def materialize_log_field(
|
||||
signoz: types.SigNoz,
|
||||
) -> Generator[Callable[[str, str, str, str], None], None, None]:
|
||||
mat_fields: List[tuple[str, str, str]] = []
|
||||
|
||||
def _materialize_log_field(
|
||||
token: str,
|
||||
name: str,
|
||||
data_type: str,
|
||||
field_type: str,
|
||||
) -> None:
|
||||
response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/logs/fields"),
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"name": name,
|
||||
"dataType": data_type,
|
||||
"type": field_type,
|
||||
"selected": True,
|
||||
},
|
||||
timeout=10,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, (
|
||||
f"Failed to materialize log field {name}: "
|
||||
f"{response.status_code} {response.text}"
|
||||
)
|
||||
mat_fields.append((field_type, data_type, name))
|
||||
|
||||
yield _materialize_log_field
|
||||
|
||||
for mat_field_type, mat_field_data_type, mat_field_name in mat_fields:
|
||||
mat_field_name = mat_field_name.replace(".", "$$")
|
||||
if mat_field_type == "resources":
|
||||
mat_field_type = "resource"
|
||||
field = f"{mat_field_type}_{mat_field_data_type}_{mat_field_name}"
|
||||
signoz.telemetrystore.conn.query(
|
||||
f"ALTER TABLE signoz_logs.logs_v2 ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' DROP INDEX IF EXISTS {field}_idx"
|
||||
)
|
||||
for table in ["logs_v2", "distributed_logs_v2"]:
|
||||
signoz.telemetrystore.conn.query(
|
||||
f"ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' DROP COLUMN IF EXISTS {field}"
|
||||
)
|
||||
signoz.telemetrystore.conn.query(
|
||||
f"ALTER TABLE signoz_logs.{table} ON CLUSTER '{signoz.telemetrystore.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER']}' DROP COLUMN IF EXISTS {field}_exists"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="ttl_legacy_logs_v2_table_setup", scope="function")
|
||||
def ttl_legacy_logs_v2_table_setup(request, signoz: types.SigNoz):
|
||||
"""
|
||||
|
||||
@@ -1,260 +0,0 @@
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http import HTTPStatus
|
||||
from typing import Callable, Dict, List
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import (
|
||||
build_group_by_field,
|
||||
build_logs_aggregation,
|
||||
index_series_by_label,
|
||||
make_query_request,
|
||||
)
|
||||
|
||||
|
||||
# we already create the evolution for resource during schema migration
|
||||
# since we have to create test data around it, we need to get the evolution time
|
||||
def _get_logs_resource_evolution_time_json(signoz: types.SigNoz) -> datetime:
|
||||
result = signoz.telemetrystore.conn.query(
|
||||
"""
|
||||
SELECT release_time
|
||||
FROM signoz_metadata.distributed_column_evolution_metadata
|
||||
WHERE signal = 'logs'
|
||||
AND field_context = 'resource'
|
||||
AND field_name = '__all__'
|
||||
AND column_name = 'resource'
|
||||
LIMIT 1
|
||||
"""
|
||||
).result_rows
|
||||
|
||||
assert result, "Expected logs resource evolution metadata to exist"
|
||||
|
||||
release_time_ns = int(result[0][0])
|
||||
return datetime.fromtimestamp(release_time_ns / 1e9, tz=timezone.utc)
|
||||
|
||||
|
||||
# Logs with timestamps before the evolution time will have resources written only to resources_string.
|
||||
# Logs with timestamps at or after the evolution time will have resources written to both resources_string and resource_json.
|
||||
def _build_evolved_log(
|
||||
timestamp: datetime,
|
||||
evolution_time: datetime,
|
||||
service_name: str,
|
||||
body: str,
|
||||
) -> Logs:
|
||||
resource_write_mode = "legacy_only" if timestamp < evolution_time else "dual_write"
|
||||
return Logs(
|
||||
timestamp=timestamp,
|
||||
resources={
|
||||
"service.name": service_name,
|
||||
"deployment.environment": "integration",
|
||||
},
|
||||
body=body,
|
||||
severity_text="INFO",
|
||||
resource_write_mode=resource_write_mode,
|
||||
)
|
||||
|
||||
|
||||
def _query_grouped_log_series(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
start: datetime,
|
||||
end: datetime,
|
||||
group_by: str = "service.name",
|
||||
aggregation: str = "count()",
|
||||
) -> Dict[str, List[Dict]]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int(start.timestamp() * 1000),
|
||||
end_ms=int(end.timestamp() * 1000),
|
||||
request_type="time_series",
|
||||
queries=[
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"name": "A",
|
||||
"signal": "logs",
|
||||
"stepInterval": 60,
|
||||
"disabled": False,
|
||||
"groupBy": [build_group_by_field(group_by)],
|
||||
"having": {"expression": ""},
|
||||
"aggregations": [build_logs_aggregation(aggregation)],
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["status"] == "success"
|
||||
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
|
||||
aggregations = results[0]["aggregations"]
|
||||
assert len(aggregations) == 1
|
||||
|
||||
return index_series_by_label(aggregations[0]["series"], group_by)
|
||||
|
||||
|
||||
def _assert_grouped_series(
|
||||
series_by_group: Dict[str, Dict],
|
||||
expected_values_by_group: Dict[str, Dict[int, int]],
|
||||
) -> None:
|
||||
assert set(series_by_group.keys()) == set(expected_values_by_group.keys())
|
||||
|
||||
for group_name, expected_by_ts in expected_values_by_group.items():
|
||||
actual_values = sorted(
|
||||
series_by_group[group_name]["values"],
|
||||
key=lambda value: value["timestamp"],
|
||||
)
|
||||
expected_values = [
|
||||
{"timestamp": timestamp, "value": value}
|
||||
for timestamp, value in sorted(expected_by_ts.items())
|
||||
]
|
||||
assert actual_values == expected_values
|
||||
|
||||
|
||||
def _test_logs_resource_evolution(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
# 1. Get the evolution time.
|
||||
# 2. Ingest logs before the evolution time.
|
||||
# 3. Ingest logs after the evolution time.
|
||||
# 4. Query the logs before the evolution time.
|
||||
# 5. Query the logs after the evolution time.
|
||||
# Both aggregation and group by should be checked.
|
||||
"""
|
||||
evolution_time = _get_logs_resource_evolution_time_json(signoz)
|
||||
evolution_time = evolution_time.replace(second=0, microsecond=0)
|
||||
|
||||
before_2 = evolution_time - timedelta(minutes=10)
|
||||
before_1 = evolution_time - timedelta(minutes=5)
|
||||
after_1 = evolution_time + timedelta(minutes=5)
|
||||
after_2 = evolution_time + timedelta(minutes=10)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
_build_evolved_log(
|
||||
timestamp=before_2,
|
||||
evolution_time=evolution_time,
|
||||
service_name="svc-before-2",
|
||||
body="log before evolution 2",
|
||||
),
|
||||
_build_evolved_log(
|
||||
timestamp=before_1,
|
||||
evolution_time=evolution_time,
|
||||
service_name="svc-before-1",
|
||||
body="log before evolution 1",
|
||||
),
|
||||
_build_evolved_log(
|
||||
timestamp=after_1,
|
||||
evolution_time=evolution_time,
|
||||
service_name="svc-after-1",
|
||||
body="log after evolution 1",
|
||||
),
|
||||
_build_evolved_log(
|
||||
timestamp=after_2,
|
||||
evolution_time=evolution_time,
|
||||
service_name="svc-after-2",
|
||||
body="log after evolution 2",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
before_series = _query_grouped_log_series(
|
||||
signoz, token, before_2 - timedelta(minutes=1), before_1 + timedelta(minutes=1)
|
||||
)
|
||||
_assert_grouped_series(
|
||||
before_series,
|
||||
expected_values_by_group={
|
||||
"svc-before-2": {
|
||||
int(before_2.timestamp() * 1000): 1,
|
||||
},
|
||||
"svc-before-1": {
|
||||
int(before_1.timestamp() * 1000): 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
after_series = _query_grouped_log_series(
|
||||
signoz, token, after_1 - timedelta(minutes=1), after_2 + timedelta(minutes=1)
|
||||
)
|
||||
_assert_grouped_series(
|
||||
after_series,
|
||||
expected_values_by_group={
|
||||
"svc-after-1": {
|
||||
int(after_1.timestamp() * 1000): 1,
|
||||
},
|
||||
"svc-after-2": {
|
||||
int(after_2.timestamp() * 1000): 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
spanning_series = _query_grouped_log_series(
|
||||
signoz, token, before_2, after_2 + timedelta(minutes=1)
|
||||
)
|
||||
_assert_grouped_series(
|
||||
spanning_series,
|
||||
expected_values_by_group={
|
||||
"svc-before-2": {
|
||||
int(before_2.timestamp() * 1000): 1,
|
||||
},
|
||||
"svc-before-1": {
|
||||
int(before_1.timestamp() * 1000): 1,
|
||||
},
|
||||
"svc-after-1": {
|
||||
int(after_1.timestamp() * 1000): 1,
|
||||
},
|
||||
"svc-after-2": {
|
||||
int(after_2.timestamp() * 1000): 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
# query to check aggregation on the resource field like count_distinct(service.name)
|
||||
aggregation_series = _query_grouped_log_series(
|
||||
signoz,
|
||||
token,
|
||||
before_2,
|
||||
after_2 + timedelta(minutes=1),
|
||||
group_by="deployment.environment",
|
||||
aggregation="count_distinct(service.name)",
|
||||
)
|
||||
_assert_grouped_series(
|
||||
aggregation_series,
|
||||
expected_values_by_group={
|
||||
"integration": {
|
||||
int(before_2.timestamp() * 1000): 1,
|
||||
int(before_1.timestamp() * 1000): 1,
|
||||
int(after_1.timestamp() * 1000): 1,
|
||||
int(after_2.timestamp() * 1000): 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_logs_resource_evolution(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
_test_logs_resource_evolution(signoz, token, insert_logs)
|
||||
|
||||
|
||||
def test_logs_materialized_resource_evolution(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
materialize_log_field: Callable[[str, str, str, str], None],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
materialize_log_field(token, "service.name", "string", "resources")
|
||||
_test_logs_resource_evolution(signoz, token, insert_logs)
|
||||
Reference in New Issue
Block a user