mirror of
https://github.com/SigNoz/signoz.git
synced 2026-07-29 17:20:40 +01:00
Compare commits
1 Commits
issue-4293
...
monotonic
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
442d58bccc |
1
.github/workflows/integrationci.yaml
vendored
1
.github/workflows/integrationci.yaml
vendored
@@ -54,7 +54,6 @@ jobs:
|
||||
- querierscalar
|
||||
- queriercommon
|
||||
- rawexportdata
|
||||
- promqlconformance
|
||||
- querierauthz
|
||||
- role
|
||||
- rootuser
|
||||
|
||||
@@ -24695,149 +24695,6 @@ paths:
|
||||
summary: Replace variables
|
||||
tags:
|
||||
- querier
|
||||
/prometheus/api/v1/query:
|
||||
get:
|
||||
deprecated: false
|
||||
description: Evaluate a PromQL expression at a single instant. Request and
|
||||
response follow the Prometheus HTTP API (https://prometheus.io/docs/prometheus/latest/querying/api/);
|
||||
the /prometheus prefix distinguishes these PromQL-only endpoints from the
|
||||
SigNoz query APIs. Also accepts POST with form-encoded parameters.
|
||||
operationId: PrometheusInstantQuery
|
||||
parameters:
|
||||
- description: PromQL expression to evaluate
|
||||
in: query
|
||||
name: query
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- description: 'Evaluation timestamp: float unix seconds or RFC3339. Defaults
|
||||
to the server''s current time.'
|
||||
in: query
|
||||
name: time
|
||||
schema:
|
||||
type: string
|
||||
- description: 'Evaluation timeout: float seconds or a Prometheus duration
|
||||
string (e.g. 30s).'
|
||||
in: query
|
||||
name: timeout
|
||||
schema:
|
||||
type: string
|
||||
- description: Set to any value to include query statistics in the response.
|
||||
in: query
|
||||
name: stats
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
properties:
|
||||
result: {}
|
||||
resultType:
|
||||
enum:
|
||||
- matrix
|
||||
- vector
|
||||
- scalar
|
||||
- string
|
||||
type: string
|
||||
stats: {}
|
||||
type: object
|
||||
status:
|
||||
enum:
|
||||
- success
|
||||
type: string
|
||||
type: object
|
||||
description: Query evaluated successfully
|
||||
"400":
|
||||
description: Unparsable expression or parameters (errorType bad_data)
|
||||
"422":
|
||||
description: Expression failed to evaluate (errorType execution)
|
||||
"503":
|
||||
description: Query timed out or was canceled
|
||||
summary: Prometheus instant query
|
||||
tags:
|
||||
- prometheus
|
||||
/prometheus/api/v1/query_range:
|
||||
get:
|
||||
deprecated: false
|
||||
description: Evaluate a PromQL expression over a range of time on a fixed
|
||||
step grid. Request and response follow the Prometheus HTTP API
|
||||
(https://prometheus.io/docs/prometheus/latest/querying/api/); the
|
||||
/prometheus prefix distinguishes these PromQL-only endpoints from the
|
||||
SigNoz query APIs. Grids are capped at 11,000 points per series. Also
|
||||
accepts POST with form-encoded parameters.
|
||||
operationId: PrometheusRangeQuery
|
||||
parameters:
|
||||
- description: PromQL expression to evaluate
|
||||
in: query
|
||||
name: query
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- description: 'Start timestamp: float unix seconds or RFC3339.'
|
||||
in: query
|
||||
name: start
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- description: 'End timestamp: float unix seconds or RFC3339.'
|
||||
in: query
|
||||
name: end
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- description: 'Grid step: float seconds or a Prometheus duration string
|
||||
(e.g. 30s). Must be positive.'
|
||||
in: query
|
||||
name: step
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- description: 'Evaluation timeout: float seconds or a Prometheus duration
|
||||
string (e.g. 30s).'
|
||||
in: query
|
||||
name: timeout
|
||||
schema:
|
||||
type: string
|
||||
- description: Set to any value to include query statistics in the response.
|
||||
in: query
|
||||
name: stats
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
properties:
|
||||
result: {}
|
||||
resultType:
|
||||
enum:
|
||||
- matrix
|
||||
type: string
|
||||
stats: {}
|
||||
type: object
|
||||
status:
|
||||
enum:
|
||||
- success
|
||||
type: string
|
||||
type: object
|
||||
description: Query evaluated successfully
|
||||
"400":
|
||||
description: Unparsable expression or parameters, or a grid past the 11,000-point
|
||||
cap (errorType bad_data)
|
||||
"422":
|
||||
description: Expression failed to evaluate (errorType execution)
|
||||
"503":
|
||||
description: Query timed out or was canceled
|
||||
summary: Prometheus range query
|
||||
tags:
|
||||
- prometheus
|
||||
servers:
|
||||
- description: The fully qualified URL to the SigNoz APIServer.
|
||||
url: https://{host}:{port}{base_path}
|
||||
|
||||
@@ -2,29 +2,6 @@ import { RenderErrorResponseDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { AxiosError } from 'axios';
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
// The wire shape these handlers can actually rely on. The generated
|
||||
// RenderErrorResponseDTO marks code/message/url/errors as required, but the
|
||||
// server omits any of them even on valid errors (e.g. a 400 with just a
|
||||
// message), so a present `error` object is all the guard can guarantee.
|
||||
type ErrorEnvelope = {
|
||||
error: {
|
||||
code?: string;
|
||||
message?: string;
|
||||
url?: string;
|
||||
errors?: { message?: string }[];
|
||||
};
|
||||
};
|
||||
|
||||
function isErrorEnvelope(data: unknown): data is ErrorEnvelope {
|
||||
return (
|
||||
typeof data === 'object' &&
|
||||
data !== null &&
|
||||
'error' in data &&
|
||||
typeof (data as ErrorEnvelope).error === 'object' &&
|
||||
(data as ErrorEnvelope).error !== null
|
||||
);
|
||||
}
|
||||
|
||||
// @deprecated Use convertToApiError instead
|
||||
export function ErrorResponseHandlerForGeneratedAPIs(
|
||||
error: AxiosError<RenderErrorResponseDTO>,
|
||||
@@ -33,29 +10,15 @@ export function ErrorResponseHandlerForGeneratedAPIs(
|
||||
// The request was made and the server responded with a status code
|
||||
// that falls out of the range of 2xx
|
||||
if (response) {
|
||||
// The body isn't guaranteed to be an error envelope — e.g. a gateway 5xx
|
||||
// with an HTML/empty body during a deploy. Verify the shape before reading
|
||||
// it; otherwise synthesize a consistent error from the status.
|
||||
const data: unknown = response.data;
|
||||
if (isErrorEnvelope(data)) {
|
||||
const { code, message, url, errors } = data.error;
|
||||
throw new APIError({
|
||||
httpStatusCode: response.status || 500,
|
||||
error: {
|
||||
code: code ?? '',
|
||||
message: message ?? '',
|
||||
url: url ?? '',
|
||||
errors: (errors ?? []).map((e) => ({ message: e.message ?? '' })),
|
||||
},
|
||||
});
|
||||
}
|
||||
throw new APIError({
|
||||
httpStatusCode: response.status || 500,
|
||||
error: {
|
||||
code: 'UPSTREAM_UNAVAILABLE',
|
||||
message: error.message || 'Something went wrong',
|
||||
url: '',
|
||||
errors: [],
|
||||
code: response.data.error.code,
|
||||
message: response.data.error.message,
|
||||
url: response.data.error.url ?? '',
|
||||
errors: (response.data.error.errors ?? []).map((e) => ({
|
||||
message: e.message ?? '',
|
||||
})),
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -99,7 +62,9 @@ export function convertToApiError(
|
||||
return new APIError({
|
||||
httpStatusCode: response?.status || error.status || 500,
|
||||
error: {
|
||||
code: errorData?.code || 'UPSTREAM_UNAVAILABLE',
|
||||
code:
|
||||
errorData?.code ||
|
||||
String(response?.status || error.code || 'unknown_error'),
|
||||
message:
|
||||
errorData?.message ||
|
||||
response?.statusText ||
|
||||
|
||||
@@ -2,38 +2,19 @@ import { AxiosError } from 'axios';
|
||||
import { ErrorV2Resp } from 'types/api';
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
function isErrorV2Resp(data: unknown): data is ErrorV2Resp {
|
||||
return (
|
||||
typeof data === 'object' &&
|
||||
data !== null &&
|
||||
'error' in data &&
|
||||
typeof (data as ErrorV2Resp).error?.code === 'string'
|
||||
);
|
||||
}
|
||||
|
||||
// reference - https://axios-http.com/docs/handling_errors
|
||||
export function ErrorResponseHandlerV2(error: AxiosError<ErrorV2Resp>): never {
|
||||
const { response, request } = error;
|
||||
// The request was made and the server responded with a status code
|
||||
// that falls out of the range of 2xx
|
||||
if (response) {
|
||||
// response.data isn't guaranteed to be a V2 envelope (e.g. a gateway 5xx
|
||||
// with an HTML/empty body during a deploy), so verify the shape first.
|
||||
const data: unknown = response.data;
|
||||
if (isErrorV2Resp(data)) {
|
||||
const { code, message, url, errors } = data.error;
|
||||
throw new APIError({
|
||||
httpStatusCode: response.status || 500,
|
||||
error: { code, message, url, errors },
|
||||
});
|
||||
}
|
||||
throw new APIError({
|
||||
httpStatusCode: response.status || 500,
|
||||
error: {
|
||||
code: 'UPSTREAM_UNAVAILABLE',
|
||||
message: error.message || 'Something went wrong',
|
||||
url: '',
|
||||
errors: [],
|
||||
code: response.data.error.code,
|
||||
message: response.data.error.message,
|
||||
url: response.data.error.url,
|
||||
errors: response.data.error.errors,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
import { ErrorResponseHandlerV2 } from 'api/ErrorResponseHandlerV2';
|
||||
import { AxiosError } from 'axios';
|
||||
import { ErrorV2Resp } from 'types/api';
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
function asAxiosError(partial: Partial<AxiosError>): AxiosError<ErrorV2Resp> {
|
||||
return partial as AxiosError<ErrorV2Resp>;
|
||||
}
|
||||
|
||||
// ErrorResponseHandlerV2 always throws — capture the APIError so assertions stay
|
||||
// unconditional.
|
||||
function runHandler(error: AxiosError<ErrorV2Resp>): APIError {
|
||||
try {
|
||||
ErrorResponseHandlerV2(error);
|
||||
} catch (thrown) {
|
||||
return thrown as APIError;
|
||||
}
|
||||
throw new Error('expected ErrorResponseHandlerV2 to throw');
|
||||
}
|
||||
|
||||
type ExpectedError = {
|
||||
httpStatusCode: number;
|
||||
code: string;
|
||||
message: string;
|
||||
errors: { message: string }[];
|
||||
};
|
||||
|
||||
// One row per response shape the handler must normalize. New shapes (with
|
||||
// different bodies) can be added here without a new test block.
|
||||
const cases: {
|
||||
name: string;
|
||||
error: AxiosError<ErrorV2Resp>;
|
||||
expected: ExpectedError;
|
||||
}[] = [
|
||||
{
|
||||
name: 'well-formed V2 error envelope',
|
||||
error: asAxiosError({
|
||||
message: 'Request failed with status code 400',
|
||||
response: {
|
||||
status: 400,
|
||||
data: {
|
||||
error: {
|
||||
code: 'bad_request',
|
||||
message: 'Invalid dashboard payload',
|
||||
url: 'https://signoz.io/docs',
|
||||
errors: [{ message: 'name is required' }, { message: 'name too long' }],
|
||||
},
|
||||
},
|
||||
} as AxiosError['response'],
|
||||
}),
|
||||
expected: {
|
||||
httpStatusCode: 400,
|
||||
code: 'bad_request',
|
||||
message: 'Invalid dashboard payload',
|
||||
errors: [{ message: 'name is required' }, { message: 'name too long' }],
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression: during a deployment the gateway returns a 5xx with a
|
||||
// non-envelope body. Reading response.data.error.code used to throw a
|
||||
// TypeError from inside the handler itself. See engineering-pod#5760.
|
||||
name: '5xx with a non-envelope HTML body',
|
||||
error: asAxiosError({
|
||||
message: 'Request failed with status code 503',
|
||||
response: {
|
||||
status: 503,
|
||||
data: '<html><body>503 Service Temporarily Unavailable</body></html>',
|
||||
} as AxiosError['response'],
|
||||
}),
|
||||
expected: {
|
||||
httpStatusCode: 503,
|
||||
code: 'UPSTREAM_UNAVAILABLE',
|
||||
message: 'Request failed with status code 503',
|
||||
errors: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: '5xx with an empty body',
|
||||
error: asAxiosError({
|
||||
message: 'Request failed with status code 502',
|
||||
response: {
|
||||
status: 502,
|
||||
data: undefined,
|
||||
} as AxiosError['response'],
|
||||
}),
|
||||
expected: {
|
||||
httpStatusCode: 502,
|
||||
code: 'UPSTREAM_UNAVAILABLE',
|
||||
message: 'Request failed with status code 502',
|
||||
errors: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'no response received (network error)',
|
||||
error: asAxiosError({
|
||||
message: 'Network Error',
|
||||
code: 'ERR_NETWORK',
|
||||
name: 'AxiosError',
|
||||
request: {},
|
||||
}),
|
||||
expected: {
|
||||
httpStatusCode: 500,
|
||||
code: 'ERR_NETWORK',
|
||||
message: 'Network Error',
|
||||
errors: [],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
describe('ErrorResponseHandlerV2', () => {
|
||||
it.each(cases)(
|
||||
'normalizes $name into a consistent APIError',
|
||||
({ error, expected }) => {
|
||||
const apiError = runHandler(error);
|
||||
|
||||
expect(apiError).toBeInstanceOf(APIError);
|
||||
expect(apiError.getHttpStatusCode()).toBe(expected.httpStatusCode);
|
||||
expect(apiError.getErrorCode()).toBe(expected.code);
|
||||
expect(apiError.getErrorMessage()).toBe(expected.message);
|
||||
// The sub-error messages feed several parts of the UI, so assert them.
|
||||
expect(apiError.getErrorDetails().error.errors).toStrictEqual(
|
||||
expected.errors,
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import { buildCompositeKey } from 'container/OptionsMenu/utils';
|
||||
import { useGetQueryKeySuggestions } from 'hooks/querySuggestions/useGetQueryKeySuggestions';
|
||||
import {
|
||||
FieldContext,
|
||||
FieldDataType,
|
||||
SignalType,
|
||||
TelemetryFieldKey,
|
||||
} from 'types/api/v5/queryRange';
|
||||
@@ -54,7 +55,7 @@ function OtherFields({
|
||||
key: buildCompositeKey(attr.name, attr.fieldContext as string),
|
||||
signal: attr.signal as SignalType,
|
||||
fieldContext: attr.fieldContext as FieldContext,
|
||||
fieldDataType: attr.fieldDataType,
|
||||
fieldDataType: attr.fieldDataType as FieldDataType,
|
||||
}),
|
||||
);
|
||||
const addedIds = new Set(
|
||||
|
||||
@@ -25,12 +25,12 @@ import CodeMirror, {
|
||||
} from '@uiw/react-codemirror';
|
||||
import { Button, Popover, Tooltip } from 'antd';
|
||||
import { getKeySuggestions } from 'api/querySuggestions/getKeySuggestions';
|
||||
import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
|
||||
import { QueryBuilderKeys } from 'constants/queryBuilder';
|
||||
import { tracesAggregateOperatorOptions } from 'constants/queryBuilderOperators';
|
||||
import { useIsDarkMode } from 'hooks/useDarkMode';
|
||||
import { Info, TriangleAlert } from '@signozhq/icons';
|
||||
import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { FieldDataType } from 'types/api/v5/queryRange';
|
||||
import { TracesAggregatorOperator } from 'types/common/queryBuilder';
|
||||
|
||||
import { useQueryBuilderV2Context } from '../../QueryBuilderV2Context';
|
||||
@@ -323,16 +323,16 @@ function QueryAggregationSelect({
|
||||
TracesAggregatorOperator.RATE,
|
||||
];
|
||||
|
||||
const fieldDataType: FieldDataType | undefined =
|
||||
const fieldDataType =
|
||||
functionContextForFetch &&
|
||||
operatorsWithoutDataType.includes(functionContextForFetch)
|
||||
? undefined
|
||||
: 'number';
|
||||
: QUERY_BUILDER_KEY_TYPES.NUMBER;
|
||||
|
||||
return getKeySuggestions({
|
||||
signal: queryData.dataSource,
|
||||
searchText: '',
|
||||
fieldDataType,
|
||||
fieldDataType: fieldDataType as QUERY_BUILDER_KEY_TYPES,
|
||||
});
|
||||
},
|
||||
{
|
||||
|
||||
@@ -92,76 +92,52 @@
|
||||
color: var(--l3-foreground);
|
||||
}
|
||||
|
||||
.only-btn,
|
||||
.only-btn {
|
||||
cursor: not-allowed;
|
||||
color: var(--l3-foreground);
|
||||
}
|
||||
|
||||
.toggle-btn {
|
||||
cursor: not-allowed;
|
||||
color: var(--l3-foreground);
|
||||
}
|
||||
}
|
||||
|
||||
// Stack "Only"/"All" and "Toggle" in a single cell so revealing
|
||||
// one swaps in place instead of shifting the row layout.
|
||||
.value-actions {
|
||||
display: grid;
|
||||
align-items: center;
|
||||
justify-items: end;
|
||||
|
||||
> * {
|
||||
grid-area: 1 / 1;
|
||||
}
|
||||
.only-btn {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.only-btn,
|
||||
.toggle-btn {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.toggle-btn:hover {
|
||||
background-color: unset;
|
||||
}
|
||||
|
||||
.only-btn:hover {
|
||||
background-color: unset;
|
||||
}
|
||||
}
|
||||
|
||||
.checkbox-value-section:hover {
|
||||
.toggle-btn {
|
||||
display: none;
|
||||
}
|
||||
.only-btn {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 21px;
|
||||
opacity: 0;
|
||||
transform: translateX(4px);
|
||||
// `display` is animated as a discrete property so the button
|
||||
// stays mounted through its fade-out on exit.
|
||||
transition:
|
||||
opacity 0.16s ease,
|
||||
transform 0.16s ease,
|
||||
display 0.16s allow-discrete;
|
||||
|
||||
&:hover {
|
||||
background-color: unset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Hovering the label/value area reveals the "Only"/"All" action.
|
||||
.checkbox-value-section:hover .only-btn {
|
||||
display: flex;
|
||||
opacity: 1;
|
||||
transform: translateX(0);
|
||||
|
||||
// Entry animation start state (fires on the display switch).
|
||||
@starting-style {
|
||||
opacity: 0;
|
||||
transform: translateX(4px);
|
||||
}
|
||||
}
|
||||
|
||||
// Hovering the checkbox reveals the "Toggle" action.
|
||||
.check-box:hover ~ .checkbox-value-section .toggle-btn {
|
||||
display: flex;
|
||||
opacity: 1;
|
||||
transform: translateX(0);
|
||||
|
||||
@starting-style {
|
||||
opacity: 0;
|
||||
transform: translateX(4px);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.only-btn,
|
||||
.value:hover {
|
||||
.toggle-btn {
|
||||
transition: none;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
height: 21px;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,14 +53,12 @@ function CheckboxValueRow({
|
||||
</Typography.Text>
|
||||
</TooltipSimple>
|
||||
)}
|
||||
<div className="value-actions">
|
||||
<Button type="text" className="only-btn">
|
||||
{onlyButtonLabel}
|
||||
</Button>
|
||||
<Button type="text" className="toggle-btn">
|
||||
Toggle
|
||||
</Button>
|
||||
</div>
|
||||
<Button type="text" className="only-btn">
|
||||
{onlyButtonLabel}
|
||||
</Button>
|
||||
<Button type="text" className="toggle-btn">
|
||||
Toggle
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
.warning-popover-overlay {
|
||||
--antd-arrow-background-color: var(--l2-background);
|
||||
}
|
||||
|
||||
.warning-content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background-color: var(--l2-background);
|
||||
|
||||
// === SECTION: Summary (Top)
|
||||
&__summary-section {
|
||||
@@ -15,25 +10,16 @@
|
||||
|
||||
&__summary {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing-8);
|
||||
padding: 16px;
|
||||
}
|
||||
|
||||
&__summary-left {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
align-items: baseline;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
&__icon-wrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
flex-shrink: 0;
|
||||
height: var(--spacing-12);
|
||||
}
|
||||
|
||||
&__summary-text {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
|
||||
@@ -2,7 +2,6 @@ import { ReactNode, useMemo } from 'react';
|
||||
import { Color } from '@signozhq/design-tokens';
|
||||
import { Button, Popover, PopoverProps } from 'antd';
|
||||
import ErrorIcon from 'assets/Error';
|
||||
import cx from 'classnames';
|
||||
import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
|
||||
import { BookOpenText, ChevronsDown, TriangleAlert } from '@signozhq/icons';
|
||||
import KeyValueLabel from 'periscope/components/KeyValueLabel';
|
||||
@@ -31,18 +30,16 @@ export function WarningContent({ warning }: WarningContentProps): JSX.Element {
|
||||
{/* Summary Header */}
|
||||
<section className="warning-content__summary-section">
|
||||
<header className="warning-content__summary">
|
||||
{(warningCode || warningMessage) && (
|
||||
<div className="warning-content__summary-left">
|
||||
<div className="warning-content__icon-wrapper">
|
||||
<ErrorIcon />
|
||||
</div>
|
||||
|
||||
<div className="warning-content__summary-text">
|
||||
<h2 className="warning-content__warning-code">{warningCode}</h2>
|
||||
<p className="warning-content__warning-message">{warningMessage}</p>
|
||||
</div>
|
||||
<div className="warning-content__summary-left">
|
||||
<div className="warning-content__icon-wrapper">
|
||||
<ErrorIcon />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="warning-content__summary-text">
|
||||
<h2 className="warning-content__warning-code">{warningCode}</h2>
|
||||
<p className="warning-content__warning-message">{warningMessage}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{warningUrl && (
|
||||
<div className="warning-content__summary-right">
|
||||
@@ -157,10 +154,6 @@ function WarningPopover({
|
||||
overlayInnerStyle={{ padding: 0 }}
|
||||
autoAdjustOverflow
|
||||
{...popoverProps}
|
||||
overlayClassName={cx(
|
||||
'warning-popover-overlay',
|
||||
popoverProps.overlayClassName,
|
||||
)}
|
||||
>
|
||||
{children || (
|
||||
<TriangleAlert
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
// ** Helpers
|
||||
import { MetrictypesTypeDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import {
|
||||
MetrictypesTemporalityDTO,
|
||||
MetrictypesTypeDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
import { defaultTraceSelectedColumns } from 'container/OptionsMenu/constants';
|
||||
import { createIdFromObjectFields } from 'lib/createIdFromObjectFields';
|
||||
import { createNewBuilderItemName } from 'lib/newQueryBuilder/createNewBuilderItemName';
|
||||
@@ -389,11 +392,19 @@ const METRIC_TYPE_TO_ATTRIBUTE_TYPE: Record<
|
||||
export function toAttributeType(
|
||||
metricType: MetrictypesTypeDTO | undefined,
|
||||
isMonotonic?: boolean,
|
||||
temporality?: MetrictypesTemporalityDTO,
|
||||
): ATTRIBUTE_TYPES | '' {
|
||||
if (!metricType) {
|
||||
return '';
|
||||
}
|
||||
if (metricType === MetrictypesTypeDTO.sum && isMonotonic === false) {
|
||||
// Monotonicity carries meaning only for cumulative sums: a non-monotonic
|
||||
// cumulative sum is a gauge for all practical purposes. Delta sums are
|
||||
// counters regardless of monotonicity.
|
||||
if (
|
||||
metricType === MetrictypesTypeDTO.sum &&
|
||||
isMonotonic === false &&
|
||||
temporality === MetrictypesTemporalityDTO.cumulative
|
||||
) {
|
||||
return ATTRIBUTE_TYPES.GAUGE;
|
||||
}
|
||||
return METRIC_TYPE_TO_ATTRIBUTE_TYPE[metricType] || '';
|
||||
|
||||
@@ -33,6 +33,7 @@ function AllAttributes({
|
||||
metricName,
|
||||
metricType,
|
||||
isMonotonic,
|
||||
temporality,
|
||||
minTime,
|
||||
maxTime,
|
||||
}: AllAttributesProps): JSX.Element {
|
||||
@@ -71,6 +72,7 @@ function AllAttributes({
|
||||
groupBy,
|
||||
limit,
|
||||
isMonotonic,
|
||||
temporality,
|
||||
);
|
||||
handleExplorerTabChange(
|
||||
PANEL_TYPES.TIME_SERIES,
|
||||
@@ -89,7 +91,7 @@ function AllAttributes({
|
||||
[MetricsExplorerEventKeys.AttributeKey]: groupBy,
|
||||
});
|
||||
},
|
||||
[metricName, metricType, isMonotonic, handleExplorerTabChange],
|
||||
[metricName, metricType, isMonotonic, temporality, handleExplorerTabChange],
|
||||
);
|
||||
|
||||
const goToMetricsExploreWithAppliedAttribute = useCallback(
|
||||
@@ -101,6 +103,7 @@ function AllAttributes({
|
||||
undefined,
|
||||
undefined,
|
||||
isMonotonic,
|
||||
temporality,
|
||||
);
|
||||
handleExplorerTabChange(
|
||||
PANEL_TYPES.TIME_SERIES,
|
||||
@@ -120,7 +123,7 @@ function AllAttributes({
|
||||
[MetricsExplorerEventKeys.AttributeValue]: value,
|
||||
});
|
||||
},
|
||||
[metricName, metricType, isMonotonic, handleExplorerTabChange],
|
||||
[metricName, metricType, isMonotonic, temporality, handleExplorerTabChange],
|
||||
);
|
||||
|
||||
const handleKeyMenuItemClick = useCallback(
|
||||
|
||||
@@ -6,7 +6,7 @@ import { Skeleton } from 'antd';
|
||||
import { Typography } from '@signozhq/ui/typography';
|
||||
import {
|
||||
useGetMetricAlerts,
|
||||
useGetMetricDashboardsV2,
|
||||
useGetMetricDashboards,
|
||||
} from 'api/generated/services/metrics';
|
||||
import { QueryParams } from 'constants/query';
|
||||
import ROUTES from 'constants/routes';
|
||||
@@ -38,7 +38,7 @@ function DashboardsAndAlertsPopover({
|
||||
data: dashboardsData,
|
||||
isLoading: isLoadingDashboards,
|
||||
isError: isErrorDashboards,
|
||||
} = useGetMetricDashboardsV2(
|
||||
} = useGetMetricDashboards(
|
||||
{
|
||||
metricName,
|
||||
},
|
||||
@@ -55,8 +55,7 @@ function DashboardsAndAlertsPopover({
|
||||
|
||||
const dashboards = useMemo(() => {
|
||||
const currentDashboards = dashboardsData?.data.dashboards ?? [];
|
||||
// The API returns one entry per referencing panel, so a dashboard repeats
|
||||
// once per panel that uses the metric.
|
||||
// Remove duplicate dashboards
|
||||
return currentDashboards.filter(
|
||||
(dashboard, index, self) =>
|
||||
index === self.findIndex((t) => t.dashboardId === dashboard.dashboardId),
|
||||
|
||||
@@ -86,6 +86,7 @@ function MetricDetails({
|
||||
undefined,
|
||||
undefined,
|
||||
metadata?.isMonotonic,
|
||||
metadata?.temporality,
|
||||
);
|
||||
handleExplorerTabChange(
|
||||
PANEL_TYPES.TIME_SERIES,
|
||||
@@ -108,6 +109,7 @@ function MetricDetails({
|
||||
handleExplorerTabChange,
|
||||
metadata?.type,
|
||||
metadata?.isMonotonic,
|
||||
metadata?.temporality,
|
||||
]);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -196,6 +198,7 @@ function MetricDetails({
|
||||
metricName={metricName}
|
||||
metricType={metadata?.type}
|
||||
isMonotonic={metadata?.isMonotonic}
|
||||
temporality={metadata?.temporality}
|
||||
minTime={minTime}
|
||||
maxTime={maxTime}
|
||||
/>
|
||||
|
||||
@@ -23,7 +23,7 @@ const useGetMetricAlertsMock = jest.spyOn(
|
||||
);
|
||||
const useGetMetricDashboardsMock = jest.spyOn(
|
||||
metricsExplorerHooks,
|
||||
'useGetMetricDashboardsV2',
|
||||
'useGetMetricDashboards',
|
||||
);
|
||||
|
||||
describe('DashboardsAndAlertsPopover', () => {
|
||||
@@ -153,15 +153,11 @@ describe('DashboardsAndAlertsPopover', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('collapses multiple panels of the same dashboard into one entry', async () => {
|
||||
it('renders unique dashboards even when there are duplicates', async () => {
|
||||
useGetMetricDashboardsMock.mockReturnValue(
|
||||
getMockDashboardsData({
|
||||
data: {
|
||||
dashboards: [
|
||||
MOCK_DASHBOARD_1,
|
||||
MOCK_DASHBOARD_2,
|
||||
{ ...MOCK_DASHBOARD_1, panelId: '3', panelName: 'Panel 3' },
|
||||
],
|
||||
dashboards: [MOCK_DASHBOARD_1, MOCK_DASHBOARD_2, MOCK_DASHBOARD_1],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -2,7 +2,7 @@ import * as metricsExplorerHooks from 'api/generated/services/metrics';
|
||||
import {
|
||||
GetMetricAlerts200,
|
||||
GetMetricAttributes200,
|
||||
GetMetricDashboardsV2200,
|
||||
GetMetricDashboards200,
|
||||
GetMetricHighlights200,
|
||||
GetMetricMetadata200,
|
||||
MetrictypesTemporalityDTO,
|
||||
@@ -40,14 +40,14 @@ export function getMockMetricHighlightsData(
|
||||
export const MOCK_DASHBOARD_1 = {
|
||||
dashboardName: 'Dashboard 1',
|
||||
dashboardId: '1',
|
||||
panelId: '1',
|
||||
panelName: 'Panel 1',
|
||||
widgetId: '1',
|
||||
widgetName: 'Widget 1',
|
||||
};
|
||||
export const MOCK_DASHBOARD_2 = {
|
||||
dashboardName: 'Dashboard 2',
|
||||
dashboardId: '2',
|
||||
panelId: '2',
|
||||
panelName: 'Panel 2',
|
||||
widgetId: '2',
|
||||
widgetName: 'Widget 2',
|
||||
};
|
||||
export const MOCK_ALERT_1 = {
|
||||
alertName: 'Alert 1',
|
||||
@@ -59,7 +59,7 @@ export const MOCK_ALERT_2 = {
|
||||
};
|
||||
|
||||
export function getMockDashboardsData(
|
||||
overrides?: Partial<GetMetricDashboardsV2200>,
|
||||
overrides?: Partial<GetMetricDashboards200>,
|
||||
{
|
||||
isLoading = false,
|
||||
isError = false,
|
||||
@@ -67,7 +67,7 @@ export function getMockDashboardsData(
|
||||
isLoading?: boolean;
|
||||
isError?: boolean;
|
||||
} = {},
|
||||
): ReturnType<typeof metricsExplorerHooks.useGetMetricDashboardsV2> {
|
||||
): ReturnType<typeof metricsExplorerHooks.useGetMetricDashboards> {
|
||||
return {
|
||||
data: {
|
||||
data: {
|
||||
@@ -79,7 +79,7 @@ export function getMockDashboardsData(
|
||||
|
||||
isLoading,
|
||||
isError,
|
||||
} as ReturnType<typeof metricsExplorerHooks.useGetMetricDashboardsV2>;
|
||||
} as ReturnType<typeof metricsExplorerHooks.useGetMetricDashboards>;
|
||||
}
|
||||
|
||||
export function getMockAlertsData(
|
||||
|
||||
@@ -147,6 +147,44 @@ describe('MetricDetails utils', () => {
|
||||
expect(query.builder.queryData[0]?.spaceAggregation).toBe('sum');
|
||||
});
|
||||
|
||||
it('should treat a non-monotonic cumulative SUM as a gauge', () => {
|
||||
const query = getMetricDetailsQuery(
|
||||
TEST_METRIC_NAME,
|
||||
MetrictypesTypeDTO.sum,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
MetrictypesTemporalityDTO.cumulative,
|
||||
);
|
||||
|
||||
expect(query.builder.queryData[0]?.aggregateAttribute?.type).toBe(
|
||||
ATTRIBUTE_TYPES.GAUGE,
|
||||
);
|
||||
expect(query.builder.queryData[0]?.aggregateOperator).toBe('avg');
|
||||
expect(query.builder.queryData[0]?.timeAggregation).toBe('avg');
|
||||
expect(query.builder.queryData[0]?.spaceAggregation).toBe('avg');
|
||||
});
|
||||
|
||||
it('should treat a non-monotonic delta SUM as a counter', () => {
|
||||
const query = getMetricDetailsQuery(
|
||||
TEST_METRIC_NAME,
|
||||
MetrictypesTypeDTO.sum,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
MetrictypesTemporalityDTO.delta,
|
||||
);
|
||||
|
||||
expect(query.builder.queryData[0]?.aggregateAttribute?.type).toBe(
|
||||
ATTRIBUTE_TYPES.SUM,
|
||||
);
|
||||
expect(query.builder.queryData[0]?.aggregateOperator).toBe('rate');
|
||||
expect(query.builder.queryData[0]?.timeAggregation).toBe('rate');
|
||||
expect(query.builder.queryData[0]?.spaceAggregation).toBe('sum');
|
||||
});
|
||||
|
||||
it('should create correct query for GAUGE metric type', () => {
|
||||
const query = getMetricDetailsQuery(
|
||||
TEST_METRIC_NAME,
|
||||
|
||||
@@ -35,6 +35,7 @@ export interface AllAttributesProps {
|
||||
metricName: string;
|
||||
metricType: MetrictypesTypeDTO | undefined;
|
||||
isMonotonic?: boolean;
|
||||
temporality?: MetrictypesTemporalityDTO;
|
||||
minTime?: number;
|
||||
maxTime?: number;
|
||||
}
|
||||
|
||||
@@ -89,16 +89,21 @@ export function getMetricDetailsQuery(
|
||||
groupBy?: string,
|
||||
limit?: number,
|
||||
isMonotonic?: boolean,
|
||||
temporality?: MetrictypesTemporalityDTO,
|
||||
): Query {
|
||||
let timeAggregation;
|
||||
let spaceAggregation;
|
||||
let aggregateOperator;
|
||||
const isNonMonotonicSum =
|
||||
metricType === MetrictypesTypeDTO.sum && isMonotonic === false;
|
||||
// Monotonicity carries meaning only for cumulative sums; delta sums are
|
||||
// counters regardless of monotonicity.
|
||||
const isNonMonotonicCumulativeSum =
|
||||
metricType === MetrictypesTypeDTO.sum &&
|
||||
isMonotonic === false &&
|
||||
temporality === MetrictypesTemporalityDTO.cumulative;
|
||||
|
||||
switch (metricType) {
|
||||
case MetrictypesTypeDTO.sum:
|
||||
if (isNonMonotonicSum) {
|
||||
if (isNonMonotonicCumulativeSum) {
|
||||
timeAggregation = 'avg';
|
||||
spaceAggregation = 'avg';
|
||||
aggregateOperator = 'avg';
|
||||
@@ -131,7 +136,7 @@ export function getMetricDetailsQuery(
|
||||
break;
|
||||
}
|
||||
|
||||
const attributeType = toAttributeType(metricType, isMonotonic);
|
||||
const attributeType = toAttributeType(metricType, isMonotonic, temporality);
|
||||
|
||||
return {
|
||||
...initialQueriesMap[DataSource.METRICS],
|
||||
|
||||
@@ -113,7 +113,7 @@ const useOptionsMenu = ({
|
||||
(suggestion) => ({
|
||||
name: suggestion.name,
|
||||
signal: suggestion.signal as SignalType,
|
||||
fieldDataType: suggestion.fieldDataType,
|
||||
fieldDataType: suggestion.fieldDataType as FieldDataType,
|
||||
fieldContext: suggestion.fieldContext as FieldContext,
|
||||
}),
|
||||
);
|
||||
@@ -192,7 +192,7 @@ const useOptionsMenu = ({
|
||||
name: e.name,
|
||||
signal: e.signal as SignalType,
|
||||
fieldContext: e.fieldContext as FieldContext,
|
||||
fieldDataType: e.fieldDataType,
|
||||
fieldDataType: e.fieldDataType as FieldDataType,
|
||||
}));
|
||||
}
|
||||
if (dataSource === DataSource.TRACES) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from '@testing-library/react';
|
||||
import {
|
||||
MetricsexplorertypesListMetricDTO,
|
||||
MetrictypesTemporalityDTO,
|
||||
MetrictypesTypeDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
import { ENTITY_VERSION_V5 } from 'constants/app';
|
||||
@@ -70,7 +71,7 @@ function makeMetric(
|
||||
type: MetrictypesTypeDTO.sum,
|
||||
isMonotonic: true,
|
||||
description: '',
|
||||
temporality: 'cumulative' as never,
|
||||
temporality: MetrictypesTemporalityDTO.cumulative,
|
||||
unit: '',
|
||||
...overrides,
|
||||
};
|
||||
@@ -393,12 +394,13 @@ describe('selecting a metric type updates the aggregation options', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('non-monotonic Sum metric is treated as Gauge', () => {
|
||||
it('non-monotonic cumulative Sum metric is treated as Gauge', () => {
|
||||
returnMetrics([
|
||||
makeMetric({
|
||||
metricName: 'active_connections',
|
||||
type: MetrictypesTypeDTO.sum,
|
||||
isMonotonic: false,
|
||||
temporality: MetrictypesTemporalityDTO.cumulative,
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -427,6 +429,36 @@ describe('selecting a metric type updates the aggregation options', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('non-monotonic delta Sum metric remains a counter with Rate/Increase options', () => {
|
||||
returnMetrics([
|
||||
makeMetric({
|
||||
metricName: 'gcp_reject_connections_count',
|
||||
type: MetrictypesTypeDTO.sum,
|
||||
isMonotonic: false,
|
||||
temporality: MetrictypesTemporalityDTO.delta,
|
||||
}),
|
||||
]);
|
||||
|
||||
render(<MetricQueryHarness query={makeQuery()} />);
|
||||
|
||||
const input = screen.getByRole('combobox');
|
||||
fireEvent.change(input, {
|
||||
target: { value: 'gcp_reject_connections_count' },
|
||||
});
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(getOptionLabels('time-agg-options')).toStrictEqual([
|
||||
'Rate',
|
||||
'Increase',
|
||||
]);
|
||||
expect(getOptionLabels('space-agg-options')).toStrictEqual([
|
||||
'Sum',
|
||||
'Avg',
|
||||
'Min',
|
||||
'Max',
|
||||
]);
|
||||
});
|
||||
|
||||
it('Histogram metric shows no time options and P50–P99 space options', () => {
|
||||
returnMetrics([
|
||||
makeMetric({
|
||||
|
||||
@@ -34,7 +34,7 @@ export type MetricNameSelectorProps = {
|
||||
function getAttributeType(
|
||||
metric: MetricsexplorertypesListMetricDTO,
|
||||
): ATTRIBUTE_TYPES | '' {
|
||||
return toAttributeType(metric.type, metric.isMonotonic);
|
||||
return toAttributeType(metric.type, metric.isMonotonic, metric.temporality);
|
||||
}
|
||||
|
||||
function createAutocompleteData(
|
||||
|
||||
@@ -4,11 +4,11 @@ import { Input } from '@signozhq/ui/input';
|
||||
import { Skeleton } from 'antd';
|
||||
import { getKeySuggestions } from 'api/querySuggestions/getKeySuggestions';
|
||||
import OverlayScrollbar from 'components/OverlayScrollbar/OverlayScrollbar';
|
||||
import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
|
||||
import useDebounce from 'hooks/useDebounce';
|
||||
import { ContextMenu } from 'periscope/components/ContextMenu';
|
||||
import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { MetricAggregation } from 'types/api/v5/queryRange';
|
||||
import { fieldDataTypeToDataType } from 'utils/fieldDataType';
|
||||
|
||||
import { BreakoutOptionsProps } from './contextConfig';
|
||||
import { BreakoutAttributeType } from './types';
|
||||
@@ -80,7 +80,7 @@ function BreakoutOptions({
|
||||
keyArray.forEach((keyData) => {
|
||||
transformedOptions.push({
|
||||
key: keyData.name,
|
||||
dataType: fieldDataTypeToDataType(keyData.fieldDataType),
|
||||
dataType: keyData.fieldDataType as QUERY_BUILDER_KEY_TYPES,
|
||||
type: '',
|
||||
});
|
||||
});
|
||||
|
||||
@@ -238,10 +238,6 @@ describe('TableDrilldown Breakout Functionality', () => {
|
||||
expect(aggregateQueryData.groupBy).toHaveLength(1);
|
||||
expect(aggregateQueryData.groupBy[0].key).toBe('deployment.environment');
|
||||
|
||||
// The picked field's type travels with it — dropping it leaves the breakout query
|
||||
// untyped, so a drilldown on its result can't tell a number from a string.
|
||||
expect(aggregateQueryData.groupBy[0].dataType).toBe('string');
|
||||
|
||||
// Verify that orderBy has been cleared (as per getBreakoutQuery logic)
|
||||
expect(aggregateQueryData.orderBy).toStrictEqual([]);
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { OPERATORS, PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import cloneDeep from 'lodash-es/cloneDeep';
|
||||
import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { IBuilderQuery, Query } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
import { addFilterToSelectedQuery, FilterData } from './drilldownUtils';
|
||||
@@ -88,11 +89,7 @@ export const getBreakoutQuery = (
|
||||
.filter((item: IBuilderQuery) => item.queryName === aggregateData.queryName)
|
||||
.map((item: IBuilderQuery) => ({
|
||||
...item,
|
||||
// The picked field's type travels with it: dropped, the breakout query goes out
|
||||
// untyped and a drilldown on its result can't tell a number from a string.
|
||||
groupBy: [
|
||||
{ key: groupBy.key, dataType: groupBy.dataType, type: groupBy.type },
|
||||
],
|
||||
groupBy: [{ key: groupBy.key, type: groupBy.type } as BaseAutocompleteData],
|
||||
orderBy: [],
|
||||
legend: item.legend && groupBy.key ? `{{${groupBy.key}}}` : '',
|
||||
}));
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { ReactNode } from 'react';
|
||||
import {
|
||||
BaseAutocompleteData,
|
||||
DataTypes,
|
||||
} from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
|
||||
import { BaseAutocompleteData } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import { IBuilderQuery } from 'types/api/queryBuilder/queryBuilderData';
|
||||
|
||||
export type ContextMenuItem = ReactNode;
|
||||
@@ -33,8 +31,7 @@ export interface AggregateContextMenuConfig {
|
||||
|
||||
export interface BreakoutAttributeType {
|
||||
key: string;
|
||||
/** The picked field's type, in the vocabulary the group-by it becomes is read with. */
|
||||
dataType: DataTypes;
|
||||
dataType: QUERY_BUILDER_KEY_TYPES;
|
||||
type: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -11,9 +11,3 @@
|
||||
border: 1px solid var(--l3-border) !important;
|
||||
box-shadow: none !important;
|
||||
}
|
||||
|
||||
/* Highlights the dashboard name inside the delete-confirm title. */
|
||||
.deleteName {
|
||||
color: var(--danger-background);
|
||||
font-weight: var(--font-weight-medium);
|
||||
}
|
||||
|
||||
@@ -27,13 +27,16 @@ import logEvent from 'api/common/logEvent';
|
||||
import { cloneDashboardV2 } from 'api/generated/services/dashboard';
|
||||
import type { DashboardtypesGettableDashboardV2DTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import ROUTES from 'constants/routes';
|
||||
import { useDeleteDashboard } from 'hooks/dashboard/useDeleteDashboard';
|
||||
import { useSafeNavigate } from 'hooks/useSafeNavigate';
|
||||
import history from 'lib/history';
|
||||
import { DashboardDetailEvents } from 'pages/DashboardPageV2/constants/events';
|
||||
import { useAppContext } from 'providers/App/App';
|
||||
import { useErrorModal } from 'providers/ErrorModalProvider';
|
||||
import APIError from 'types/api/error';
|
||||
import { USER_ROLES } from 'types/roles';
|
||||
|
||||
import ConfirmDeleteDialog from '../../components/ConfirmDeleteDialog/ConfirmDeleteDialog';
|
||||
import DisabledControlTooltip from '../../components/DisabledControlTooltip/DisabledControlTooltip';
|
||||
import DisabledMenuItemLabel from '../../components/DisabledMenuItemLabel/DisabledMenuItemLabel';
|
||||
import DashboardSettings from '../../DashboardSettings';
|
||||
@@ -42,7 +45,6 @@ import SectionTitleModal from '../../PanelsAndSectionsLayout/Section/SectionTitl
|
||||
import JsonEditorDrawer from '../JsonEditorDrawer/JsonEditorDrawer';
|
||||
import SettingsDrawer from '../SettingsDrawer';
|
||||
import styles from './DashboardActions.module.scss';
|
||||
import { useDeleteDashboardAction } from './useDeleteDashboardAction';
|
||||
import { DASHBOARD_LOCKED_REASON } from '../../hooks/useDashboardEditGuard';
|
||||
import { useDashboardStore } from '../../store/useDashboardStore';
|
||||
|
||||
@@ -82,12 +84,8 @@ function DashboardActions({
|
||||
const [isCloning, setIsCloning] = useState<boolean>(false);
|
||||
const [isNewSectionOpen, setIsNewSectionOpen] = useState<boolean>(false);
|
||||
|
||||
const { contextHolder: deleteConfirmHolder, confirmDeleteDashboard } =
|
||||
useDeleteDashboardAction({
|
||||
dashboardId: dashboard.id,
|
||||
dashboardName: title,
|
||||
panelCount: Object.keys(dashboard.spec.panels).length,
|
||||
});
|
||||
const [isDeleteOpen, setIsDeleteOpen] = useState<boolean>(false);
|
||||
const deleteDashboardMutation = useDeleteDashboard(dashboard.id);
|
||||
|
||||
// Open the settings drawer when something in the tree requests it (e.g. the
|
||||
// variables bar's "Add variable" button).
|
||||
@@ -134,6 +132,19 @@ function DashboardActions({
|
||||
}
|
||||
}, [dashboard.id, title, safeNavigate, showErrorModal]);
|
||||
|
||||
const handleConfirmDelete = useCallback((): void => {
|
||||
deleteDashboardMutation.mutate(undefined, {
|
||||
onSuccess: () => {
|
||||
void logEvent(DashboardDetailEvents.Deleted, {
|
||||
dashboardId: dashboard.id,
|
||||
panelCount: Object.keys(dashboard.spec.panels).length,
|
||||
});
|
||||
setIsDeleteOpen(false);
|
||||
history.replace(ROUTES.ALL_DASHBOARD);
|
||||
},
|
||||
});
|
||||
}, [deleteDashboardMutation, dashboard.id, dashboard.spec.panels]);
|
||||
|
||||
const handleOpenSettings = useCallback((): void => {
|
||||
void logEvent(DashboardDetailEvents.SettingsOpened, {
|
||||
dashboardId: dashboard.id,
|
||||
@@ -237,7 +248,7 @@ function DashboardActions({
|
||||
icon: <Trash2 size={14} />,
|
||||
danger: true,
|
||||
disabled: isLocked,
|
||||
onClick: confirmDeleteDashboard,
|
||||
onClick: (): void => setIsDeleteOpen(true),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -255,7 +266,6 @@ function DashboardActions({
|
||||
handleClone,
|
||||
onLockToggle,
|
||||
handleEnterFullScreen,
|
||||
confirmDeleteDashboard,
|
||||
]);
|
||||
|
||||
return (
|
||||
@@ -338,7 +348,14 @@ function DashboardActions({
|
||||
isOpen={isJsonEditorOpen}
|
||||
onClose={(): void => setIsJsonEditorOpen(false)}
|
||||
/>
|
||||
{deleteConfirmHolder}
|
||||
<ConfirmDeleteDialog
|
||||
open={isDeleteOpen}
|
||||
title={`Delete dashboard"?`}
|
||||
description={`Are you sure you want to delete this dashboard - "${title}"? This action cannot be undone.`}
|
||||
isLoading={deleteDashboardMutation.isLoading}
|
||||
onConfirm={handleConfirmDelete}
|
||||
onClose={(): void => setIsDeleteOpen(false)}
|
||||
/>
|
||||
<SectionTitleModal
|
||||
open={isNewSectionOpen}
|
||||
heading="New section"
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
import { type ReactNode, useCallback } from 'react';
|
||||
import { useQueryClient } from 'react-query';
|
||||
import { toast } from '@signozhq/ui/sonner';
|
||||
import { Typography } from '@signozhq/ui/typography';
|
||||
import logEvent from 'api/common/logEvent';
|
||||
import {
|
||||
invalidateListDashboardsForUserV2,
|
||||
useDeleteDashboardV2,
|
||||
} from 'api/generated/services/dashboard';
|
||||
import { useDeleteConfirm } from 'components/DeleteConfirmModal/useDeleteConfirm';
|
||||
import ROUTES from 'constants/routes';
|
||||
import { useDashboardPreferencesStore } from 'hooks/dashboard/useDashboardPreference';
|
||||
import history from 'lib/history';
|
||||
import { DashboardDetailEvents } from 'pages/DashboardPageV2/constants/events';
|
||||
import { useErrorModal } from 'providers/ErrorModalProvider';
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
import styles from './DashboardActions.module.scss';
|
||||
|
||||
interface UseDeleteDashboardActionArgs {
|
||||
dashboardId: string;
|
||||
dashboardName: string;
|
||||
panelCount: number;
|
||||
}
|
||||
|
||||
interface UseDeleteDashboardAction {
|
||||
/** Must be rendered in the calling component for the modal to appear. */
|
||||
contextHolder: ReactNode;
|
||||
confirmDeleteDashboard: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes the open dashboard through the v2 endpoint behind the shared
|
||||
* destructive confirmation, then drops its local preferences, refreshes the
|
||||
* dashboard list cache and sends the user back to the list.
|
||||
*/
|
||||
export function useDeleteDashboardAction({
|
||||
dashboardId,
|
||||
dashboardName,
|
||||
panelCount,
|
||||
}: UseDeleteDashboardActionArgs): UseDeleteDashboardAction {
|
||||
const queryClient = useQueryClient();
|
||||
const { showErrorModal } = useErrorModal();
|
||||
const { contextHolder, confirmDelete } = useDeleteConfirm();
|
||||
const removePreferences = useDashboardPreferencesStore(
|
||||
(state) => state.removePreferences,
|
||||
);
|
||||
|
||||
const { mutate: deleteDashboard } = useDeleteDashboardV2({
|
||||
mutation: {
|
||||
onSuccess: async (): Promise<void> => {
|
||||
void logEvent(DashboardDetailEvents.Deleted, { dashboardId, panelCount });
|
||||
removePreferences(dashboardId);
|
||||
await invalidateListDashboardsForUserV2(queryClient);
|
||||
toast.success('Dashboard deleted successfully');
|
||||
history.replace(ROUTES.ALL_DASHBOARD);
|
||||
},
|
||||
onError: (error: unknown): void => {
|
||||
showErrorModal(error as APIError);
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const confirmDeleteDashboard = useCallback((): void => {
|
||||
confirmDelete({
|
||||
title: (
|
||||
<Typography.Title level={5}>
|
||||
Are you sure you want to delete the
|
||||
<Typography.Text className={styles.deleteName}>
|
||||
{' '}
|
||||
{dashboardName}{' '}
|
||||
</Typography.Text>
|
||||
dashboard?
|
||||
</Typography.Title>
|
||||
),
|
||||
content: 'This action cannot be undone.',
|
||||
// Keeps the Delete button loading until the mutation settles, then closes.
|
||||
onConfirm: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
deleteDashboard(
|
||||
{ pathParams: { id: dashboardId } },
|
||||
{ onSettled: () => resolve() },
|
||||
);
|
||||
}),
|
||||
});
|
||||
}, [confirmDelete, dashboardName, deleteDashboard, dashboardId]);
|
||||
|
||||
return { contextHolder, confirmDeleteDashboard };
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { useCallback } from 'react';
|
||||
import { SolidAlertTriangle, X } from '@signozhq/icons';
|
||||
import { Badge } from '@signozhq/ui/badge';
|
||||
import { Button } from '@signozhq/ui/button';
|
||||
import { DialogWrapper } from '@signozhq/ui/dialog';
|
||||
import { Divider } from '@signozhq/ui/divider';
|
||||
@@ -14,7 +13,6 @@ import DisabledControlTooltip from '../../components/DisabledControlTooltip/Disa
|
||||
import styles from './Header.module.scss';
|
||||
|
||||
interface HeaderProps {
|
||||
/** Unsaved edits exist — shows the "Unsaved Changes" badge and gates the discard confirmation on close (not the Save button). */
|
||||
isDirty: boolean;
|
||||
isSaving: boolean;
|
||||
showSwitchToView?: boolean;
|
||||
@@ -68,11 +66,6 @@ function Header({
|
||||
/>
|
||||
<Divider type="vertical" />
|
||||
<Typography.Text>Configure panel</Typography.Text>
|
||||
{isDirty && (
|
||||
<Badge color="warning" data-testid="panel-editor-v2-unsaved-badge">
|
||||
Unsaved Changes
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
<div className={styles.actions}>
|
||||
<HeaderRightSection
|
||||
@@ -95,7 +88,7 @@ function Header({
|
||||
variant="solid"
|
||||
color="primary"
|
||||
data-testid="panel-editor-v2-save"
|
||||
disabled={readOnly || isSaving}
|
||||
disabled={readOnly || !isDirty || isSaving}
|
||||
loading={!readOnly && isSaving}
|
||||
onClick={readOnly ? undefined : onSave}
|
||||
>
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { ComponentProps } from 'react';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { TooltipProvider } from '@signozhq/ui/tooltip';
|
||||
@@ -24,9 +23,7 @@ jest.mock('api/common/logEvent', () => ({
|
||||
|
||||
const mockUseIsAIAssistantEnabled = useIsAIAssistantEnabled as jest.Mock;
|
||||
|
||||
function renderHeader(
|
||||
props: Partial<ComponentProps<typeof Header>> = {},
|
||||
): void {
|
||||
function renderHeader(): void {
|
||||
// AppLayout supplies the TooltipProvider in the app; the header is rendered bare here.
|
||||
render(
|
||||
<MemoryRouter>
|
||||
@@ -36,7 +33,6 @@ function renderHeader(
|
||||
isSaving={false}
|
||||
onSave={jest.fn()}
|
||||
onClose={jest.fn()}
|
||||
{...props}
|
||||
/>
|
||||
</TooltipProvider>
|
||||
</MemoryRouter>,
|
||||
@@ -70,34 +66,4 @@ describe('PanelEditor Header', () => {
|
||||
).not.toBeInTheDocument();
|
||||
expect(screen.getByTestId('panel-editor-v2-save')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('keeps Save enabled even when there are no unsaved edits', () => {
|
||||
mockUseIsAIAssistantEnabled.mockReturnValue(false);
|
||||
|
||||
renderHeader({ isDirty: false });
|
||||
|
||||
expect(screen.getByTestId('panel-editor-v2-save')).toBeEnabled();
|
||||
});
|
||||
|
||||
it('disables Save only while read-only or saving', () => {
|
||||
mockUseIsAIAssistantEnabled.mockReturnValue(false);
|
||||
|
||||
renderHeader({ isDirty: true, readOnly: true, readOnlyReason: 'Locked' });
|
||||
|
||||
expect(screen.getByTestId('panel-editor-v2-save')).toBeDisabled();
|
||||
});
|
||||
|
||||
it('shows the Unsaved Changes badge only when there are unsaved edits', () => {
|
||||
mockUseIsAIAssistantEnabled.mockReturnValue(false);
|
||||
|
||||
renderHeader({ isDirty: false });
|
||||
expect(
|
||||
screen.queryByTestId('panel-editor-v2-unsaved-badge'),
|
||||
).not.toBeInTheDocument();
|
||||
|
||||
renderHeader({ isDirty: true });
|
||||
expect(
|
||||
screen.getByTestId('panel-editor-v2-unsaved-badge'),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { render, screen, waitFor } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { toast } from '@signozhq/ui/sonner';
|
||||
import type { DashboardtypesPanelDTO } from 'api/generated/services/sigNoz.schemas';
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import { getPanelDefinition } from 'pages/DashboardPageV2/DashboardContainer/Panels/registry';
|
||||
@@ -101,12 +100,6 @@ jest.mock('@signozhq/ui/resizable', () => ({
|
||||
jest.mock('@signozhq/ui/sonner', () => ({
|
||||
toast: { success: jest.fn(), error: jest.fn() },
|
||||
}));
|
||||
const mockShowErrorModal = jest.fn();
|
||||
jest.mock('providers/ErrorModalProvider', () => ({
|
||||
useErrorModal: (): { showErrorModal: jest.Mock } => ({
|
||||
showErrorModal: mockShowErrorModal,
|
||||
}),
|
||||
}));
|
||||
|
||||
// Children mocked to capture props (and expose a Save trigger / footer slot).
|
||||
const mockHeaderProps = jest.fn();
|
||||
@@ -268,13 +261,13 @@ describe('PanelEditorContainer composition', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps a query-less new panel clean but still serializes its seed query', () => {
|
||||
it('keeps a query-less new panel unsaveable but still serializes its seed query', () => {
|
||||
setup(makePanel('signoz/TimeSeriesPanel'), { isNew: true });
|
||||
|
||||
expect(mockUseQuerySync).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ alwaysSerializeQuery: true }),
|
||||
);
|
||||
// No query and no edits yet → not dirty, so closing won't prompt to discard.
|
||||
// No query and no edits yet → nothing to save, so Save stays disabled.
|
||||
expect(mockHeaderProps).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ isDirty: false }),
|
||||
);
|
||||
@@ -297,7 +290,7 @@ describe('PanelEditorContainer composition', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('marks a new panel that already has a query dirty (e.g. list auto-runs one)', () => {
|
||||
it('marks a new panel that already has a query saveable (e.g. list auto-runs one)', () => {
|
||||
const seededQuery = {
|
||||
spec: { plugin: { kind: 'signoz/BuilderQuery', spec: { signal: 'logs' } } },
|
||||
};
|
||||
@@ -334,24 +327,6 @@ describe('PanelEditorContainer composition', () => {
|
||||
expect(mockSave).toHaveBeenCalledWith(panel.spec);
|
||||
});
|
||||
|
||||
it('surfaces a save failure through the error modal', async () => {
|
||||
// The raw thrown value flows straight to the modal, which normalizes it to an
|
||||
// APIError itself (that normalization is covered by ErrorModalProvider's tests).
|
||||
const failure = {
|
||||
response: {
|
||||
status: 400,
|
||||
data: { error: { code: 'INVALID', message: 'Panel name already exists' } },
|
||||
},
|
||||
};
|
||||
mockSave.mockRejectedValueOnce(failure);
|
||||
setup(makePanel('signoz/TimeSeriesPanel'));
|
||||
|
||||
await userEvent.click(screen.getByTestId('editor-save'));
|
||||
|
||||
await waitFor(() => expect(mockShowErrorModal).toHaveBeenCalledWith(failure));
|
||||
expect(toast.error).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('marks the saved panel to be scrolled into view on the dashboard', async () => {
|
||||
setup(makePanel('signoz/TimeSeriesPanel'));
|
||||
|
||||
|
||||
@@ -117,49 +117,6 @@ describe('usePanelEditorQuerySync (real query builder)', () => {
|
||||
},
|
||||
);
|
||||
|
||||
it('a NEW panel stays query-dirty after Stage & Run commits the edited query into the draft', async () => {
|
||||
// Repro of the P0: new panel → edit → Run commits the query into the draft; the
|
||||
// dirty baseline must not drift onto it, or Save re-disables.
|
||||
const editedInUrl: Query = {
|
||||
...initialQueriesMap[DataSource.METRICS],
|
||||
id: 'edited-new-panel',
|
||||
builder: {
|
||||
...initialQueriesMap[DataSource.METRICS].builder,
|
||||
queryData: [
|
||||
{
|
||||
...initialQueriesMap[DataSource.METRICS].builder.queryData[0],
|
||||
legend: 'edited-legend',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const committed = toPerses(editedInUrl, panelType);
|
||||
|
||||
const { result, rerender } = renderHook(
|
||||
({ draftQueries }: { draftQueries: DashboardtypesQueryDTO[] }) =>
|
||||
usePanelEditorQuerySync({
|
||||
draft: makePanel(draftQueries),
|
||||
panelType,
|
||||
setSpec: jest.fn(),
|
||||
refetch: jest.fn(),
|
||||
alwaysSerializeQuery: true,
|
||||
signal: DataSource.METRICS as unknown as TelemetrytypesSignalDTO,
|
||||
// savedQueries omitted — new panel.
|
||||
}),
|
||||
{
|
||||
wrapper: makeUrlWrapper(editedInUrl),
|
||||
initialProps: { draftQueries: [] as DashboardtypesQueryDTO[] },
|
||||
},
|
||||
);
|
||||
|
||||
// The edited query (from the URL) diverges from the seeded default → dirty.
|
||||
await waitFor(() => expect(result.current.isQueryDirty).toBe(true));
|
||||
|
||||
// Stage & Run commits the edited query into the draft → must stay dirty.
|
||||
rerender({ draftQueries: committed });
|
||||
expect(result.current.isQueryDirty).toBe(true);
|
||||
});
|
||||
|
||||
it('an untouched panel with a minimal/older stored query is NOT dirty (drift fix)', async () => {
|
||||
// An older saved query carries only a few fields; the builder re-emits many more
|
||||
// (source, stepInterval, filter, spaceAggregation, …). Comparing raw would read
|
||||
|
||||
@@ -79,11 +79,6 @@ export function usePanelEditorQuerySync({
|
||||
// in-editor edit in the URL survives a refresh / browser Back-Forward.
|
||||
useShareBuilderUrl({ defaultValue: seedQuery });
|
||||
|
||||
// Frozen seed for the new-panel dirty baseline: Stage & Run commits the live query
|
||||
// into `draft.spec.queries`, so re-reading `seedQuery` would drift the baseline onto
|
||||
// the run query and Save would re-disable right after a run.
|
||||
const initialSeedRef = useRef(seedQuery);
|
||||
|
||||
// Commit the live query into the draft (what the preview fetches).
|
||||
const commitQuery = useCallback(
|
||||
(query: Query): boolean => {
|
||||
@@ -157,19 +152,16 @@ export function usePanelEditorQuerySync({
|
||||
// the V5 envelope level. Anchoring to `savedQueries` (not the builder-seed) keeps a
|
||||
// handed-off / URL-restored edit reading as dirty; routing both sides through the same
|
||||
// `fromPerses → toPerses` round-trip stops builder-added defaults (absent from an older
|
||||
// stored query) reading an untouched panel as modified. New panel: fall back to the
|
||||
// frozen initial seed (see `initialSeedRef`).
|
||||
// stored query) reading an untouched panel as modified. New panel: fall back to seed.
|
||||
const baselineEnvelopes = useMemo(
|
||||
() =>
|
||||
toQueryEnvelopes(
|
||||
toPerses(
|
||||
savedQueries
|
||||
? fromPerses(savedQueries, panelType)
|
||||
: initialSeedRef.current,
|
||||
savedQueries ? fromPerses(savedQueries, panelType) : seedQuery,
|
||||
panelType,
|
||||
),
|
||||
),
|
||||
[savedQueries, panelType],
|
||||
[savedQueries, seedQuery, panelType],
|
||||
);
|
||||
const isQueryDirty = useMemo(
|
||||
() =>
|
||||
|
||||
@@ -19,7 +19,6 @@ import {
|
||||
SectionKind,
|
||||
} from 'pages/DashboardPageV2/DashboardContainer/Panels/types/sections';
|
||||
import { getBuilderQueries } from 'pages/DashboardPageV2/DashboardContainer/Panels/utils/getBuilderQueries';
|
||||
import { useErrorModal } from 'providers/ErrorModalProvider';
|
||||
|
||||
import { getExecStats } from '../queryV5/v5ResponseData';
|
||||
import { usePanelInteractions } from '../PanelsAndSectionsLayout/Panel/hooks/usePanelInteractions';
|
||||
@@ -160,10 +159,9 @@ function PanelEditorContainer({
|
||||
return section?.controls;
|
||||
}, [panelDefinition]);
|
||||
|
||||
// Unsaved-edits flag driving the discard confirmation on close (Save is always
|
||||
// enabled). Read the seed `panel`, not the live `draft` — the staged-query sync
|
||||
// commits the seed into the draft on open, which would falsely dirty an untouched
|
||||
// query-less new panel.
|
||||
// New panels are savable once seeded with a query (List auto-seeds one). Read the
|
||||
// seed `panel`, not the live `draft` — the staged-query sync commits the seed into
|
||||
// the draft on open, which would falsely dirty an untouched query-less new panel.
|
||||
const isDirty = useMemo(
|
||||
() => isSpecDirty || isQueryDirty || (isNew && panel.spec.queries.length > 0),
|
||||
[isSpecDirty, isQueryDirty, isNew, panel.spec.queries.length],
|
||||
@@ -228,7 +226,6 @@ function PanelEditorContainer({
|
||||
});
|
||||
|
||||
const setScrollTargetId = useScrollIntoViewStore((s) => s.setScrollTargetId);
|
||||
const { showErrorModal } = useErrorModal();
|
||||
|
||||
const onSave = useCallback(async (): Promise<void> => {
|
||||
if (!isEditable) {
|
||||
@@ -239,22 +236,12 @@ function PanelEditorContainer({
|
||||
const savedPanelId = await save(buildSaveSpec(draft.spec));
|
||||
// Reveal the saved panel once the dashboard re-renders.
|
||||
setScrollTargetId(savedPanelId);
|
||||
toast.success('Panel saved', {
|
||||
position: 'top-center',
|
||||
});
|
||||
toast.success('Panel saved');
|
||||
onSaved();
|
||||
} catch (err) {
|
||||
showErrorModal(err);
|
||||
} catch {
|
||||
toast.error('Failed to save panel');
|
||||
}
|
||||
}, [
|
||||
isEditable,
|
||||
save,
|
||||
buildSaveSpec,
|
||||
draft.spec,
|
||||
setScrollTargetId,
|
||||
onSaved,
|
||||
showErrorModal,
|
||||
]);
|
||||
}, [isEditable, save, buildSaveSpec, draft.spec, setScrollTargetId, onSaved]);
|
||||
|
||||
// Leaving an existing panel's editor (without saving) still returns to it, so
|
||||
// the dashboard lands on that panel rather than scrolled to the top. A new,
|
||||
|
||||
@@ -51,7 +51,7 @@ describe('panelStatusFromError', () => {
|
||||
|
||||
it('falls back to the error message when there is no structured body', () => {
|
||||
expect(panelStatusFromError(new Error('boom'))).toStrictEqual({
|
||||
code: 'UPSTREAM_UNAVAILABLE',
|
||||
code: 'unknown_error',
|
||||
message: 'boom',
|
||||
docsUrl: undefined,
|
||||
messages: [],
|
||||
|
||||
@@ -12,13 +12,13 @@ import {
|
||||
deleteDashboardV2,
|
||||
invalidateListDashboardsForUserV2,
|
||||
} from 'api/generated/services/dashboard';
|
||||
import { useDeleteConfirm } from 'components/DeleteConfirmModal/useDeleteConfirm';
|
||||
import { DashboardListEvents } from 'pages/DashboardsListPageV2/constants/events';
|
||||
import { useAppContext } from 'providers/App/App';
|
||||
import { useErrorModal } from 'providers/ErrorModalProvider';
|
||||
import APIError from 'types/api/error';
|
||||
import { USER_ROLES } from 'types/roles';
|
||||
|
||||
import { useDeleteConfirm } from '../DeleteConfirmModal/useDeleteConfirm';
|
||||
import styles from './ActionsPopover.module.scss';
|
||||
|
||||
interface Props {
|
||||
|
||||
@@ -6,11 +6,11 @@ import { Typography } from '@signozhq/ui/typography';
|
||||
import { Bookmark, PenLine, Plus, Search, Trash2 } from '@signozhq/icons';
|
||||
import cx from 'classnames';
|
||||
|
||||
import { useDeleteConfirm } from 'components/DeleteConfirmModal/useDeleteConfirm';
|
||||
import { DashboardListEvents } from 'pages/DashboardsListPageV2/constants/events';
|
||||
|
||||
import type { SavedView } from '../../types';
|
||||
import { type BuiltinView } from '../../utils/views';
|
||||
import { useDeleteConfirm } from '../DeleteConfirmModal/useDeleteConfirm';
|
||||
import ViewNamePopover from './ViewNamePopover';
|
||||
|
||||
import styles from './ViewsRail.module.scss';
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { FieldDataType } from 'types/api/v5/queryRange';
|
||||
import { QUERY_BUILDER_KEY_TYPES } from 'constants/antlrQueryConstants';
|
||||
|
||||
export interface QueryKeyDataSuggestionsProps {
|
||||
label: string;
|
||||
@@ -7,12 +7,7 @@ export interface QueryKeyDataSuggestionsProps {
|
||||
apply?: string;
|
||||
detail?: string;
|
||||
fieldContext?: 'resource' | 'scope' | 'attribute' | 'span';
|
||||
/**
|
||||
* The field's type as the API reports it. Was declared as the antlr key-type enum
|
||||
* (`string | number | boolean`), which the endpoint never returns — every consumer cast it
|
||||
* back to `FieldDataType`.
|
||||
*/
|
||||
fieldDataType?: FieldDataType;
|
||||
fieldDataType?: QUERY_BUILDER_KEY_TYPES;
|
||||
name: string;
|
||||
signal: 'traces' | 'logs' | 'metrics';
|
||||
}
|
||||
@@ -31,7 +26,7 @@ export interface QueryKeyRequestProps {
|
||||
signal: 'traces' | 'logs' | 'metrics';
|
||||
searchText: string;
|
||||
fieldContext?: 'resource' | 'scope' | 'attribute' | 'span';
|
||||
fieldDataType?: FieldDataType;
|
||||
fieldDataType?: QUERY_BUILDER_KEY_TYPES;
|
||||
metricName?: string;
|
||||
metricNamespace?: string;
|
||||
signalSource?: 'meter' | '';
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import type { FieldDataType } from 'types/api/v5/queryRange';
|
||||
|
||||
import { fieldDataTypeToDataType } from '../fieldDataType';
|
||||
|
||||
describe('fieldDataTypeToDataType', () => {
|
||||
it('maps the reported numeric spellings onto the query-builder numerics', () => {
|
||||
expect(fieldDataTypeToDataType('number')).toBe(DataTypes.Float64);
|
||||
expect(fieldDataTypeToDataType('int64')).toBe(DataTypes.Int64);
|
||||
expect(fieldDataTypeToDataType('float64')).toBe(DataTypes.Float64);
|
||||
});
|
||||
|
||||
it('maps the list spellings onto the array types', () => {
|
||||
expect(fieldDataTypeToDataType('[]string')).toBe(DataTypes.ArrayString);
|
||||
expect(fieldDataTypeToDataType('[]bool')).toBe(DataTypes.ArrayBool);
|
||||
expect(fieldDataTypeToDataType('[]int64')).toBe(DataTypes.ArrayInt64);
|
||||
expect(fieldDataTypeToDataType('[]float64')).toBe(DataTypes.ArrayFloat64);
|
||||
expect(fieldDataTypeToDataType('[]number')).toBe(DataTypes.ArrayFloat64);
|
||||
});
|
||||
|
||||
it('passes through the spellings the two vocabularies share', () => {
|
||||
expect(fieldDataTypeToDataType('string')).toBe(DataTypes.String);
|
||||
expect(fieldDataTypeToDataType('bool')).toBe(DataTypes.bool);
|
||||
});
|
||||
|
||||
it('returns EMPTY for empty, missing and not-yet-known types', () => {
|
||||
expect(fieldDataTypeToDataType('')).toBe(DataTypes.EMPTY);
|
||||
expect(fieldDataTypeToDataType(undefined)).toBe(DataTypes.EMPTY);
|
||||
// The backend can store `[]json` / `[]dynamic`, which the API union doesn't list.
|
||||
expect(fieldDataTypeToDataType('[]json' as FieldDataType)).toBe(
|
||||
DataTypes.EMPTY,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -42,12 +42,7 @@ export function toAPIError(
|
||||
try {
|
||||
ErrorResponseHandlerForGeneratedAPIs(error);
|
||||
} catch (apiError) {
|
||||
// UPSTREAM_UNAVAILABLE means the handler couldn't find a backend error code
|
||||
// (non-envelope body); prefer the caller's context-specific defaultMessage.
|
||||
if (
|
||||
apiError instanceof APIError &&
|
||||
apiError.getErrorCode() !== 'UPSTREAM_UNAVAILABLE'
|
||||
) {
|
||||
if (apiError instanceof APIError) {
|
||||
return apiError;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
|
||||
import type { FieldDataType } from 'types/api/v5/queryRange';
|
||||
|
||||
/**
|
||||
* The field metadata APIs and the query builder spell field types differently: the metadata
|
||||
* collapses the numerics to `number` and writes lists as `[]string`, while `DataTypes` — which
|
||||
* keys the operator and attribute-value lookups — uses `float64` and `array(string)`. Convert
|
||||
* where a reported type becomes a query-builder one, so those lookups can't miss.
|
||||
*/
|
||||
const TO_DATA_TYPE: Record<FieldDataType, DataTypes> = {
|
||||
'': DataTypes.EMPTY,
|
||||
string: DataTypes.String,
|
||||
bool: DataTypes.bool,
|
||||
int64: DataTypes.Int64,
|
||||
float64: DataTypes.Float64,
|
||||
number: DataTypes.Float64,
|
||||
'[]string': DataTypes.ArrayString,
|
||||
'[]bool': DataTypes.ArrayBool,
|
||||
'[]int64': DataTypes.ArrayInt64,
|
||||
'[]float64': DataTypes.ArrayFloat64,
|
||||
'[]number': DataTypes.ArrayFloat64,
|
||||
};
|
||||
|
||||
/**
|
||||
* Exhaustive over `FieldDataType`, so a type added to the API union fails to compile here
|
||||
* rather than going missing at runtime. The fallback is for a spelling the union doesn't know
|
||||
* yet — the backend can store `[]json` and `[]dynamic` — not for any of the cases above.
|
||||
*/
|
||||
export const fieldDataTypeToDataType = (
|
||||
fieldDataType?: FieldDataType,
|
||||
): DataTypes => TO_DATA_TYPE[fieldDataType ?? ''] ?? DataTypes.EMPTY;
|
||||
2
go.mod
2
go.mod
@@ -4,7 +4,7 @@ go 1.25.7
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2
|
||||
github.com/AfterShip/clickhouse-sql-parser v0.5.2
|
||||
github.com/AfterShip/clickhouse-sql-parser v0.4.16
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.44.0
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.2
|
||||
github.com/SigNoz/clickhouse-go-mock v0.14.0
|
||||
|
||||
4
go.sum
4
go.sum
@@ -66,8 +66,8 @@ dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/AfterShip/clickhouse-sql-parser v0.5.2 h1:KCxdmvuVawVF4yl0ZS33q7olgmLZwvZG5BjWHp6o414=
|
||||
github.com/AfterShip/clickhouse-sql-parser v0.5.2/go.mod h1:Qi3qvPTfZb/aFwI5V4WFOahgjsLJa4MzVijIAfwOhDw=
|
||||
github.com/AfterShip/clickhouse-sql-parser v0.4.16 h1:gpl+wXclYUKT0p4+gBq22XeRYWwEoZ9f35vogqMvkLQ=
|
||||
github.com/AfterShip/clickhouse-sql-parser v0.4.16/go.mod h1:W0Z82wJWkJxz2RVun/RMwxue3g7ut47Xxl+SFqdJGus=
|
||||
github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
|
||||
|
||||
@@ -14,8 +14,6 @@ var (
|
||||
FeatureEnableAIObservability = featuretypes.MustNewName("enable_ai_observability")
|
||||
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
|
||||
FeatureUseInfraMonitoringV2 = featuretypes.MustNewName("use_infra_monitoring_v2")
|
||||
|
||||
FeatureUsePrometheusClickhouseV2 = featuretypes.MustNewName("use_prometheus_clickhouse_v2")
|
||||
)
|
||||
|
||||
func MustNewRegistry() featuretypes.Registry {
|
||||
@@ -108,14 +106,6 @@ func MustNewRegistry() featuretypes.Registry {
|
||||
DefaultVariant: featuretypes.MustNewName("disabled"),
|
||||
Variants: featuretypes.NewBooleanVariants(),
|
||||
},
|
||||
&featuretypes.Feature{
|
||||
Name: FeatureUsePrometheusClickhouseV2,
|
||||
Kind: featuretypes.KindBoolean,
|
||||
Stage: featuretypes.StageExperimental,
|
||||
Description: "Runs PromQL queries on the clickhousev2 provider alongside the served engine result and logs any difference; serving is unaffected.",
|
||||
DefaultVariant: featuretypes.MustNewName("disabled"),
|
||||
Variants: featuretypes.NewBooleanVariants(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/util/annotations"
|
||||
)
|
||||
|
||||
// statementRecorder collects the statements a PromQL evaluation would run.
|
||||
// Safe for concurrent use: the engine may Select selectors concurrently.
|
||||
type statementRecorder struct {
|
||||
mu sync.Mutex
|
||||
statements []prometheus.CapturedStatement
|
||||
}
|
||||
|
||||
func (r *statementRecorder) record(query string, args []any) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.statements = append(r.statements, prometheus.CapturedStatement{Query: query, Args: args})
|
||||
}
|
||||
|
||||
func (r *statementRecorder) Statements() []prometheus.CapturedStatement {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
out := make([]prometheus.CapturedStatement, len(r.statements))
|
||||
copy(out, r.statements)
|
||||
return out
|
||||
}
|
||||
|
||||
type captureQueryable struct {
|
||||
client *client
|
||||
recorder *statementRecorder
|
||||
}
|
||||
|
||||
func (c *captureQueryable) Querier(mint, maxt int64) (storage.Querier, error) {
|
||||
return &captureQuerier{
|
||||
querier: querier{mint: mint, maxt: maxt, client: c.client},
|
||||
recorder: c.recorder,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// captureQuerier builds the same SQL as the live querier but records it and
|
||||
// returns no data. The fingerprint filter always takes the subquery form:
|
||||
// without executing the series lookup, the inline literal set is unknown.
|
||||
type captureQuerier struct {
|
||||
querier
|
||||
recorder *statementRecorder
|
||||
}
|
||||
|
||||
func (c *captureQuerier) Select(ctx context.Context, _ bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
|
||||
|
||||
start, end := c.window(hints)
|
||||
|
||||
samplesQuery, args, err := buildSamplesQuery(start, end, metricNamesFromMatchers(matchers), matchers, c.lastSamplePerStepFor(ctx, hints))
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
c.recorder.record(samplesQuery, args)
|
||||
|
||||
return storage.EmptySeriesSet()
|
||||
}
|
||||
|
||||
func (c *captureQuerier) LabelValues(context.Context, string, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
func (c *captureQuerier) LabelNames(context.Context, *storage.LabelHints, ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
// metricNamesFromMatchers extracts the statically known metric name, if any.
|
||||
// The live path derives names from the matched series; the capture path has
|
||||
// no execution results, so only a __name__ equality contributes.
|
||||
func metricNamesFromMatchers(matchers []*labels.Matcher) []string {
|
||||
for _, m := range matchers {
|
||||
if m.Name == metricNameLabel && m.Type == labels.MatchEqual && m.Value != "" {
|
||||
return []string{m.Value}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,119 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var updateGolden = flag.Bool("update", false, "rewrite the classification golden file")
|
||||
|
||||
const goldenFile = "testdata/classification_golden.json"
|
||||
|
||||
// corpusFile is the conformance corpus the integration suite replays; the
|
||||
// golden freezes how the classifier routes every one of its expressions.
|
||||
const corpusFile = "../../../tests/integration/testdata/promqltestcorpus/corpus.json"
|
||||
|
||||
type goldenEntry struct {
|
||||
Expr string `json:"expr"`
|
||||
StartMs int64 `json:"start_ms"`
|
||||
EndMs int64 `json:"end_ms"`
|
||||
StepMs int64 `json:"step_ms"`
|
||||
// Plan is the routing decision: "full" (whole query in ClickHouse),
|
||||
// "hybrid" (units substituted, engine on top), "fallback" (engine over
|
||||
// the native querier).
|
||||
Plan string `json:"plan"`
|
||||
// Units is the substituted-unit count for hybrid plans.
|
||||
Units int `json:"units,omitempty"`
|
||||
// Reason is the coarse fallback bucket (fallbackShape).
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// TestClassificationGolden freezes the classifier's routing decision for
|
||||
// every (expression, grid) of the conformance corpus. Routing is a
|
||||
// correctness surface of its own: a change that silently sends rate() to the
|
||||
// engine path costs the pushdown, and one that silently starts transpiling a
|
||||
// shape never proven equivalent risks wrong numbers — both must show up in
|
||||
// review as a diff of this file, with the corpus suite's clickhousev2 leg
|
||||
// judging whether the new routing still returns the reference answers.
|
||||
//
|
||||
// Regenerate after intentional classifier changes:
|
||||
//
|
||||
// go test ./pkg/prometheus/clickhouseprometheusv2 -run TestClassificationGolden -update
|
||||
func TestClassificationGolden(t *testing.T) {
|
||||
raw, err := os.ReadFile(corpusFile)
|
||||
require.NoError(t, err)
|
||||
|
||||
var corpus struct {
|
||||
Cases []struct {
|
||||
Expr string `json:"expr"`
|
||||
StartMs int64 `json:"start_ms"`
|
||||
EndMs int64 `json:"end_ms"`
|
||||
StepMs int64 `json:"step_ms"`
|
||||
} `json:"cases"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(raw, &corpus))
|
||||
require.NotEmpty(t, corpus.Cases)
|
||||
|
||||
promParser := parser.NewParser(parser.Options{})
|
||||
seen := map[goldenEntry]bool{}
|
||||
var entries []goldenEntry
|
||||
for _, c := range corpus.Cases {
|
||||
key := goldenEntry{Expr: c.Expr, StartMs: c.StartMs, EndMs: c.EndMs, StepMs: c.StepMs}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
|
||||
expr, err := promParser.ParseExpr(c.Expr)
|
||||
require.NoError(t, err, "corpus expression must parse: %q", c.Expr)
|
||||
|
||||
entry := key
|
||||
plan, ok := classify(expr, gridContext{startMs: c.StartMs, endMs: c.EndMs, stepMs: c.StepMs})
|
||||
switch {
|
||||
case ok && plan.full:
|
||||
entry.Plan = "full"
|
||||
case ok:
|
||||
entry.Plan = "hybrid"
|
||||
entry.Units = len(plan.units)
|
||||
default:
|
||||
entry.Plan = "fallback"
|
||||
entry.Reason = fallbackShape(expr)
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
a, b := entries[i], entries[j]
|
||||
if a.Expr != b.Expr {
|
||||
return a.Expr < b.Expr
|
||||
}
|
||||
if a.StartMs != b.StartMs {
|
||||
return a.StartMs < b.StartMs
|
||||
}
|
||||
if a.EndMs != b.EndMs {
|
||||
return a.EndMs < b.EndMs
|
||||
}
|
||||
return a.StepMs < b.StepMs
|
||||
})
|
||||
|
||||
got, err := json.MarshalIndent(entries, "", " ")
|
||||
require.NoError(t, err)
|
||||
got = append(got, '\n')
|
||||
|
||||
if *updateGolden {
|
||||
require.NoError(t, os.MkdirAll(filepath.Dir(goldenFile), 0o755))
|
||||
require.NoError(t, os.WriteFile(goldenFile, got, 0o644))
|
||||
return
|
||||
}
|
||||
|
||||
want, err := os.ReadFile(goldenFile)
|
||||
require.NoError(t, err, "golden missing — generate it with -update")
|
||||
require.Equal(t, string(want), string(got),
|
||||
"classification routing changed; if intentional, regenerate with -update and justify the diff in review")
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"math"
|
||||
"slices"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
promValue "github.com/prometheus/prometheus/model/value"
|
||||
)
|
||||
|
||||
// seriesLookup holds a series lookup's result: matched fingerprints with
|
||||
// their labels, and the distinct metric names seen on them.
|
||||
type seriesLookup struct {
|
||||
fingerprints map[uint64]labels.Labels
|
||||
metricNames []string
|
||||
}
|
||||
|
||||
// client executes the series, samples and raw queries against ClickHouse.
|
||||
type client struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
lookbackMs int64
|
||||
}
|
||||
|
||||
func newClient(settings factory.ScopedProviderSettings, telemetryStore telemetrystore.TelemetryStore, cfg prometheus.Config) *client {
|
||||
lookback := cfg.LookbackDelta
|
||||
if lookback <= 0 {
|
||||
// Mirror the engine: promql defaults an unset lookback to 5m.
|
||||
lookback = defaultLookbackDelta
|
||||
}
|
||||
return &client{
|
||||
settings: settings,
|
||||
telemetryStore: telemetryStore,
|
||||
lookbackMs: lookback.Milliseconds(),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *client) withContext(ctx context.Context, functionName string) context.Context {
|
||||
return ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(),
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-prometheus-v2",
|
||||
instrumentationtypes.CodeFunctionName: functionName,
|
||||
})
|
||||
}
|
||||
|
||||
func (c *client) selectSeries(ctx context.Context, query string, args []any) (*seriesLookup, error) {
|
||||
ctx = c.withContext(ctx, "selectSeries")
|
||||
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
lookup := &seriesLookup{fingerprints: make(map[uint64]labels.Labels)}
|
||||
names := make(map[string]struct{})
|
||||
|
||||
var fingerprint uint64
|
||||
var labelsJSON string
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(&fingerprint, &labelsJSON); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lset, err := unmarshalLabels(labelsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lookup.fingerprints[fingerprint] = lset
|
||||
if name := lset.Get(metricNameLabel); name != "" {
|
||||
names[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for name := range names {
|
||||
lookup.metricNames = append(lookup.metricNames, name)
|
||||
}
|
||||
slices.Sort(lookup.metricNames)
|
||||
|
||||
return lookup, nil
|
||||
}
|
||||
|
||||
// unmarshalLabels parses the labels JSON column, dropping empty-valued
|
||||
// labels: empty means "absent" in Prometheus, but stored attribute JSON can
|
||||
// carry them.
|
||||
func unmarshalLabels(s string) (labels.Labels, error) {
|
||||
m := make(map[string]string)
|
||||
if err := json.Unmarshal([]byte(s), &m); err != nil {
|
||||
return labels.EmptyLabels(), err
|
||||
}
|
||||
builder := labels.NewScratchBuilder(len(m))
|
||||
for k, v := range m {
|
||||
if v == "" {
|
||||
continue
|
||||
}
|
||||
builder.Add(k, v)
|
||||
}
|
||||
builder.Sort()
|
||||
return builder.Labels(), nil
|
||||
}
|
||||
|
||||
// selectSamples assembles per-series sample slices from a samples query (raw
|
||||
// or last-sample-per-step; same column shape), whose rows arrive ordered by
|
||||
// (fingerprint, unix_milli). Fingerprints missing from the lookup are skipped:
|
||||
// the samples query's semi-join re-runs the series predicates and can match
|
||||
// series registered after the lookup ran — the lookup is the read snapshot.
|
||||
// Stale flags map to the engine's StaleNaN. Duplicate
|
||||
// timestamps pass through: uniqueness is ingest's job, and v1 feeds them to
|
||||
// the engine as-is over the same dirty data.
|
||||
func (c *client) selectSamples(ctx context.Context, query string, args []any, lookup *seriesLookup) ([]*series, error) {
|
||||
ctx = c.withContext(ctx, "selectSamples")
|
||||
rows, err := c.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var (
|
||||
result []*series
|
||||
current *series
|
||||
fingerprint uint64
|
||||
prevFp uint64
|
||||
timestampMs int64
|
||||
val float64
|
||||
flags uint32
|
||||
first = true
|
||||
haveCurrent bool
|
||||
staleMarker = math.Float64frombits(promValue.StaleNaN)
|
||||
unknownCount int
|
||||
)
|
||||
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(&fingerprint, ×tampMs, &val, &flags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if first || fingerprint != prevFp {
|
||||
first = false
|
||||
prevFp = fingerprint
|
||||
lset, ok := lookup.fingerprints[fingerprint]
|
||||
if !ok {
|
||||
unknownCount++
|
||||
haveCurrent = false
|
||||
continue
|
||||
}
|
||||
current = &series{lset: lset}
|
||||
result = append(result, current)
|
||||
haveCurrent = true
|
||||
}
|
||||
if !haveCurrent {
|
||||
continue
|
||||
}
|
||||
|
||||
if flags&1 == 1 {
|
||||
val = staleMarker
|
||||
}
|
||||
current.ts = append(current.ts, timestampMs)
|
||||
current.vs = append(current.vs, val)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if unknownCount > 0 {
|
||||
c.settings.Logger().DebugContext(ctx, "skipped samples of fingerprints missing from series lookup",
|
||||
slog.Int("unknown_fingerprints", unknownCount))
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -1,337 +0,0 @@
|
||||
// Package clickhouseprometheusv2 is the second-generation ClickHouse-backed
|
||||
// Prometheus provider. It exists because the v1 provider fetches every raw
|
||||
// sample of a query's union window through the remote-read protobuf layer
|
||||
// and hands it to the engine — the cost is a function of ingested data, not
|
||||
// of the question asked, which is how a dashboard of PromQL panels takes an
|
||||
// instance down.
|
||||
//
|
||||
// Every query runs in one of two ways, decided per query:
|
||||
//
|
||||
// - Transpiled: the query is evaluated entirely inside ClickHouse and only
|
||||
// final (or near-final) per-group grid arrays come back, built on the
|
||||
// timeSeries*ToGrid aggregate functions (the supported ClickHouse floor
|
||||
// is >= 25.6, so they are assumed available).
|
||||
// - Engine: the stock promql engine evaluates over this package's native
|
||||
// storage.Querier. This is the path for everything not transpilable.
|
||||
//
|
||||
// Correctness is the constraint that shaped both paths: a PromQL result that
|
||||
// differs from upstream Prometheus is a lost user, so anything that cannot
|
||||
// reproduce engine semantics exactly falls back rather than approximate.
|
||||
// The rest of this comment is the PromQL -> SQL story, because that mapping
|
||||
// is where correctness is won or lost.
|
||||
//
|
||||
// # The evaluation model the SQL must reproduce
|
||||
//
|
||||
// A PromQL range query is an instant query evaluated at every grid point
|
||||
// t_i = start + i*step, i = 0..(end-start)/step. At each t_i:
|
||||
//
|
||||
// - an instant selector resolves to the latest sample in the left-open
|
||||
// lookback window (t_i - lookback, t_i], and to nothing when that latest
|
||||
// sample is a stale marker — even if older real samples sit inside the
|
||||
// window;
|
||||
// - a range selector [r] collects every sample in (t_i - r, t_i], stale
|
||||
// markers excluded;
|
||||
// - offset d shifts both windows to (t_i - d - w, t_i - d].
|
||||
//
|
||||
// The transpilation invariant follows from this: every transpiled construct
|
||||
// produces, per output series, one array with exactly one slot per grid
|
||||
// point — slot i holds the value at t_i, NULL means absent. This is what
|
||||
// makes composition correct, not just convenient: the engine evaluates
|
||||
// these operators independently per t_i, so any representation that gets
|
||||
// every slot right gets the whole query right, and spatial aggregation over
|
||||
// arrays is sound because it combines values that belong to the same t_i by
|
||||
// construction. Slot index i maps back to t_i = start + i*step at scan time
|
||||
// (toMatrix). Everything below is about filling those slots with exactly
|
||||
// the numbers the engine would compute — and each equivalence was validated
|
||||
// against the vendored engine on live data before its shape entered the
|
||||
// allowlist; anything unproven stays on the engine path.
|
||||
//
|
||||
// # Classification: finding what a statement can answer
|
||||
//
|
||||
// classify walks the parsed AST looking for "core units" — maximal subtrees
|
||||
// of the shape
|
||||
//
|
||||
// [agg by/without (...)] [fn(] selector[range] [offset d] [)] [op scalar]...
|
||||
//
|
||||
// classifyCore peels that chain from the outside in: an optional
|
||||
// sum/min/max/avg/count aggregation, then one of the allowlisted functions
|
||||
// or a bare instant selector, then the selector with its offset; on the way
|
||||
// out it accumulates number-literal arithmetic, comparisons (including
|
||||
// bool) and unary minus into a scalar-op pipeline. A node qualifies only if
|
||||
// its type, arguments and children are in the proven set — an allowlist, so
|
||||
// an overlooked construct becomes a fallback instead of a wrong number.
|
||||
//
|
||||
// Three unit kinds come out of this, each with its own SQL form:
|
||||
// unitRange (rate, irate, increase, delta, idelta over a range selector),
|
||||
// unitInstant (instant vector selection, bare or comparison-filtered) and
|
||||
// unitOverTime (avg/min/max/sum/count/last _over_time).
|
||||
//
|
||||
// If the entire tree is one unit, the plan is "full": the statement's rows
|
||||
// are the query result. Otherwise every maximal unit is cut out and replaced
|
||||
// in the expression with a synthetic selector __signoz_transpiled_N__, and
|
||||
// the rewritten expression runs in the engine over the units' materialized
|
||||
// results ("hybrid") — histogram_quantile, topk, or/and/unless and vector
|
||||
// matching keep exact engine semantics while their expensive inputs were
|
||||
// aggregated server-side.
|
||||
//
|
||||
// Classification refuses when exact semantics cannot be guaranteed
|
||||
// server-side: the @ modifier anywhere and default-resolution subqueries
|
||||
// (their resolution is a server runtime setting the transpiler cannot see);
|
||||
// duration expressions (offset step(), [range()], ...) anywhere — they are
|
||||
// resolved into the selector's static fields only at evaluation time, so at
|
||||
// classification time those fields still hold their zero values and
|
||||
// transpiling would silently use the wrong offset or range;
|
||||
// steps or ranges that are not whole seconds (the grid functions take
|
||||
// whole-second parameters); grouping by or matching on __name__ in hybrid
|
||||
// plans (the synthetic name would leak into results); name-keeping units —
|
||||
// bare/comparison instant selectors and last_over_time keep their real
|
||||
// __name__ (keepsName), which substitution would replace, so they transpile
|
||||
// only as full plans; and every function outside the allowlist (changes,
|
||||
// resets, quantile_over_time, absent, native-histogram functions, ...).
|
||||
//
|
||||
// Units inside a fixed-resolution subquery evaluate on the subquery's own
|
||||
// grid instead of the query grid: epoch-aligned multiples of the resolution
|
||||
// strictly after outerStart - offset - range, ending at outer end - offset —
|
||||
// the exact derivation the engine uses, because a grid shifted by one step
|
||||
// changes which samples every window sees.
|
||||
//
|
||||
// # From one unit to one statement
|
||||
//
|
||||
// buildUnitSQL renders each unit as a single statement. For
|
||||
// sum by (pod) (rate(m{job="api"}[5m])) the skeleton is:
|
||||
//
|
||||
// SELECT gkey, sumForEach(grid) AS grid FROM (
|
||||
// SELECT series.gkey AS gkey,
|
||||
// timeSeriesRateToGrid(<start>, <end>, <step>, <range>)(fromUnixTimestamp64Milli(unix_milli), value) AS grid
|
||||
// FROM signoz_metrics.distributed_samples_v4 AS points
|
||||
// INNER JOIN (
|
||||
// SELECT fingerprint, <group key expr> AS gkey
|
||||
// FROM signoz_metrics.time_series_v4
|
||||
// WHERE <series predicates>
|
||||
// GROUP BY fingerprint, gkey
|
||||
// ) AS series ON points.fingerprint = series.fingerprint
|
||||
// WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified']
|
||||
// AND points.fingerprint IN (<matched fingerprints>)
|
||||
// AND unix_milli > <start - range> AND unix_milli <= <end>
|
||||
// AND bitAnd(flags, 1) = 0
|
||||
// GROUP BY points.fingerprint, series.gkey
|
||||
// ) GROUP BY gkey
|
||||
// SETTINGS allow_experimental_ts_to_grid_aggregate_function = 1
|
||||
//
|
||||
// Reading it inside out:
|
||||
//
|
||||
// The time window is the selector's semantics verbatim: strict > on the
|
||||
// lower bound and <= on the upper is the left-open (t - w, t] rule, with the
|
||||
// whole window shifted by the offset. bitAnd(flags, 1) = 0 drops stale
|
||||
// markers, which PromQL excludes from range vectors.
|
||||
//
|
||||
// The inner GROUP BY computes one grid array per series.
|
||||
// timeSeriesRateToGrid(start, end, step, range) is a parametric aggregate:
|
||||
// fed (timestamp, value) pairs it produces Array(Nullable(Float64)) with one
|
||||
// slot per grid point. Correct because it implements the engine's
|
||||
// extrapolatedRate decision for decision — counter resets, the zero-point
|
||||
// clamp, the extrapolation thresholds, the >= 2 samples rule, the left-open
|
||||
// window — verified by feeding identical samples to both and comparing
|
||||
// slot for slot: the only difference ever observed is the last bit
|
||||
// (ClickHouse's C++ and Go round the same formula differently), which is
|
||||
// the floating-point floor, not a semantic gap. irate/delta/idelta map to
|
||||
// their own timeSeries*ToGrid functions with the same verification;
|
||||
// increase has no function of its own and is emitted as
|
||||
// arrayMap(x -> x * <range seconds>, <rate expr>), exact by definition —
|
||||
// extrapolatedRate computes the same extrapolated delta for both and
|
||||
// divides by the range only when isRate, so multiplying it back is the
|
||||
// identity, not an approximation. The grid parameters are rendered as
|
||||
// literals, not bound args — they are aggregate-function parameters — and
|
||||
// the experimental gate rides as a SETTINGS clause on the statement itself
|
||||
// so telemetrystore hooks cannot clobber it.
|
||||
//
|
||||
// The join annotates each series with its group key, in one of two forms.
|
||||
// by (...) extracts each listed label as a plain column
|
||||
// (JSONExtractString(labels, 'pod') AS g0) and groups on the columns
|
||||
// directly: the projection is a known short list and the label names live
|
||||
// in Go, so building, sorting and stringifying every label pair per row
|
||||
// would be waste. Correct because column-tuple equality is label-set
|
||||
// equality on the projection, and an extracted '' is the label being
|
||||
// absent — Prometheus semantics for by() over missing labels, and empties
|
||||
// are skipped when the columns turn back into labels. without and
|
||||
// no-aggregation project a label SET that varies per series, so they get
|
||||
// the canonical key: toJSONString of the sorted [label, value] pairs the
|
||||
// unit projects (without excludes the listed labels plus __name__; no
|
||||
// aggregation keeps everything, the name coming off in Go per the engine's
|
||||
// name-dropping rules). There the sort is load-bearing — stored JSON key
|
||||
// order is not canonical across fingerprints, and two orderings of the same
|
||||
// labels must land in one group — empty values are filtered for the same
|
||||
// absent-label reason, and the same string parses back into the output
|
||||
// label set (labelsFromGroupKey).
|
||||
//
|
||||
// The outer GROUP BY is the spatial aggregation: sum/min/max/avg/count
|
||||
// by/without become the -ForEach combinators. Element-wise aggregation over
|
||||
// grid arrays is the engine's per-t_i aggregation, because slot i of every
|
||||
// input array refers to the same t_i; the combinators skip NULLs, which is
|
||||
// the engine aggregating only the series present at t_i, and an index where
|
||||
// every series is absent stays NULL. Two edges need explicit handling:
|
||||
// countForEach wraps in a mapping of 0 back to NULL, because a count over
|
||||
// an all-absent index is an absent point, not 0; and a unit without
|
||||
// aggregation still passes through maxForEach — the identity for the common
|
||||
// one-fingerprint group, and a deterministic NULL-skipping merge when a
|
||||
// regex __name__ selector collapses distinct metrics onto one projected
|
||||
// label set. One caveat is inherent: summation order over series differs
|
||||
// from the engine's, so spatial aggregates can differ in the last ULP —
|
||||
// float addition is not associative; no ordering reproduces the engine's
|
||||
// bit-exactly from inside a GROUP BY.
|
||||
//
|
||||
// # Instant selectors: staleness needs two aggregates
|
||||
//
|
||||
// unitInstant uses window = lookback and must reproduce the shadowing rule:
|
||||
// the point is absent when the latest in-window sample is a stale marker.
|
||||
// timeSeriesLastToGrid alone cannot express that — skipping stale rows in
|
||||
// WHERE would resurrect the older real sample the marker was written to
|
||||
// bury. So stale rows stay in the scan for this kind only, and the grid
|
||||
// expression compares three aggregates per slot:
|
||||
//
|
||||
// arrayMap((tall, tok, vok) -> if(tall IS NULL OR tok IS NULL OR tall != tok, NULL, vok),
|
||||
// timeSeriesLastToGrid(...)(ts, toFloat64(unix_milli)), -- last sample overall
|
||||
// timeSeriesLastToGridIf(...)(ts, toFloat64(unix_milli), bitAnd(flags, 1) = 0), -- last non-stale, its timestamp
|
||||
// timeSeriesLastToGridIf(...)(ts, value, bitAnd(flags, 1) = 0)) -- last non-stale, its value
|
||||
//
|
||||
// Correct by cases on a slot's window. No samples at all: both timestamp
|
||||
// aggregates are NULL, the slot is NULL — absent, as the engine says. Latest
|
||||
// sample non-stale: it is the latest overall and the latest non-stale, the
|
||||
// timestamps agree, the slot takes its value — the engine's pick. Latest
|
||||
// sample stale: the last-overall timestamp is the marker's, the
|
||||
// last-non-stale timestamp is older (or NULL when only markers are in
|
||||
// window), they disagree, the slot is NULL — the marker shadows, exactly
|
||||
// the engine's rule. Timestamps are unique per series (ingest dedups), so
|
||||
// timestamp equality identifies "the same sample" without ambiguity. The
|
||||
// -If combinator's applicability to these experimental aggregates was
|
||||
// probed before being trusted, not assumed.
|
||||
//
|
||||
// # Windowed *_over_time: whole buckets instead of a grid function
|
||||
//
|
||||
// avg/min/max/sum/count _over_time aggregate every raw sample in the window,
|
||||
// and no timeSeries*ToGrid function computes them. (last_over_time is the
|
||||
// exception: the last sample of a range vector — stale markers excluded from
|
||||
// range vectors by PromQL, excluded here in WHERE — is exactly
|
||||
// timeSeriesLastToGrid.) These transpile only when the range is a whole
|
||||
// multiple of the step, and then the window needs no per-sample fan-out at
|
||||
// all: with W = range/step, the window (t_k - range, t_k] is exactly the
|
||||
// union of W step buckets — both are left-open on the same boundaries — so
|
||||
// bucket membership fully determines window membership. Each sample lands
|
||||
// in exactly one bucket by a plain GROUP BY:
|
||||
//
|
||||
// intDiv(unix_milli - <start> + <range> - 1, <step>) AS jj
|
||||
//
|
||||
// (ceil((ts - start)/step) shifted by W-1 so the earliest in-window sample
|
||||
// sits at 0; slot k's window is buckets jj in [k, k+W-1]). The alternative —
|
||||
// fanning each sample into all W windows that cover it — multiplies rows by
|
||||
// W, which for a long range over a short step is a row explosion measured
|
||||
// in billions; the bucketed form's row count is series x buckets, the size
|
||||
// of the output, regardless of W.
|
||||
//
|
||||
// The shard level aggregates per (series, group key, bucket): a bucket
|
||||
// count plus the function's value aggregate (sum for sum/avg, min, max).
|
||||
// The assembly level places the partials into dense arrays
|
||||
// (groupArrayInsertAt — positions are unique, one row per bucket; counts
|
||||
// and sums default to 0, which contributes nothing) and slides: slot k
|
||||
// combines its at-most-W bucket partials by direct aggregation, so window
|
||||
// sums are added the way the engine adds them — no prefix-sum differencing,
|
||||
// whose large-minus-large cancellation would drift past the shadow
|
||||
// tolerance on counter-sized values. Correct per slot because the bucket
|
||||
// union is the exact window multiset and avg/min/max/sum/count are
|
||||
// order-insensitive on a multiset (sum/avg up to summation order, the float
|
||||
// caveat above). A slot with zero window count is absent; min/max filter
|
||||
// their slices on the bucket counts, so an empty bucket's default can never
|
||||
// be mistaken for a value — a real sample can legitimately be +Inf.
|
||||
//
|
||||
// Ranges that don't divide the step, and windows wider than
|
||||
// maxWindowBuckets buckets (the slide costs W combines per slot), fall back
|
||||
// to the engine path, which is exact.
|
||||
//
|
||||
// # Scalar ops, full plans, hybrid plans
|
||||
//
|
||||
// The scalar-op pipeline applies in Go to the returned arrays
|
||||
// (applyScalarOps), slot by slot: arithmetic operators compute, comparisons
|
||||
// filter (the slot keeps the vector-side value or becomes NULL) or return
|
||||
// 0/1 under bool. Correct trivially: it is the same float64 operation the
|
||||
// engine would apply to the same slot value, in the same operator order the
|
||||
// AST dictates — running it in Go instead of another SQL layer changes
|
||||
// where, not what.
|
||||
//
|
||||
// A full plan's arrays map straight to the result matrix. A hybrid plan
|
||||
// materializes each unit's arrays as synthetic series under its
|
||||
// __signoz_transpiled_N__ name and evaluates the rewritten expression over
|
||||
// a storage that serves synthetic names from memory and everything else
|
||||
// live. Substitution is sound because a unit's output is a plain instant
|
||||
// vector to the engine — same values at same timestamps under a different
|
||||
// name, and the name cannot matter: plans that group by or match on
|
||||
// __name__ were refused at classification, and name-keeping units are never
|
||||
// substituted. One subtlety makes it exact: stale markers are written at
|
||||
// absent grid points, because the engine's lookback would otherwise
|
||||
// resurrect a point from up to lookback earlier — the marker encodes
|
||||
// "absent here" the way the engine itself encodes it. Units evaluate
|
||||
// concurrently; each is one series lookup plus one grid statement. A step
|
||||
// of 0 is an instant query: a single evaluation at end.
|
||||
//
|
||||
// # Series lookup
|
||||
//
|
||||
// Both paths resolve matchers the same way, once per selector
|
||||
// (selectSeries), against the series tables holding one row per
|
||||
// (fingerprint, bucket) at 1h/6h/1d/1w granularities; timeSeriesTableFor
|
||||
// picks the table whose bucket fits the window and rounds the window start
|
||||
// down to the bucket boundary. How matchers become SQL, and why regexes are
|
||||
// anchored, is documented at applySeriesConditions. Empty-valued labels come
|
||||
// off at this boundary: an empty value means "label absent" in Prometheus,
|
||||
// but stored attribute JSON can carry them.
|
||||
//
|
||||
// # The engine path
|
||||
//
|
||||
// Queries that do not transpile run in the stock engine over this package's
|
||||
// storage.Querier, which is still not the v1 path. Samples are fetched per
|
||||
// selector using the engine's per-selector hints, not the query-wide union
|
||||
// window, so foo / foo offset 1d reads two narrow windows instead of the
|
||||
// widest one twice. Instant selectors of subquery-free queries fetch only
|
||||
// the last sample per step bucket (lastSamplePerStep): buckets anchor at the
|
||||
// selector's first evaluation timestamp — recovered from the hints as
|
||||
// hints.Start + lookback - 1ms, the inverse of how the engine derives
|
||||
// hints.Start — so bucket boundaries coincide with evaluation timestamps and
|
||||
// a non-final sample of a bucket can never be the latest sample in
|
||||
// (t - lookback, t] for any grid t. Real timestamps are preserved, so the
|
||||
// engine's own lookback and staleness handling stay exact. Range selectors
|
||||
// always fetch raw — every sample feeds the range function — and the
|
||||
// subquery-free proof travels in the context as prometheus.QueryTraits,
|
||||
// because subquery selectors evaluate at the subquery's step while the
|
||||
// hints carry the top-level step. Row assembly maps stale flags to the
|
||||
// engine's StaleNaN and merges series with identical label sets
|
||||
// (sortAndMerge) — the engine assumes storages never emit duplicates.
|
||||
//
|
||||
// # Sharding
|
||||
//
|
||||
// samples_v4 and time_series_v4 (and all their rollups) shard on the same
|
||||
// key — cityHash64(env, temporality, metric_name, fingerprint) — so a
|
||||
// series' samples and catalog rows live on the same shard. The transpiled
|
||||
// statement above exploits that: the distributed samples table at the
|
||||
// top-level FROM makes ClickHouse rewrite the whole inner query per shard,
|
||||
// where the join against the shard-local series table and the per-series
|
||||
// grid aggregation run next to the data; the initiator only merges
|
||||
// aggregate states and applies the spatial -ForEach step. Same layout as
|
||||
// the telemetrymetrics statement builder. The group-key join alone
|
||||
// restricts the transpiled scan to the matched series; the engine path's
|
||||
// samples fetch restricts by the same predicates as a shard-local
|
||||
// semi-join, not a GLOBAL broadcast of the matched set. The temporality
|
||||
// filter on every
|
||||
// samples statement is a semantic no-op — the matched fingerprints already
|
||||
// come from those temporalities — that engages the leading samples
|
||||
// primary-key column. Delta-temporality series stay invisible to PromQL
|
||||
// here exactly as they are in v1: the rollout gate is parity with v1, and
|
||||
// making Delta visible is its own change with its own semantics to design —
|
||||
// a Delta stream fed to rate() as-if-cumulative would be wrong, not just
|
||||
// new.
|
||||
//
|
||||
// # Observability
|
||||
//
|
||||
// Every statement carries a log_comment with
|
||||
// code.namespace=clickhouse-prometheus-v2 and code.function.name naming the
|
||||
// call site (selectSeries, selectSamples, transpiledUnit, LabelValues,
|
||||
// LabelNames), so this provider's work is attributable in system.query_log
|
||||
// without guessing from query text.
|
||||
package clickhouseprometheusv2
|
||||
@@ -1,84 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
)
|
||||
|
||||
// Provider ties the package together: its own engine and parser, the
|
||||
// ClickHouse client behind the native storage.Querier, and the transpiler
|
||||
// executor. See the package documentation for what runs where and why. It is
|
||||
// exported as a concrete type — pkg/querier holds it directly for shadow
|
||||
// comparison and pinned serving, and an interface with a single
|
||||
// implementation would only hide that dependency.
|
||||
type Provider struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
engine *prometheus.Engine
|
||||
parser prometheus.Parser
|
||||
client *client
|
||||
executor *executor
|
||||
}
|
||||
|
||||
var (
|
||||
_ prometheus.Prometheus = (*Provider)(nil)
|
||||
_ prometheus.StatementCapturer = (*Provider)(nil)
|
||||
)
|
||||
|
||||
func NewFactory(telemetryStore telemetrystore.TelemetryStore) factory.ProviderFactory[prometheus.Prometheus, prometheus.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("clickhousev2"), func(ctx context.Context, providerSettings factory.ProviderSettings, config prometheus.Config) (prometheus.Prometheus, error) {
|
||||
return New(ctx, providerSettings, config, telemetryStore)
|
||||
})
|
||||
}
|
||||
|
||||
func New(_ context.Context, providerSettings factory.ProviderSettings, config prometheus.Config, telemetryStore telemetrystore.TelemetryStore) (*Provider, error) {
|
||||
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2")
|
||||
|
||||
engine := prometheus.NewEngine(settings.Logger(), config)
|
||||
parser := prometheus.NewParser()
|
||||
client := newClient(settings, telemetryStore, config)
|
||||
|
||||
return &Provider{
|
||||
settings: settings,
|
||||
engine: engine,
|
||||
parser: parser,
|
||||
client: client,
|
||||
executor: &executor{client: client, engine: engine, parser: parser},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TryExecuteRange evaluates transpilable query shapes directly in ClickHouse
|
||||
// (see transpiler.go). ok=false means the shape is not transpilable and the
|
||||
// caller should evaluate through Engine over Storage instead.
|
||||
func (p *Provider) TryExecuteRange(ctx context.Context, query string, start, end time.Time, step time.Duration) (promql.Matrix, bool, error) {
|
||||
return p.executor.TryExecuteRange(ctx, query, start, end, step)
|
||||
}
|
||||
|
||||
func (p *Provider) Engine() *prometheus.Engine {
|
||||
return p.engine
|
||||
}
|
||||
|
||||
func (p *Provider) Parser() prometheus.Parser {
|
||||
return p.parser
|
||||
}
|
||||
|
||||
func (p *Provider) Storage() storage.Queryable {
|
||||
return p
|
||||
}
|
||||
|
||||
func (p *Provider) Querier(mint, maxt int64) (storage.Querier, error) {
|
||||
return &querier{mint: mint, maxt: maxt, client: p.client}, nil
|
||||
}
|
||||
|
||||
// CapturingStorage implements prometheus.StatementCapturer: a storage that
|
||||
// records each selector's SQL without executing it, for the preview path.
|
||||
// A fresh recorder per call keeps concurrent dry-runs isolated.
|
||||
func (p *Provider) CapturingStorage() (storage.Queryable, prometheus.StatementRecorder) {
|
||||
recorder := &statementRecorder{}
|
||||
return &captureQueryable{client: p.client, recorder: recorder}, recorder
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/util/annotations"
|
||||
)
|
||||
|
||||
// defaultLookbackDelta mirrors promql's default when the config leaves the
|
||||
// lookback unset; the engine and the storage must agree on it for
|
||||
// last-sample-per-step bucket anchoring.
|
||||
const defaultLookbackDelta = 5 * time.Minute
|
||||
|
||||
// querier is a native storage.Querier over ClickHouse: Select builds SQL
|
||||
// directly from the matchers and hints, with no remote-read protobuf layer.
|
||||
type querier struct {
|
||||
mint, maxt int64
|
||||
client *client
|
||||
}
|
||||
|
||||
var _ storage.Querier = (*querier)(nil)
|
||||
|
||||
func (q *querier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
|
||||
start, end := q.window(hints)
|
||||
|
||||
seriesQuery, seriesArgs, err := buildSeriesQuery(start, end, matchers)
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
lookup, err := q.client.selectSeries(ctx, seriesQuery, seriesArgs)
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
if len(lookup.fingerprints) == 0 {
|
||||
return storage.EmptySeriesSet()
|
||||
}
|
||||
|
||||
list, err := q.fetchSamples(ctx, start, end, matchers, lookup, q.lastSamplePerStepFor(ctx, hints))
|
||||
if err != nil {
|
||||
return storage.ErrSeriesSet(err)
|
||||
}
|
||||
|
||||
// The engine assumes storages never emit duplicate label sets.
|
||||
list = sortAndMerge(list)
|
||||
return newSeriesSet(list)
|
||||
}
|
||||
|
||||
func (q *querier) LabelValues(ctx context.Context, name string, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
if name == metricNameLabel {
|
||||
sb.Select("DISTINCT metric_name AS value")
|
||||
} else {
|
||||
sb.Select(fmt.Sprintf("DISTINCT JSONExtractString(labels, %s) AS value", sb.Var(name)))
|
||||
}
|
||||
adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
sb.Where("value != ''")
|
||||
if hints != nil && hints.Limit > 0 {
|
||||
sb.Limit(hints.Limit)
|
||||
}
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
values, err := q.selectStrings(ctx, "LabelValues", query, args)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
slices.Sort(values)
|
||||
return values, nil, nil
|
||||
}
|
||||
|
||||
func (q *querier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("DISTINCT arrayJoin(JSONExtractKeys(labels)) AS name")
|
||||
adjustedStart, table := timeSeriesTableFor(q.mint, q.maxt)
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sb, adjustedStart, q.maxt, matchers); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if hints != nil && hints.Limit > 0 {
|
||||
sb.Limit(hints.Limit)
|
||||
}
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
names, err := q.selectStrings(ctx, "LabelNames", query, args)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
slices.Sort(names)
|
||||
return names, nil, nil
|
||||
}
|
||||
|
||||
func (q *querier) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// window returns the per-selector fetch window from the hints (already
|
||||
// adjusted for offset, @, range and lookback); mint/maxt span the union of
|
||||
// all selectors and are the fallback.
|
||||
func (q *querier) window(hints *storage.SelectHints) (int64, int64) {
|
||||
if hints != nil && hints.Start != 0 && hints.End != 0 && hints.Start <= hints.End {
|
||||
return hints.Start, hints.End
|
||||
}
|
||||
return q.mint, q.maxt
|
||||
}
|
||||
|
||||
// lastSamplePerStepFor decides whether the fetch can keep only the last
|
||||
// sample per step bucket (see lastSamplePerStep). Only instant selectors
|
||||
// (hints.Range == 0) of subquery-free queries qualify: range selectors need
|
||||
// every raw sample, and subquery selectors evaluate at the subquery's own
|
||||
// step while hints carry the top-level step — the subquery-free proof
|
||||
// arrives as QueryTraits in the context. The first evaluation timestamp is
|
||||
// recovered as hints.Start + lookback - 1ms, inverting how the engine
|
||||
// derives hints.Start.
|
||||
func (q *querier) lastSamplePerStepFor(ctx context.Context, hints *storage.SelectHints) *lastSamplePerStep {
|
||||
if hints == nil || hints.Range != 0 || hints.Start <= 0 {
|
||||
return nil
|
||||
}
|
||||
traits, ok := prometheus.QueryTraitsFromContext(ctx)
|
||||
if !ok || !traits.SubqueryFree {
|
||||
return nil
|
||||
}
|
||||
firstEval := hints.Start + q.client.lookbackMs - 1
|
||||
if firstEval > hints.End {
|
||||
// Defensive: never anchor a bucket past the window.
|
||||
firstEval = hints.End
|
||||
}
|
||||
return &lastSamplePerStep{firstEvalMs: firstEval, stepMs: hints.Step}
|
||||
}
|
||||
|
||||
// fetchSamples runs the samples query for the matched series (see
|
||||
// buildSamplesQuery).
|
||||
func (q *querier) fetchSamples(ctx context.Context, start, end int64, matchers []*labels.Matcher, lookup *seriesLookup, lastPerStep *lastSamplePerStep) ([]*series, error) {
|
||||
query, args, err := buildSamplesQuery(start, end, lookup.metricNames, matchers, lastPerStep)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q.client.selectSamples(ctx, query, args, lookup)
|
||||
}
|
||||
|
||||
func (q *querier) selectStrings(ctx context.Context, fn, query string, args []any) ([]string, error) {
|
||||
ctx = q.client.withContext(ctx, fn)
|
||||
rows, err := q.client.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []string
|
||||
var v string
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -1,184 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/prometheus/prometheus/model/histogram"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/util/annotations"
|
||||
)
|
||||
|
||||
// series is one time series with samples as parallel slices ordered by
|
||||
// timestamp. Deliberately not storage.NewListSeries: that boxes every sample
|
||||
// as an interface value, a per-sample allocation this fetch path exists to
|
||||
// avoid.
|
||||
type series struct {
|
||||
lset labels.Labels
|
||||
ts []int64
|
||||
vs []float64
|
||||
}
|
||||
|
||||
var _ storage.Series = (*series)(nil)
|
||||
|
||||
func (s *series) Labels() labels.Labels {
|
||||
return s.lset
|
||||
}
|
||||
|
||||
func (s *series) Iterator(it chunkenc.Iterator) chunkenc.Iterator {
|
||||
if fit, ok := it.(*floatIterator); ok {
|
||||
fit.reset(s)
|
||||
return fit
|
||||
}
|
||||
fit := &floatIterator{}
|
||||
fit.reset(s)
|
||||
return fit
|
||||
}
|
||||
|
||||
// floatIterator implements chunkenc.Iterator over a series' sample slices.
|
||||
type floatIterator struct {
|
||||
s *series
|
||||
i int
|
||||
}
|
||||
|
||||
var _ chunkenc.Iterator = (*floatIterator)(nil)
|
||||
|
||||
func (it *floatIterator) reset(s *series) {
|
||||
it.s = s
|
||||
it.i = -1
|
||||
}
|
||||
|
||||
func (it *floatIterator) Next() chunkenc.ValueType {
|
||||
it.i++
|
||||
if it.i >= len(it.s.ts) {
|
||||
return chunkenc.ValNone
|
||||
}
|
||||
return chunkenc.ValFloat
|
||||
}
|
||||
|
||||
func (it *floatIterator) Seek(t int64) chunkenc.ValueType { //nolint:govet // stdmethods flags io.Seeker; this is chunkenc.Iterator's Seek
|
||||
if it.i < 0 {
|
||||
it.i = 0
|
||||
}
|
||||
if it.i >= len(it.s.ts) {
|
||||
return chunkenc.ValNone
|
||||
}
|
||||
// The current position, once valid, must not move backwards.
|
||||
if it.s.ts[it.i] >= t {
|
||||
return chunkenc.ValFloat
|
||||
}
|
||||
it.i += sort.Search(len(it.s.ts)-it.i, func(j int) bool {
|
||||
return it.s.ts[it.i+j] >= t
|
||||
})
|
||||
if it.i >= len(it.s.ts) {
|
||||
return chunkenc.ValNone
|
||||
}
|
||||
return chunkenc.ValFloat
|
||||
}
|
||||
|
||||
func (it *floatIterator) At() (int64, float64) {
|
||||
return it.s.ts[it.i], it.s.vs[it.i]
|
||||
}
|
||||
|
||||
func (it *floatIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (it *floatIterator) AtFloatHistogram(*histogram.FloatHistogram) (int64, *histogram.FloatHistogram) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func (it *floatIterator) AtT() int64 {
|
||||
return it.s.ts[it.i]
|
||||
}
|
||||
|
||||
// AtST returns the current start timestamp; not tracked by this storage.
|
||||
func (it *floatIterator) AtST() int64 {
|
||||
return 0
|
||||
}
|
||||
|
||||
func (it *floatIterator) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// seriesSet iterates a fully materialized, label-sorted list of series.
|
||||
type seriesSet struct {
|
||||
series []*series
|
||||
i int
|
||||
}
|
||||
|
||||
var _ storage.SeriesSet = (*seriesSet)(nil)
|
||||
|
||||
func newSeriesSet(list []*series) *seriesSet {
|
||||
return &seriesSet{series: list, i: -1}
|
||||
}
|
||||
|
||||
func (s *seriesSet) Next() bool {
|
||||
s.i++
|
||||
return s.i < len(s.series)
|
||||
}
|
||||
|
||||
func (s *seriesSet) At() storage.Series {
|
||||
return s.series[s.i]
|
||||
}
|
||||
|
||||
func (s *seriesSet) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *seriesSet) Warnings() annotations.Annotations {
|
||||
return nil
|
||||
}
|
||||
|
||||
// sortAndMerge orders series by label set and merges identical label sets
|
||||
// by timestamp (first sample wins ties): distinct fingerprints can carry
|
||||
// identical label sets, and the engine assumes storages never emit
|
||||
// duplicates.
|
||||
func sortAndMerge(list []*series) []*series {
|
||||
if len(list) < 2 {
|
||||
return list
|
||||
}
|
||||
sort.Slice(list, func(i, j int) bool {
|
||||
return labels.Compare(list[i].lset, list[j].lset) < 0
|
||||
})
|
||||
out := list[:1]
|
||||
for _, s := range list[1:] {
|
||||
last := out[len(out)-1]
|
||||
if labels.Compare(last.lset, s.lset) != 0 {
|
||||
out = append(out, s)
|
||||
continue
|
||||
}
|
||||
merged := mergeSamples(last, s)
|
||||
out[len(out)-1] = merged
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeSamples(a, b *series) *series {
|
||||
ts := make([]int64, 0, len(a.ts)+len(b.ts))
|
||||
vs := make([]float64, 0, len(a.ts)+len(b.ts))
|
||||
i, j := 0, 0
|
||||
for i < len(a.ts) && j < len(b.ts) {
|
||||
switch {
|
||||
case a.ts[i] < b.ts[j]:
|
||||
ts = append(ts, a.ts[i])
|
||||
vs = append(vs, a.vs[i])
|
||||
i++
|
||||
case a.ts[i] > b.ts[j]:
|
||||
ts = append(ts, b.ts[j])
|
||||
vs = append(vs, b.vs[j])
|
||||
j++
|
||||
default:
|
||||
ts = append(ts, a.ts[i])
|
||||
vs = append(vs, a.vs[i])
|
||||
i++
|
||||
j++
|
||||
}
|
||||
}
|
||||
ts = append(ts, a.ts[i:]...)
|
||||
vs = append(vs, a.vs[i:]...)
|
||||
ts = append(ts, b.ts[j:]...)
|
||||
vs = append(vs, b.vs[j:]...)
|
||||
return &series{lset: a.lset, ts: ts, vs: vs}
|
||||
}
|
||||
@@ -1,164 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
)
|
||||
|
||||
// buildSeriesQuery renders the series lookup: one row per matched fingerprint
|
||||
// with its labels.
|
||||
func buildSeriesQuery(start, end int64, matchers []*labels.Matcher) (string, []any, error) {
|
||||
adjustedStart, table := timeSeriesTableFor(start, end)
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("fingerprint", "any(labels)")
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sb, adjustedStart, end, matchers); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
sb.GroupBy("fingerprint")
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, nil
|
||||
}
|
||||
|
||||
// buildSamplesQuery renders the samples fetch for the matched series: a
|
||||
// semi-join re-runs the series predicates against the shard-local series
|
||||
// table (complete by fingerprint co-locality, see localTimeSeriesTable — a
|
||||
// GLOBAL broadcast would ship the matched set to every shard, and ClickHouse
|
||||
// materializes the subquery's set per shard before the scan, so it still
|
||||
// engages the fingerprint primary-key column). metricNames (observed on the
|
||||
// matched series when the selector had no __name__ equality) narrows the
|
||||
// primary-key scan. A non-nil lastPerStep groups to the last sample per
|
||||
// step bucket.
|
||||
func buildSamplesQuery(start, end int64, metricNames []string, matchers []*labels.Matcher, lastPerStep *lastSamplePerStep) (string, []any, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
if lastPerStep != nil {
|
||||
// Aliases must not shadow source columns: ClickHouse resolves aliases
|
||||
// in WHERE too, and "max(unix_milli) AS unix_milli" would put an
|
||||
// aggregate into the WHERE clause (error 184).
|
||||
sb.Select("fingerprint", "max(unix_milli) AS ts", "argMax(value, unix_milli) AS val", "argMax(flags, unix_milli) AS fl")
|
||||
} else {
|
||||
sb.Select("fingerprint", "unix_milli", "value", "flags")
|
||||
}
|
||||
sb.From(fmt.Sprintf("%s.%s", databaseName, distributedSamplesV4))
|
||||
|
||||
switch len(metricNames) {
|
||||
case 0:
|
||||
// No name constraint derivable; the primary-key prefix goes unused.
|
||||
case 1:
|
||||
sb.Where(sb.EQ("metric_name", metricNames[0]))
|
||||
default:
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
}
|
||||
// Semantically redundant (the fingerprints already come from these
|
||||
// temporalities) but engages the leading primary-key column.
|
||||
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
|
||||
sub := sqlbuilder.NewSelectBuilder()
|
||||
sub.Select("fingerprint")
|
||||
adjustedStart, table := timeSeriesTableFor(start, end)
|
||||
sub.From(fmt.Sprintf("%s.%s", databaseName, localTimeSeriesTable(table)))
|
||||
if err := applySeriesConditions(sub, adjustedStart, end, matchers); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
sb.Where(sb.In("fingerprint", sub))
|
||||
sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
|
||||
|
||||
if lastPerStep != nil {
|
||||
sb.GroupBy("fingerprint")
|
||||
if expr := lastPerStep.bucketExpr(); expr != "" {
|
||||
sb.GroupBy(expr)
|
||||
}
|
||||
sb.OrderBy("fingerprint", "ts")
|
||||
} else {
|
||||
sb.OrderBy("fingerprint", "unix_milli")
|
||||
}
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, nil
|
||||
}
|
||||
|
||||
// applySeriesConditions adds the WHERE conditions of a series table scan:
|
||||
// __name__ matchers (all four types) translate to the metric_name column,
|
||||
// every other matcher to a JSONExtractString condition on the labels column.
|
||||
// An equality matcher against "" matches series without the label, mirroring
|
||||
// PromQL, because JSONExtractString returns "" for missing keys. Regexes are
|
||||
// anchored: PromQL matchers match the whole value, ClickHouse match()
|
||||
// searches for a substring.
|
||||
func applySeriesConditions(sb *sqlbuilder.SelectBuilder, start, end int64, matchers []*labels.Matcher) error {
|
||||
for _, m := range matchers {
|
||||
if m.Name != metricNameLabel {
|
||||
continue
|
||||
}
|
||||
switch m.Type {
|
||||
case labels.MatchEqual:
|
||||
sb.Where(sb.EQ("metric_name", m.Value))
|
||||
case labels.MatchNotEqual:
|
||||
sb.Where(sb.NE("metric_name", m.Value))
|
||||
case labels.MatchRegexp:
|
||||
sb.Where(fmt.Sprintf("match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
|
||||
case labels.MatchNotRegexp:
|
||||
sb.Where(fmt.Sprintf("NOT match(metric_name, %s)", sb.Var(anchorRegex(m.Value))))
|
||||
default:
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q for __name__", m.Type)
|
||||
}
|
||||
}
|
||||
|
||||
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
|
||||
// Inclusive upper bound: registration rows are hour-floored (and 6h/1d/1w
|
||||
// for the rollup tables) by the exporter, so a series first registered in
|
||||
// the bucket starting exactly at `end` would otherwise be invisible while
|
||||
// its samples (<= end) are in range.
|
||||
sb.Where(sb.GTE("unix_milli", start), sb.LTE("unix_milli", end))
|
||||
|
||||
for _, m := range matchers {
|
||||
if m.Name == metricNameLabel {
|
||||
continue
|
||||
}
|
||||
switch m.Type {
|
||||
case labels.MatchEqual:
|
||||
sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) = %s", sb.Var(m.Name), sb.Var(m.Value)))
|
||||
case labels.MatchNotEqual:
|
||||
sb.Where(fmt.Sprintf("JSONExtractString(labels, %s) != %s", sb.Var(m.Name), sb.Var(m.Value)))
|
||||
case labels.MatchRegexp:
|
||||
sb.Where(fmt.Sprintf("match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
|
||||
case labels.MatchNotRegexp:
|
||||
sb.Where(fmt.Sprintf("NOT match(JSONExtractString(labels, %s), %s)", sb.Var(m.Name), sb.Var(anchorRegex(m.Value))))
|
||||
default:
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported matcher type %q", m.Type)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// anchorRegex turns a PromQL regex into its fully-anchored form (see
|
||||
// applySeriesConditions).
|
||||
func anchorRegex(v string) string {
|
||||
return "^(?:" + v + ")$"
|
||||
}
|
||||
|
||||
// lastSamplePerStep reduces an instant-selector fetch to the last sample of
|
||||
// each step bucket: bucket 0 is (start, firstEval], bucket i is
|
||||
// (firstEval+(i-1)·step, firstEval+i·step]. Anchoring buckets at the first
|
||||
// evaluation timestamp makes every bucket boundary an evaluation timestamp,
|
||||
// so a non-final sample of a bucket can never be the latest sample in any
|
||||
// (t-lookback, t] the engine resolves — the reduction is lossless. Real
|
||||
// timestamps are preserved, so the engine's own lookback and staleness
|
||||
// handling remain exact.
|
||||
type lastSamplePerStep struct {
|
||||
firstEvalMs int64
|
||||
stepMs int64
|
||||
}
|
||||
|
||||
func (t *lastSamplePerStep) bucketExpr() string {
|
||||
if t.stepMs <= 0 {
|
||||
// Instant query: a single evaluation at firstEval; one bucket.
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"if(unix_milli <= %d, 0, intDiv(unix_milli - %d - 1, %d) + 1)",
|
||||
t.firstEvalMs, t.firstEvalMs, t.stepMs,
|
||||
)
|
||||
}
|
||||
@@ -1,139 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func mustMatcher(t *testing.T, mt labels.MatchType, name, value string) *labels.Matcher {
|
||||
t.Helper()
|
||||
m, err := labels.NewMatcher(mt, name, value)
|
||||
require.NoError(t, err)
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTimeSeriesTableFor(t *testing.T) {
|
||||
base := time.Date(2026, 7, 10, 3, 27, 0, 0, time.UTC).UnixMilli()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
span time.Duration
|
||||
wantTable string
|
||||
roundTo time.Duration
|
||||
}{
|
||||
{"under 6h uses hourly table", 2 * time.Hour, distributedTimeSeriesV4, time.Hour},
|
||||
{"under 1d uses 6h table", 12 * time.Hour, distributedTimeSeriesV46hrs, 6 * time.Hour},
|
||||
{"under 1w uses 1d table", 3 * 24 * time.Hour, distributedTimeSeriesV41day, 24 * time.Hour},
|
||||
{"over 1w uses 1w table", 10 * 24 * time.Hour, distributedTimeSeriesV41week, 7 * 24 * time.Hour},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
start, table := timeSeriesTableFor(base, base+tt.span.Milliseconds())
|
||||
assert.Equal(t, tt.wantTable, table)
|
||||
assert.Zero(t, start%tt.roundTo.Milliseconds())
|
||||
assert.LessOrEqual(t, start, base)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSeriesQuery(t *testing.T) {
|
||||
start := int64(1_700_000_000_000)
|
||||
end := start + time.Hour.Milliseconds()
|
||||
// The series table window rounds down to the table's bucket boundary.
|
||||
adjustedStart := start - (start % time.Hour.Milliseconds())
|
||||
|
||||
t.Run("equality name and label matchers", func(t *testing.T) {
|
||||
query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total"),
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t,
|
||||
"SELECT fingerprint, any(labels) FROM signoz_metrics.distributed_time_series_v4 WHERE metric_name = ? AND temporality IN ['Cumulative', 'Unspecified'] AND unix_milli >= ? AND unix_milli <= ? AND JSONExtractString(labels, ?) = ? GROUP BY fingerprint",
|
||||
query,
|
||||
)
|
||||
assert.Equal(t, []any{"http_requests_total", adjustedStart, end, "job", "api"}, args)
|
||||
})
|
||||
|
||||
t.Run("regex matchers are anchored", func(t *testing.T) {
|
||||
_, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "up"),
|
||||
mustMatcher(t, labels.MatchRegexp, "instance", "prod.*"),
|
||||
mustMatcher(t, labels.MatchNotRegexp, "env", "dev|test"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []any{"up", adjustedStart, end, "instance", "^(?:prod.*)$", "env", "^(?:dev|test)$"}, args)
|
||||
})
|
||||
|
||||
t.Run("regex name matcher uses metric_name column", func(t *testing.T) {
|
||||
query, args, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchRegexp, "__name__", "node_cpu.*|node_memory.*"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "match(metric_name, ?)")
|
||||
assert.NotContains(t, query, "JSONExtractString")
|
||||
assert.Equal(t, []any{"^(?:node_cpu.*|node_memory.*)$", adjustedStart, end}, args)
|
||||
})
|
||||
|
||||
t.Run("no name matcher omits metric_name condition", func(t *testing.T) {
|
||||
query, _, err := buildSeriesQuery(start, end, []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, query, "metric_name")
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildSamplesQuery(t *testing.T) {
|
||||
start := int64(1_700_000_000_000)
|
||||
end := start + time.Hour.Milliseconds()
|
||||
adjustedStart := start - (start % time.Hour.Milliseconds())
|
||||
matchers := []*labels.Matcher{
|
||||
mustMatcher(t, labels.MatchEqual, "__name__", "up"),
|
||||
mustMatcher(t, labels.MatchEqual, "job", "api"),
|
||||
}
|
||||
|
||||
t.Run("raw fetch filters by a shard-local semi-join", func(t *testing.T) {
|
||||
query, args, err := buildSamplesQuery(start, end, []string{"up"}, matchers, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "fingerprint IN (SELECT fingerprint FROM signoz_metrics.time_series_v4 WHERE ")
|
||||
assert.NotContains(t, query, "GLOBAL IN")
|
||||
assert.Contains(t, query, "ORDER BY fingerprint, unix_milli")
|
||||
// Args follow placeholder order: samples metric name, the semi-join's
|
||||
// series predicates, then the samples window bounds.
|
||||
assert.Equal(t, []any{"up", "up", adjustedStart, end, "job", "api", start, end}, args)
|
||||
})
|
||||
|
||||
t.Run("last-sample-per-step groups by step bucket anchored at first eval", func(t *testing.T) {
|
||||
lastPerStep := &lastSamplePerStep{firstEvalMs: start + 299_999, stepMs: 60_000}
|
||||
query, _, err := buildSamplesQuery(start, end, []string{"up"}, matchers, lastPerStep)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "argMax(value, unix_milli) AS val")
|
||||
assert.Contains(t, query, "argMax(flags, unix_milli) AS fl")
|
||||
assert.Contains(t, query, "GROUP BY fingerprint, if(unix_milli <= 1700000299999, 0, intDiv(unix_milli - 1700000299999 - 1, 60000) + 1)")
|
||||
assert.Contains(t, query, "ORDER BY fingerprint, ts")
|
||||
// Aliases must not shadow the source columns referenced in WHERE.
|
||||
assert.NotContains(t, query, "AS unix_milli")
|
||||
assert.NotContains(t, query, "AS value")
|
||||
assert.NotContains(t, query, "AS flags")
|
||||
})
|
||||
|
||||
t.Run("instant query keeps one bucket", func(t *testing.T) {
|
||||
lastPerStep := &lastSamplePerStep{firstEvalMs: end, stepMs: 0}
|
||||
query, _, err := buildSamplesQuery(start, end, []string{"up"}, matchers, lastPerStep)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "GROUP BY fingerprint ORDER BY fingerprint, ts")
|
||||
assert.NotContains(t, query, "intDiv")
|
||||
})
|
||||
|
||||
t.Run("multiple metric names from regex selector", func(t *testing.T) {
|
||||
query, args, err := buildSamplesQuery(start, end, []string{"node_cpu", "node_memory"}, matchers, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, query, "metric_name IN (?, ?)")
|
||||
assert.Equal(t, []any{"node_cpu", "node_memory", "up", adjustedStart, end, "job", "api", start, end}, args)
|
||||
})
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import "time"
|
||||
|
||||
// TODO(srikanthccv): consolidate the metrics table names and the
|
||||
// window-to-table selection across the query builders (telemetrymetrics)
|
||||
// and the prometheus providers instead of each package carrying its own
|
||||
// copy.
|
||||
|
||||
const (
|
||||
// metricNameLabel is the reserved PromQL label holding the metric name.
|
||||
metricNameLabel string = "__name__"
|
||||
|
||||
databaseName string = "signoz_metrics"
|
||||
distributedTimeSeriesV4 string = "distributed_time_series_v4"
|
||||
distributedTimeSeriesV46hrs string = "distributed_time_series_v4_6hrs"
|
||||
distributedTimeSeriesV41day string = "distributed_time_series_v4_1day"
|
||||
distributedTimeSeriesV41week string = "distributed_time_series_v4_1week"
|
||||
distributedSamplesV4 string = "distributed_samples_v4"
|
||||
|
||||
localTimeSeriesV4 string = "time_series_v4"
|
||||
localTimeSeriesV46hrs string = "time_series_v4_6hrs"
|
||||
localTimeSeriesV41day string = "time_series_v4_1day"
|
||||
localTimeSeriesV41week string = "time_series_v4_1week"
|
||||
)
|
||||
|
||||
// localTimeSeriesTable maps a distributed time series table to its shard-local
|
||||
// table. Samples and time series shard on the same key
|
||||
// (cityHash64(env, temporality, metric_name, fingerprint)), so a query whose
|
||||
// top-level FROM is the distributed samples table can join or semi-join the
|
||||
// local time series table inside each shard: the shard rewrite runs the
|
||||
// subquery against the shard's own series rows, which are exactly the series
|
||||
// of the shard's samples. No broadcast, no initiator-side join.
|
||||
func localTimeSeriesTable(distributed string) string {
|
||||
switch distributed {
|
||||
case distributedTimeSeriesV46hrs:
|
||||
return localTimeSeriesV46hrs
|
||||
case distributedTimeSeriesV41day:
|
||||
return localTimeSeriesV41day
|
||||
case distributedTimeSeriesV41week:
|
||||
return localTimeSeriesV41week
|
||||
default:
|
||||
return localTimeSeriesV4
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
oneHourInMilliseconds = time.Hour.Milliseconds()
|
||||
sixHoursInMilliseconds = time.Hour.Milliseconds() * 6
|
||||
oneDayInMilliseconds = time.Hour.Milliseconds() * 24
|
||||
oneWeekInMilliseconds = time.Hour.Milliseconds() * 24 * 7
|
||||
)
|
||||
|
||||
// timeSeriesTableFor returns the adjusted start and the time series table for
|
||||
// the window. Time series tables hold one row per (fingerprint, bucket), with
|
||||
// bucket granularities of 1h, 6h, 1d and 1w; the start is rounded down to the
|
||||
// bucket boundary so a window beginning mid-bucket still matches the bucket's
|
||||
// row.
|
||||
func timeSeriesTableFor(start, end int64) (int64, string) {
|
||||
switch {
|
||||
case end-start < sixHoursInMilliseconds:
|
||||
return start - (start % oneHourInMilliseconds), distributedTimeSeriesV4
|
||||
case end-start < oneDayInMilliseconds:
|
||||
return start - (start % sixHoursInMilliseconds), distributedTimeSeriesV46hrs
|
||||
case end-start < oneWeekInMilliseconds:
|
||||
return start - (start % oneDayInMilliseconds), distributedTimeSeriesV41day
|
||||
default:
|
||||
return start - (start % oneWeekInMilliseconds), distributedTimeSeriesV41week
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,492 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
)
|
||||
|
||||
// The compiler turns PromQL subtrees into single ClickHouse queries built on
|
||||
// the timeSeries*ToGrid aggregate functions (CH >= 25.6), whose semantics
|
||||
// were verified against this repo's vendored engine: exact extrapolatedRate
|
||||
// behavior including counter resets, the counter zero-point clamp, the
|
||||
// 1.1x-average extrapolation threshold, left-open windows, the >= 2 samples
|
||||
// rule, stale-marker shadowing, and millisecond grid starts. Sample rows
|
||||
// never leave ClickHouse: one row per output series comes back, holding the
|
||||
// whole grid as an array.
|
||||
//
|
||||
// Scope (the allowlist): an optional sum/min/max/avg/count by/without
|
||||
// aggregation over a core unit — a rate/increase/delta/irate/idelta range
|
||||
// selection, an instant vector selection, or an avg/min/max/sum/count/last
|
||||
// _over_time window — plus number-literal arithmetic/comparisons and unary
|
||||
// minus on top. Units inside fixed-resolution subqueries evaluate on the
|
||||
// subquery's own grid. Everything else either falls back to the engine over
|
||||
// this package's querier, or — when a transpilable subtree sits under a
|
||||
// non-transpilable node — runs hybrid: the subtree's grids are computed in
|
||||
// ClickHouse and substituted into the engine as synthetic series (see
|
||||
// compiler_exec.go). See doc.go for the fallback list and the reasons behind
|
||||
// each entry.
|
||||
|
||||
// rangeFn is a transpilable range-vector function.
|
||||
type rangeFn string
|
||||
|
||||
const (
|
||||
fnRate rangeFn = "rate"
|
||||
fnIncrease rangeFn = "increase"
|
||||
fnDelta rangeFn = "delta"
|
||||
fnIRate rangeFn = "irate"
|
||||
fnIDelta rangeFn = "idelta"
|
||||
)
|
||||
|
||||
var gridFunction = map[rangeFn]string{
|
||||
fnRate: "timeSeriesRateToGrid",
|
||||
fnIncrease: "timeSeriesRateToGrid", // increase == rate * range seconds, exactly (same factor algebra)
|
||||
fnDelta: "timeSeriesDeltaToGrid",
|
||||
fnIRate: "timeSeriesInstantRateToGrid",
|
||||
fnIDelta: "timeSeriesInstantDeltaToGrid",
|
||||
}
|
||||
|
||||
// scalarOp is one number-literal arithmetic or comparison applied to a
|
||||
// compiled vector, evaluated in Go during assembly with the same float64
|
||||
// operations the engine uses.
|
||||
type scalarOp struct {
|
||||
op parser.ItemType
|
||||
scalar float64
|
||||
scalarOnLeft bool
|
||||
returnBool bool
|
||||
}
|
||||
|
||||
// isComparison reports whether the op is a filtering/bool comparison, which
|
||||
// preserves the metric name (arithmetic drops it).
|
||||
func (o scalarOp) isComparison() bool {
|
||||
return o.op.IsComparisonOperator()
|
||||
}
|
||||
|
||||
// unitKind is the selector shape at the bottom of a core unit.
|
||||
type unitKind int
|
||||
|
||||
const (
|
||||
// unitRange: rate/increase/delta/irate/idelta over a matrix selector.
|
||||
unitRange unitKind = iota
|
||||
// unitInstant: a plain vector selector resolved per grid point with
|
||||
// lookback and stale-marker shadowing.
|
||||
unitInstant
|
||||
// unitOverTime: avg/min/max/sum/count/last_over_time over a matrix
|
||||
// selector (aggregation over the window's samples, stale rows excluded).
|
||||
unitOverTime
|
||||
)
|
||||
|
||||
// coreUnit is one transpilable subtree: selector [-> range function] ->
|
||||
// optional aggregation -> scalar op pipeline.
|
||||
type coreUnit struct {
|
||||
kind unitKind
|
||||
matchers []*labels.Matcher
|
||||
offsetMs int64
|
||||
fn rangeFn // unitRange
|
||||
overFn string // unitOverTime: avg|min|max|sum|count|last
|
||||
rangeMs int64 // unitRange/unitOverTime window
|
||||
|
||||
hasAgg bool
|
||||
aggOp parser.ItemType // SUM MIN MAX AVG COUNT
|
||||
by bool
|
||||
grouping []string
|
||||
|
||||
ops []scalarOp
|
||||
}
|
||||
|
||||
// keepsName reports whether the unit's output series keep their real
|
||||
// __name__: bare/comparison-filtered instant selectors and last_over_time do
|
||||
// (it returns the raw sample, name included); range functions, the other
|
||||
// *_over_time functions, aggregations, arithmetic and bool comparisons all
|
||||
// drop it — a bool comparison returns 0/1, not the sample, so the engine
|
||||
// drops the name there too. Units that keep the name cannot be substituted
|
||||
// as synthetic series in hybrid plans — the synthetic name would replace
|
||||
// the real one — but transpile fine as full plans, where assembly emits the
|
||||
// real names.
|
||||
func (u *coreUnit) keepsName() bool {
|
||||
nameKeepingSelector := u.kind == unitInstant || (u.kind == unitOverTime && u.overFn == "last")
|
||||
if !nameKeepingSelector || u.hasAgg {
|
||||
return false
|
||||
}
|
||||
for _, op := range u.ops {
|
||||
if !op.isComparison() || op.returnBool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// gridContext is the evaluation grid a unit computes on. The query grid for
|
||||
// top-level units; for units inside subqueries, the subquery's own grid:
|
||||
// epoch-aligned multiples of its resolution covering the subquery window,
|
||||
// exactly as the engine derives it (engine.go, *parser.SubqueryExpr case).
|
||||
type gridContext struct {
|
||||
startMs int64
|
||||
endMs int64
|
||||
stepMs int64
|
||||
}
|
||||
|
||||
// subqueryGrid derives the inner grid for a subquery evaluated on outer:
|
||||
// interval S, end = outer end − offset, start = first multiple of S strictly
|
||||
// greater than outer start − offset − range.
|
||||
func subqueryGrid(outer gridContext, rangeMs, stepMs, offsetMs int64) gridContext {
|
||||
lower := outer.startMs - offsetMs - rangeMs
|
||||
start := stepMs * (lower / stepMs)
|
||||
if start <= lower {
|
||||
start += stepMs
|
||||
}
|
||||
return gridContext{startMs: start, endMs: outer.endMs - offsetMs, stepMs: stepMs}
|
||||
}
|
||||
|
||||
// transpiledUnit is a coreUnit scheduled for execution, named for hybrid
|
||||
// substitution, carrying the grid it evaluates on.
|
||||
type transpiledUnit struct {
|
||||
core coreUnit
|
||||
name string // __signoz_transpiled_<n>__
|
||||
grid gridContext
|
||||
}
|
||||
|
||||
// transpilePlan is the outcome of classifying a query.
|
||||
type transpilePlan struct {
|
||||
units []*transpiledUnit
|
||||
grid gridContext // the query's top-level grid
|
||||
// full is set when the entire query is units[0]; otherwise rewritten
|
||||
// holds the query with each unit replaced by a synthetic selector, to be
|
||||
// evaluated by the engine over a hybrid storage.
|
||||
full bool
|
||||
rewritten string
|
||||
}
|
||||
|
||||
const syntheticNamePrefix = "__signoz_transpiled_"
|
||||
|
||||
func syntheticName(i int) string {
|
||||
return fmt.Sprintf("%s%d__", syntheticNamePrefix, i)
|
||||
}
|
||||
|
||||
// classifyCore matches a subtree against the transpilable core shape.
|
||||
// stepMs gates second-granularity: the grid functions take whole-second step
|
||||
// and window parameters (grid *starts* are millisecond-precise).
|
||||
func classifyCore(node parser.Expr, stepMs int64) (*coreUnit, bool) {
|
||||
unit := &coreUnit{}
|
||||
|
||||
expr := node
|
||||
// Peel scalar ops and parens off the top, outermost first; ops apply in
|
||||
// evaluation order, so prepend while peeling.
|
||||
for {
|
||||
switch n := expr.(type) {
|
||||
case *parser.ParenExpr:
|
||||
expr = n.Expr
|
||||
continue
|
||||
case *parser.UnaryExpr:
|
||||
if n.Op != parser.SUB {
|
||||
expr = n.Expr // unary '+' is a no-op
|
||||
continue
|
||||
}
|
||||
// -x == -1 * x for every float64 (incl. NaN and signed zero).
|
||||
unit.ops = append([]scalarOp{{op: parser.MUL, scalar: -1}}, unit.ops...)
|
||||
expr = n.Expr
|
||||
continue
|
||||
case *parser.StepInvariantExpr:
|
||||
// @-pinned expressions evaluate on a different grid.
|
||||
return nil, false
|
||||
case *parser.BinaryExpr:
|
||||
lit, litOnLeft, ok := numberLiteralSide(n)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
if !n.Op.IsOperator() && !n.Op.IsComparisonOperator() {
|
||||
return nil, false
|
||||
}
|
||||
if n.Op == parser.ATAN2 {
|
||||
// atan2 is arithmetic in PromQL but rarely used; keep the
|
||||
// allowlist tight.
|
||||
return nil, false
|
||||
}
|
||||
returnBool := n.ReturnBool
|
||||
unit.ops = append([]scalarOp{{op: n.Op, scalar: lit, scalarOnLeft: litOnLeft, returnBool: returnBool}}, unit.ops...)
|
||||
if litOnLeft {
|
||||
expr = n.RHS
|
||||
} else {
|
||||
expr = n.LHS
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Optional aggregation.
|
||||
if agg, ok := expr.(*parser.AggregateExpr); ok {
|
||||
switch agg.Op {
|
||||
case parser.SUM, parser.MIN, parser.MAX, parser.AVG, parser.COUNT:
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
for _, g := range agg.Grouping {
|
||||
if g == metricNameLabel {
|
||||
// by(__name__)/without(__name__) over synthetic or compiled
|
||||
// output needs name bookkeeping the compiler doesn't do.
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
unit.hasAgg = true
|
||||
unit.aggOp = agg.Op
|
||||
unit.by = !agg.Without
|
||||
unit.grouping = agg.Grouping
|
||||
expr = agg.Expr
|
||||
for {
|
||||
if p, ok := expr.(*parser.ParenExpr); ok {
|
||||
expr = p.Expr
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// The grid functions take whole-second steps; stepMs == 0 is an instant
|
||||
// query (single-point grid).
|
||||
if stepMs < 0 || stepMs%1000 != 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Bare instant selector: resolved per grid point with lookback and
|
||||
// stale-marker shadowing (see compiler_sql.go).
|
||||
if vs, ok := expr.(*parser.VectorSelector); ok {
|
||||
// A duration expression (offset step(), offset range()*2, ...) is
|
||||
// resolved into OriginalOffset only at evaluation time; at
|
||||
// classification time the field still holds its zero value, so
|
||||
// transpiling would silently use the wrong offset.
|
||||
if vs.Timestamp != nil || vs.StartOrEnd != 0 || vs.Anchored || vs.Smoothed || vs.OriginalOffsetExpr != nil {
|
||||
return nil, false
|
||||
}
|
||||
offsetMs := vs.OriginalOffset.Milliseconds()
|
||||
if offsetMs < 0 {
|
||||
return nil, false
|
||||
}
|
||||
unit.kind = unitInstant
|
||||
unit.offsetMs = offsetMs
|
||||
unit.matchers = vs.LabelMatchers
|
||||
return unit, true
|
||||
}
|
||||
|
||||
// Range or *_over_time function over a plain matrix selector.
|
||||
call, ok := expr.(*parser.Call)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
var fn rangeFn
|
||||
var overFn string
|
||||
switch call.Func.Name {
|
||||
case "rate":
|
||||
fn = fnRate
|
||||
case "increase":
|
||||
fn = fnIncrease
|
||||
case "delta":
|
||||
fn = fnDelta
|
||||
case "irate":
|
||||
fn = fnIRate
|
||||
case "idelta":
|
||||
fn = fnIDelta
|
||||
case "avg_over_time", "min_over_time", "max_over_time", "sum_over_time", "count_over_time", "last_over_time":
|
||||
overFn = strings.TrimSuffix(call.Func.Name, "_over_time")
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
if len(call.Args) != 1 {
|
||||
return nil, false
|
||||
}
|
||||
ms, ok := call.Args[0].(*parser.MatrixSelector)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
vs, ok := ms.VectorSelector.(*parser.VectorSelector)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
// Duration expressions resolve at evaluation time (see the instant
|
||||
// selector case above); Range/OriginalOffset would be read as zero here.
|
||||
if vs.Timestamp != nil || vs.StartOrEnd != 0 || vs.Anchored || vs.Smoothed || vs.OriginalOffsetExpr != nil || ms.RangeExpr != nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
rangeMs := ms.Range.Milliseconds()
|
||||
offsetMs := vs.OriginalOffset.Milliseconds()
|
||||
if rangeMs <= 0 || rangeMs%1000 != 0 || offsetMs < 0 {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
if overFn != "" {
|
||||
unit.kind = unitOverTime
|
||||
unit.overFn = overFn
|
||||
} else {
|
||||
unit.kind = unitRange
|
||||
unit.fn = fn
|
||||
}
|
||||
unit.rangeMs = rangeMs
|
||||
unit.offsetMs = offsetMs
|
||||
unit.matchers = vs.LabelMatchers
|
||||
return unit, true
|
||||
}
|
||||
|
||||
// numberLiteralSide returns the number literal on one side of a binary
|
||||
// expression (peeling parens and unary minus), and which side it is on.
|
||||
func numberLiteralSide(b *parser.BinaryExpr) (float64, bool, bool) {
|
||||
if v, ok := literalValue(b.LHS); ok {
|
||||
return v, true, true
|
||||
}
|
||||
if v, ok := literalValue(b.RHS); ok {
|
||||
return v, false, true
|
||||
}
|
||||
return 0, false, false
|
||||
}
|
||||
|
||||
func literalValue(e parser.Expr) (float64, bool) {
|
||||
neg := false
|
||||
for {
|
||||
switch n := e.(type) {
|
||||
case *parser.ParenExpr:
|
||||
e = n.Expr
|
||||
continue
|
||||
case *parser.StepInvariantExpr:
|
||||
e = n.Expr
|
||||
continue
|
||||
case *parser.UnaryExpr:
|
||||
if n.Op == parser.SUB {
|
||||
neg = !neg
|
||||
}
|
||||
e = n.Expr
|
||||
continue
|
||||
case *parser.NumberLiteral:
|
||||
if neg {
|
||||
return -n.Val, true
|
||||
}
|
||||
return n.Val, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// classify builds the compile plan for a query: full when the root is a core
|
||||
// unit, hybrid when core units sit strictly below the root (including inside
|
||||
// fixed-resolution subqueries, computed on the subquery grid), none
|
||||
// otherwise.
|
||||
func classify(root parser.Expr, grid gridContext) (*transpilePlan, bool) {
|
||||
if unit, ok := classifyCore(root, grid.stepMs); ok {
|
||||
return &transpilePlan{
|
||||
units: []*transpiledUnit{{core: *unit, name: syntheticName(0), grid: grid}},
|
||||
grid: grid,
|
||||
full: true,
|
||||
}, true
|
||||
}
|
||||
|
||||
plan := &transpilePlan{grid: grid}
|
||||
rewritten := rewrite(root, grid, plan, false)
|
||||
if len(plan.units) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
plan.rewritten = rewritten.String()
|
||||
return plan, true
|
||||
}
|
||||
|
||||
// rewrite walks top-down replacing maximal transpilable subtrees with synthetic
|
||||
// vector selectors. nameSensitive marks scopes where an ancestor's semantics
|
||||
// depend on __name__ (grouping or vector matching on it): synthetic series
|
||||
// carry a synthetic __name__, so substitution there would change results.
|
||||
// Fixed-resolution subqueries recurse with the subquery's own grid; scopes
|
||||
// whose evaluation grid is unknowable (@-pinned, default-resolution
|
||||
// subqueries) are not entered.
|
||||
func rewrite(node parser.Expr, grid gridContext, plan *transpilePlan, nameSensitive bool) parser.Expr {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !nameSensitive {
|
||||
// Units whose output keeps the real __name__ (bare instant selectors)
|
||||
// cannot be substituted: the synthetic name would replace it in the
|
||||
// engine's output. They still compile as full plans.
|
||||
if unit, ok := classifyCore(node, grid.stepMs); ok && !unit.keepsName() {
|
||||
cu := &transpiledUnit{core: *unit, name: syntheticName(len(plan.units)), grid: grid}
|
||||
plan.units = append(plan.units, cu)
|
||||
return &parser.VectorSelector{
|
||||
Name: cu.name,
|
||||
LabelMatchers: []*labels.Matcher{
|
||||
labels.MustNewMatcher(labels.MatchEqual, metricNameLabel, cu.name),
|
||||
},
|
||||
PosRange: node.PositionRange(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch n := node.(type) {
|
||||
case *parser.ParenExpr:
|
||||
n.Expr = rewrite(n.Expr, grid, plan, nameSensitive)
|
||||
case *parser.UnaryExpr:
|
||||
n.Expr = rewrite(n.Expr, grid, plan, nameSensitive)
|
||||
case *parser.AggregateExpr:
|
||||
sensitive := nameSensitive || groupingUsesName(n.Grouping)
|
||||
n.Expr = rewrite(n.Expr, grid, plan, sensitive)
|
||||
// n.Param is a scalar/string; nothing transpilable inside for our core.
|
||||
case *parser.Call:
|
||||
for i, arg := range n.Args {
|
||||
n.Args[i] = rewrite(arg, grid, plan, nameSensitive)
|
||||
}
|
||||
case *parser.BinaryExpr:
|
||||
sensitive := nameSensitive || vectorMatchingUsesName(n.VectorMatching)
|
||||
n.LHS = rewrite(n.LHS, grid, plan, sensitive)
|
||||
n.RHS = rewrite(n.RHS, grid, plan, sensitive)
|
||||
case *parser.SubqueryExpr:
|
||||
// The alert-smoothing idiom fn_over_time((expr)[R:S]) dominates real
|
||||
// rule fleets; inner units evaluate on the subquery grid, and the
|
||||
// engine does the smoothing over the synthetic series. Requires an
|
||||
// explicit whole-second resolution (S == 0 needs the engine's
|
||||
// default-interval function) and no @ pinning.
|
||||
stepMs := n.Step.Milliseconds()
|
||||
rangeMs := n.Range.Milliseconds()
|
||||
offsetMs := n.OriginalOffset.Milliseconds()
|
||||
if n.Timestamp == nil && n.StartOrEnd == 0 &&
|
||||
n.RangeExpr == nil && n.StepExpr == nil && n.OriginalOffsetExpr == nil &&
|
||||
stepMs > 0 && stepMs%1000 == 0 && rangeMs%1000 == 0 && offsetMs >= 0 {
|
||||
inner := subqueryGrid(grid, rangeMs, stepMs, offsetMs)
|
||||
n.Expr = rewrite(n.Expr, inner, plan, nameSensitive)
|
||||
}
|
||||
case *parser.StepInvariantExpr, *parser.MatrixSelector,
|
||||
*parser.VectorSelector, *parser.NumberLiteral, *parser.StringLiteral:
|
||||
// Leaves, or scopes substitution must not enter.
|
||||
}
|
||||
return node
|
||||
}
|
||||
|
||||
func groupingUsesName(grouping []string) bool {
|
||||
for _, g := range grouping {
|
||||
if g == metricNameLabel {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func vectorMatchingUsesName(vm *parser.VectorMatching) bool {
|
||||
if vm == nil {
|
||||
return false
|
||||
}
|
||||
for _, l := range append(append([]string{}, vm.MatchingLabels...), vm.Include...) {
|
||||
if l == metricNameLabel {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Default (all-labels) matching ignores __name__, and by()/ignoring()
|
||||
// lists were checked above.
|
||||
return false
|
||||
}
|
||||
|
||||
// isSyntheticSelector reports whether matchers target a compiled unit.
|
||||
func isSyntheticSelector(matchers []*labels.Matcher) (string, bool) {
|
||||
for _, m := range matchers {
|
||||
if m.Name == metricNameLabel && m.Type == labels.MatchEqual && strings.HasPrefix(m.Value, syntheticNamePrefix) {
|
||||
return m.Value, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestClassifyCorpus measures real-workload compiler coverage: it classifies
|
||||
// every query of a JSON-lines corpus (one JSON-encoded PromQL string per
|
||||
// line) with the live classifier and reports full / hybrid / fallback
|
||||
// shares. Skipped unless PROMQL_CORPUS points to one or more files
|
||||
// (comma-separated). Dashboard template variables are substituted with
|
||||
// placeholder values before parsing, mirroring the production render step.
|
||||
//
|
||||
// PROMQL_CORPUS=corpus-a.jsonl,corpus-b.jsonl go test -run TestClassifyCorpus -v
|
||||
func TestClassifyCorpus(t *testing.T) {
|
||||
corpus := os.Getenv("PROMQL_CORPUS")
|
||||
if corpus == "" {
|
||||
t.Skip("PROMQL_CORPUS not set")
|
||||
}
|
||||
|
||||
varRe := regexp.MustCompile(`\{\{\s*\.?[\w.]+\s*\}\}|\[\[\s*[\w.]+\s*\]\]|\$[\w.]+`)
|
||||
promParser := parser.NewParser(parser.Options{})
|
||||
|
||||
for _, path := range strings.Split(corpus, ",") {
|
||||
f, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
|
||||
var full, hybrid, fallbackInstant, fallbackOther, parseErrs int
|
||||
fallbackReasons := map[string]int{}
|
||||
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 1024*1024), 1024*1024)
|
||||
for scanner.Scan() {
|
||||
var query string
|
||||
require.NoError(t, json.Unmarshal(scanner.Bytes(), &query))
|
||||
query = varRe.ReplaceAllString(query, "placeholder")
|
||||
|
||||
expr, err := promParser.ParseExpr(query)
|
||||
if err != nil {
|
||||
parseErrs++
|
||||
continue
|
||||
}
|
||||
|
||||
plan, ok := classify(expr, gridContext{startMs: 1_700_000_000_000, endMs: 1_700_007_200_000, stepMs: 60_000})
|
||||
switch {
|
||||
case ok && plan.full:
|
||||
full++
|
||||
case ok:
|
||||
hybrid++
|
||||
default:
|
||||
reason := fallbackShape(expr)
|
||||
fallbackReasons[reason]++
|
||||
if reason == "instant-selector shape (last-sample-per-step engine path)" {
|
||||
fallbackInstant++
|
||||
} else {
|
||||
fallbackOther++
|
||||
}
|
||||
}
|
||||
}
|
||||
require.NoError(t, scanner.Err())
|
||||
_ = f.Close()
|
||||
|
||||
total := full + hybrid + fallbackInstant + fallbackOther
|
||||
if total == 0 {
|
||||
t.Logf("%s: no parseable queries (%d parse errors)", path, parseErrs)
|
||||
continue
|
||||
}
|
||||
t.Logf("%s: %d queries — full=%d (%.0f%%) hybrid=%d (%.0f%%) fallback=%d (%.0f%%; instant-shape=%d) parse_errors=%d",
|
||||
path, total,
|
||||
full, 100*float64(full)/float64(total),
|
||||
hybrid, 100*float64(hybrid)/float64(total),
|
||||
fallbackInstant+fallbackOther, 100*float64(fallbackInstant+fallbackOther)/float64(total),
|
||||
fallbackInstant, parseErrs)
|
||||
|
||||
reasons := make([]string, 0, len(fallbackReasons))
|
||||
for r := range fallbackReasons {
|
||||
reasons = append(reasons, r)
|
||||
}
|
||||
sort.Slice(reasons, func(i, j int) bool { return fallbackReasons[reasons[i]] > fallbackReasons[reasons[j]] })
|
||||
for _, r := range reasons {
|
||||
t.Logf(" fallback %4d %s", fallbackReasons[r], r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackShape buckets a non-transpilable query by why it stays on the engine
|
||||
// path, to separate "already served well" (instant selectors on the last-sample-per-step
|
||||
// path) from genuine compiler gaps.
|
||||
func fallbackShape(expr parser.Expr) string {
|
||||
var hasMatrix, hasSubquery, hasAt, hasDurationExpr, overTime bool
|
||||
rangeFns := map[string]bool{"rate": true, "increase": true, "delta": true, "irate": true, "idelta": true}
|
||||
var unsupportedFns []string
|
||||
parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
|
||||
switch n := node.(type) {
|
||||
case *parser.MatrixSelector:
|
||||
hasMatrix = true
|
||||
if n.RangeExpr != nil {
|
||||
hasDurationExpr = true
|
||||
}
|
||||
case *parser.SubqueryExpr:
|
||||
hasSubquery = true
|
||||
if n.RangeExpr != nil || n.StepExpr != nil || n.OriginalOffsetExpr != nil {
|
||||
hasDurationExpr = true
|
||||
}
|
||||
case *parser.VectorSelector:
|
||||
if n.Timestamp != nil || n.StartOrEnd != 0 {
|
||||
hasAt = true
|
||||
}
|
||||
if n.OriginalOffsetExpr != nil {
|
||||
hasDurationExpr = true
|
||||
}
|
||||
case *parser.Call:
|
||||
if strings.HasSuffix(n.Func.Name, "_over_time") {
|
||||
overTime = true
|
||||
} else if !rangeFns[n.Func.Name] {
|
||||
unsupportedFns = append(unsupportedFns, n.Func.Name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
switch {
|
||||
case hasDurationExpr:
|
||||
return "duration expression (resolved only at evaluation time)"
|
||||
case hasSubquery:
|
||||
return "subquery"
|
||||
case hasAt:
|
||||
return "@ modifier"
|
||||
case overTime:
|
||||
return "*_over_time range function"
|
||||
case !hasMatrix:
|
||||
return "instant-selector shape (last-sample-per-step engine path)"
|
||||
case len(unsupportedFns) > 0:
|
||||
return fmt.Sprintf("range shape with unsupported function(s): %s", strings.Join(dedupe(unsupportedFns), ",")) //nolint:makezero
|
||||
default:
|
||||
return "other range shape"
|
||||
}
|
||||
}
|
||||
|
||||
func dedupe(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, s := range in {
|
||||
if !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
@@ -1,550 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
promValue "github.com/prometheus/prometheus/model/value"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/storage"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// executor evaluates transpilable PromQL directly in ClickHouse, falling
|
||||
// back (ok=false) whenever the query shape or the step doesn't qualify. The
|
||||
// timeSeries*ToGrid functions it builds on are assumed available: the
|
||||
// supported ClickHouse floor is >= 25.6.
|
||||
type executor struct {
|
||||
client *client
|
||||
engine *prometheus.Engine
|
||||
parser prometheus.Parser
|
||||
}
|
||||
|
||||
// maxWindowBuckets caps range/step for the windowed *_over_time form: every
|
||||
// grid slot combines that many bucket partials, and the fleet's windows sit
|
||||
// well under it ([1m]..[17m] at 30-60s steps) — anything larger is a
|
||||
// long-range query whose step a dashboard scales up anyway, and the engine
|
||||
// path serves the rest.
|
||||
const maxWindowBuckets = 64
|
||||
|
||||
// TryExecuteRange transpiles and runs the query in ClickHouse when its shape
|
||||
// is in the allowlist. ok=false means "not transpilable" and carries no
|
||||
// error; the caller runs the engine path.
|
||||
func (e *executor) TryExecuteRange(ctx context.Context, qs string, start, end time.Time, step time.Duration) (promql.Matrix, bool, error) {
|
||||
expr, err := e.parser.ParseExpr(qs)
|
||||
if err != nil {
|
||||
// Let the engine path produce the (enhanced) parse error.
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
plan, ok := classify(expr, queryGrid(start, end, step))
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
// timeSeriesLastToGrid widens its window to max(window, step) — probed: a
|
||||
// sample aged (window, step] still fills the slot — while the rate/delta
|
||||
// family enforces the window strictly. The Last-style kinds used to fall
|
||||
// back when window < step because of that widening; the window-sliver
|
||||
// filter (see samplesConditions) makes the widening harmless there:
|
||||
// samples exist only inside (t_k - window, t_k] slivers, so the widened
|
||||
// window intersected with the data IS the lookback window — and if a
|
||||
// future ClickHouse stops widening, the unwidened window is the sliver
|
||||
// too. Correct either way. A non-positive window still falls back: the
|
||||
// sliver argument needs a real window to filter to.
|
||||
//
|
||||
// The windowed *_over_time form gates only the range >= step regime: it
|
||||
// decomposes the window into whole step buckets (see windowedInner),
|
||||
// which is exact only when the range is a multiple of the step, and its
|
||||
// per-slot slide costs range/step bucket combines — bounded by
|
||||
// maxWindowBuckets so a long-range short-step query cannot turn the
|
||||
// slide into the bottleneck. range < step needs neither gate: the
|
||||
// windows are disjoint slivers, aggregated one slot each with no slide.
|
||||
// Every miss falls back to the engine path, which is exact.
|
||||
for _, unit := range plan.units {
|
||||
stepMs := unit.grid.stepMs
|
||||
if stepMs == 0 {
|
||||
stepMs = 1000
|
||||
}
|
||||
switch {
|
||||
case unit.core.kind == unitInstant || (unit.core.kind == unitOverTime && unit.core.overFn == "last"):
|
||||
windowMs := unit.core.rangeMs
|
||||
if unit.core.kind == unitInstant {
|
||||
windowMs = e.client.lookbackMs
|
||||
}
|
||||
if windowMs <= 0 {
|
||||
return nil, false, nil
|
||||
}
|
||||
case unit.core.kind == unitOverTime:
|
||||
if unit.core.rangeMs < unit.grid.stepMs {
|
||||
// Disjoint slivers: no divisibility or width requirement.
|
||||
continue
|
||||
}
|
||||
if unit.core.rangeMs%stepMs != 0 || unit.core.rangeMs/stepMs > maxWindowBuckets {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Evaluate every unit concurrently on its own grid (the query grid, or a
|
||||
// subquery grid); each is one series lookup plus one grid query.
|
||||
results := make([][]transpiledSeries, len(plan.units))
|
||||
eg, egCtx := errgroup.WithContext(ctx)
|
||||
for i, unit := range plan.units {
|
||||
eg.Go(func() error {
|
||||
res, err := e.executeUnit(egCtx, &unit.core, unit.grid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
results[i] = res
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err := eg.Wait(); err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
|
||||
if plan.full {
|
||||
g := plan.units[0].grid
|
||||
return toMatrix(results[0], g.startMs, g.stepMs), true, nil
|
||||
}
|
||||
|
||||
matrix, err := e.executeHybrid(ctx, plan, results)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
return matrix, true, nil
|
||||
}
|
||||
|
||||
// queryGrid derives the top-level evaluation grid; step 0 is an instant
|
||||
// query: a single evaluation at end, whatever start was.
|
||||
func queryGrid(start, end time.Time, step time.Duration) gridContext {
|
||||
startMs, endMs, stepMs := start.UnixMilli(), end.UnixMilli(), step.Milliseconds()
|
||||
if stepMs == 0 {
|
||||
startMs = endMs
|
||||
}
|
||||
return gridContext{startMs: startMs, endMs: endMs, stepMs: stepMs}
|
||||
}
|
||||
|
||||
// transpiledSeries is one output series of a unit: projected labels and one
|
||||
// value pointer per grid point (nil = absent).
|
||||
type transpiledSeries struct {
|
||||
lset labels.Labels
|
||||
values []*float64
|
||||
}
|
||||
|
||||
// executeUnit runs one core unit on its grid: series lookup (budgets,
|
||||
// fingerprints, metric names), then the single grid query, then the
|
||||
// scalar-op pipeline.
|
||||
func (e *executor) executeUnit(ctx context.Context, unit *coreUnit, grid gridContext) ([]transpiledSeries, error) {
|
||||
startMs, endMs, stepMs := grid.startMs, grid.endMs, grid.stepMs
|
||||
windowMs := unit.rangeMs
|
||||
if unit.kind == unitInstant {
|
||||
windowMs = e.client.lookbackMs
|
||||
}
|
||||
dataStart := startMs - unit.offsetMs - windowMs
|
||||
dataEnd := endMs - unit.offsetMs
|
||||
|
||||
seriesQuery, seriesArgs, err := buildSeriesQuery(dataStart, dataEnd, unit.matchers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lookup, err := e.client.selectSeries(ctx, seriesQuery, seriesArgs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(lookup.fingerprints) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
query, args, err := buildUnitSQL(unit, lookup.metricNames, dataStart, dataEnd, startMs, endMs, stepMs, e.client.lookbackMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rows, err := e.client.telemetryStore.ClickhouseDB().Query(e.client.withContext(ctx, "transpiledUnit"), query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Name-dropping units keep __name__ in the SQL group key so distinct
|
||||
// metrics never merge server-side; the name comes off here. Two metrics
|
||||
// can then share a labelset — the engine merges their samples into one
|
||||
// series when they never overlap in time (a selector spanning metrics
|
||||
// whose series alternate across lookback windows) and raises the
|
||||
// duplicate-labelset error only when two samples land on the same
|
||||
// evaluation timestamp. mergeSameLabelsetSeries reproduces exactly that.
|
||||
stripName := !unit.hasAgg && !unit.keepsName()
|
||||
|
||||
// by (...) units return one plain column per grouped label; everything
|
||||
// else returns the single canonical JSON key (see groupKeyColumns).
|
||||
keyNames := groupKeyColumns(unit)
|
||||
keyVals := make([]string, max(len(keyNames), 1))
|
||||
targets := make([]any, 0, len(keyVals)+1)
|
||||
for i := range keyVals {
|
||||
targets = append(targets, &keyVals[i])
|
||||
}
|
||||
var gridValues []*float64
|
||||
targets = append(targets, &gridValues)
|
||||
|
||||
var out []transpiledSeries
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(targets...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var lset labels.Labels
|
||||
if keyNames != nil {
|
||||
builder := labels.NewScratchBuilder(len(keyNames))
|
||||
for i, name := range keyNames {
|
||||
// An empty extracted value is the label being absent.
|
||||
if keyVals[i] != "" {
|
||||
builder.Add(name, keyVals[i])
|
||||
}
|
||||
}
|
||||
builder.Sort()
|
||||
lset = builder.Labels()
|
||||
} else {
|
||||
lset, err = labelsFromGroupKey(keyVals[0])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if stripName {
|
||||
lset = labels.NewBuilder(lset).Del(metricNameLabel).Labels()
|
||||
}
|
||||
values := make([]*float64, len(gridValues))
|
||||
copy(values, gridValues)
|
||||
applyScalarOps(unit.ops, values)
|
||||
out = append(out, transpiledSeries{lset: lset, values: values})
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if stripName {
|
||||
if out, err = mergeSameLabelsetSeries(out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return labels.Compare(out[i].lset, out[j].lset) < 0 })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mergeSameLabelsetSeries combines series left with identical labelsets by a
|
||||
// name strip, slot by slot: the engine assembles its result matrix by
|
||||
// labelset, so post-strip twins whose points interleave in time are one
|
||||
// series to it, and two values on the same evaluation timestamp are its
|
||||
// duplicate-labelset error — v1 would have errored there too, so silently
|
||||
// picking one value would be a divergence.
|
||||
func mergeSameLabelsetSeries(in []transpiledSeries) ([]transpiledSeries, error) {
|
||||
index := make(map[uint64]int, len(in))
|
||||
out := in[:0]
|
||||
for _, s := range in {
|
||||
hash := s.lset.Hash()
|
||||
idx, ok := index[hash]
|
||||
if ok && labels.Equal(out[idx].lset, s.lset) {
|
||||
dst := out[idx].values
|
||||
for k, v := range s.values {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
if dst[k] != nil {
|
||||
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "vector cannot contain metrics with the same labelset")
|
||||
}
|
||||
dst[k] = v
|
||||
}
|
||||
continue
|
||||
}
|
||||
index[hash] = len(out)
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// labelsFromGroupKey parses the toJSONString'd sorted [key, value] pairs.
|
||||
func labelsFromGroupKey(gkey string) (labels.Labels, error) {
|
||||
var pairs [][]string
|
||||
if err := json.Unmarshal([]byte(gkey), &pairs); err != nil {
|
||||
return labels.EmptyLabels(), errors.WrapInternalf(err, errors.CodeInternal, "malformed compiled group key %q", gkey)
|
||||
}
|
||||
builder := labels.NewScratchBuilder(len(pairs))
|
||||
for _, p := range pairs {
|
||||
if len(p) != 2 {
|
||||
return labels.EmptyLabels(), errors.NewInternalf(errors.CodeInternal, "malformed compiled group key pair %q", gkey)
|
||||
}
|
||||
builder.Add(p[0], p[1])
|
||||
}
|
||||
builder.Sort()
|
||||
return builder.Labels(), nil
|
||||
}
|
||||
|
||||
// applyScalarOps applies the number-literal op pipeline in place, with the
|
||||
// same float64 arithmetic and comparison-filter semantics as the engine.
|
||||
func applyScalarOps(ops []scalarOp, values []*float64) {
|
||||
for _, op := range ops {
|
||||
for i, v := range values {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
lhs, rhs := *v, op.scalar
|
||||
if op.scalarOnLeft {
|
||||
lhs, rhs = op.scalar, *v
|
||||
}
|
||||
switch op.op {
|
||||
case parser.ADD:
|
||||
res := lhs + rhs
|
||||
values[i] = &res
|
||||
case parser.SUB:
|
||||
res := lhs - rhs
|
||||
values[i] = &res
|
||||
case parser.MUL:
|
||||
res := lhs * rhs
|
||||
values[i] = &res
|
||||
case parser.DIV:
|
||||
res := lhs / rhs
|
||||
values[i] = &res
|
||||
case parser.MOD:
|
||||
res := math.Mod(lhs, rhs)
|
||||
values[i] = &res
|
||||
case parser.POW:
|
||||
res := math.Pow(lhs, rhs)
|
||||
values[i] = &res
|
||||
default:
|
||||
keep := compare(op.op, lhs, rhs)
|
||||
switch {
|
||||
case op.returnBool:
|
||||
res := 0.0
|
||||
if keep {
|
||||
res = 1.0
|
||||
}
|
||||
values[i] = &res
|
||||
case keep:
|
||||
// Filter comparisons keep the vector-side value.
|
||||
vec := *v
|
||||
values[i] = &vec
|
||||
default:
|
||||
values[i] = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compare(op parser.ItemType, lhs, rhs float64) bool {
|
||||
switch op {
|
||||
case parser.EQLC:
|
||||
return lhs == rhs
|
||||
case parser.NEQ:
|
||||
return lhs != rhs
|
||||
case parser.GTR:
|
||||
return lhs > rhs
|
||||
case parser.LSS:
|
||||
return lhs < rhs
|
||||
case parser.GTE:
|
||||
return lhs >= rhs
|
||||
case parser.LTE:
|
||||
return lhs <= rhs
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// toMatrix converts a unit result to a promql matrix on the query grid.
|
||||
func toMatrix(series []transpiledSeries, startMs, stepMs int64) promql.Matrix {
|
||||
matrix := make(promql.Matrix, 0, len(series))
|
||||
for _, s := range series {
|
||||
var floats []promql.FPoint
|
||||
for i, v := range s.values {
|
||||
if v == nil {
|
||||
continue
|
||||
}
|
||||
floats = append(floats, promql.FPoint{T: startMs + int64(i)*stepMs, F: *v})
|
||||
}
|
||||
if len(floats) == 0 {
|
||||
continue
|
||||
}
|
||||
matrix = append(matrix, promql.Series{Metric: s.lset, Floats: floats})
|
||||
}
|
||||
return matrix
|
||||
}
|
||||
|
||||
// executeHybrid substitutes each unit's grids into the engine as synthetic
|
||||
// series and evaluates the rewritten query over a storage that serves
|
||||
// synthetic selectors from memory and everything else from the live querier.
|
||||
// Absent grid points become stale markers so the engine's lookback cannot
|
||||
// resurrect the previous grid point. Each unit's synthetic samples sit on its
|
||||
// own grid (query grid, or subquery grid for units inside subqueries).
|
||||
func (e *executor) executeHybrid(ctx context.Context, plan *transpilePlan, results [][]transpiledSeries) (promql.Matrix, error) {
|
||||
synthetic := make(map[string][]*series, len(plan.units))
|
||||
staleMarker := math.Float64frombits(promValue.StaleNaN)
|
||||
|
||||
queryGrid := plan.grid
|
||||
|
||||
for i, unit := range plan.units {
|
||||
g := unit.grid
|
||||
gridLen := 1
|
||||
if g.stepMs > 0 {
|
||||
gridLen = int((g.endMs-g.startMs)/g.stepMs) + 1
|
||||
}
|
||||
list := make([]*series, 0, len(results[i]))
|
||||
for _, cs := range results[i] {
|
||||
builder := labels.NewBuilder(cs.lset)
|
||||
builder.Set(metricNameLabel, unit.name)
|
||||
s := &series{lset: builder.Labels()}
|
||||
s.ts = make([]int64, 0, gridLen)
|
||||
s.vs = make([]float64, 0, gridLen)
|
||||
for idx := 0; idx < gridLen; idx++ {
|
||||
t := g.startMs + int64(idx)*g.stepMs
|
||||
var v float64
|
||||
if idx < len(cs.values) && cs.values[idx] != nil {
|
||||
v = *cs.values[idx]
|
||||
} else {
|
||||
v = staleMarker
|
||||
}
|
||||
s.ts = append(s.ts, t)
|
||||
s.vs = append(s.vs, v)
|
||||
}
|
||||
list = append(list, s)
|
||||
}
|
||||
synthetic[unit.name] = list
|
||||
}
|
||||
|
||||
hybrid := &hybridQueryable{client: e.client, synthetic: synthetic}
|
||||
|
||||
var qry promql.Query
|
||||
var err error
|
||||
if queryGrid.stepMs == 0 {
|
||||
qry, err = e.engine.NewInstantQuery(ctx, hybrid, nil, plan.rewritten, time.UnixMilli(queryGrid.endMs))
|
||||
} else {
|
||||
qry, err = e.engine.NewRangeQuery(ctx, hybrid, nil, plan.rewritten, time.UnixMilli(queryGrid.startMs), time.UnixMilli(queryGrid.endMs), time.Duration(queryGrid.stepMs)*time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer qry.Close()
|
||||
|
||||
res := qry.Exec(ctx)
|
||||
if res.Err != nil {
|
||||
return nil, res.Err
|
||||
}
|
||||
|
||||
matrix, err := resultToMatrix(res)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Deep-copy before Close returns the result's slices to the engine pool,
|
||||
// and drop the synthetic __name__ that filter comparisons preserve.
|
||||
out := make(promql.Matrix, 0, len(matrix))
|
||||
for _, s := range matrix {
|
||||
lset := s.Metric
|
||||
if name := lset.Get(metricNameLabel); len(name) >= len(syntheticNamePrefix) && name[:len(syntheticNamePrefix)] == syntheticNamePrefix {
|
||||
builder := labels.NewBuilder(lset)
|
||||
builder.Del(metricNameLabel)
|
||||
lset = builder.Labels()
|
||||
}
|
||||
floats := make([]promql.FPoint, len(s.Floats))
|
||||
copy(floats, s.Floats)
|
||||
out = append(out, promql.Series{Metric: lset.Copy(), Floats: floats})
|
||||
}
|
||||
// The strip can leave twins: two units' outputs distinguishable only by
|
||||
// their synthetic names (e.g. -metric_a or -metric_b, both {} to the
|
||||
// engine's real evaluation once names dropped). The engine assembles its
|
||||
// matrix by labelset, merging such temporally-disjoint elements into one
|
||||
// series; reproduce that, with its duplicate error on same-timestamp
|
||||
// overlap.
|
||||
out, err = mergeMatrixByLabelset(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return labels.Compare(out[i].Metric, out[j].Metric) < 0 })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mergeMatrixByLabelset merges series sharing a labelset by interleaving
|
||||
// their points in timestamp order; a timestamp present in both is the
|
||||
// engine's duplicate-labelset error.
|
||||
func mergeMatrixByLabelset(matrix promql.Matrix) (promql.Matrix, error) {
|
||||
index := make(map[uint64]int, len(matrix))
|
||||
out := matrix[:0]
|
||||
for _, s := range matrix {
|
||||
hash := s.Metric.Hash()
|
||||
idx, ok := index[hash]
|
||||
if ok && labels.Equal(out[idx].Metric, s.Metric) {
|
||||
merged := make([]promql.FPoint, 0, len(out[idx].Floats)+len(s.Floats))
|
||||
a, b := out[idx].Floats, s.Floats
|
||||
for len(a) > 0 && len(b) > 0 {
|
||||
switch {
|
||||
case a[0].T < b[0].T:
|
||||
merged, a = append(merged, a[0]), a[1:]
|
||||
case b[0].T < a[0].T:
|
||||
merged, b = append(merged, b[0]), b[1:]
|
||||
default:
|
||||
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "vector cannot contain metrics with the same labelset")
|
||||
}
|
||||
}
|
||||
out[idx].Floats = append(append(merged, a...), b...)
|
||||
continue
|
||||
}
|
||||
index[hash] = len(out)
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func resultToMatrix(res *promql.Result) (promql.Matrix, error) {
|
||||
switch v := res.Value.(type) {
|
||||
case promql.Matrix:
|
||||
return v, nil
|
||||
case promql.Vector:
|
||||
matrix := make(promql.Matrix, 0, len(v))
|
||||
for _, s := range v {
|
||||
matrix = append(matrix, promql.Series{Metric: s.Metric, Floats: []promql.FPoint{{T: s.T, F: s.F}}})
|
||||
}
|
||||
return matrix, nil
|
||||
case promql.Scalar:
|
||||
return promql.Matrix{{Metric: labels.EmptyLabels(), Floats: []promql.FPoint{{T: v.T, F: v.V}}}}, nil
|
||||
default:
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "unexpected hybrid result type %T", res.Value)
|
||||
}
|
||||
}
|
||||
|
||||
// hybridQueryable serves synthetic (compiled) selectors from memory and
|
||||
// everything else from the live storage.
|
||||
type hybridQueryable struct {
|
||||
client *client
|
||||
synthetic map[string][]*series
|
||||
}
|
||||
|
||||
func (h *hybridQueryable) Querier(mint, maxt int64) (storage.Querier, error) {
|
||||
return &hybridQuerier{
|
||||
querier: querier{mint: mint, maxt: maxt, client: h.client},
|
||||
synthetic: h.synthetic,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type hybridQuerier struct {
|
||||
querier
|
||||
synthetic map[string][]*series
|
||||
}
|
||||
|
||||
func (h *hybridQuerier) Select(ctx context.Context, sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
|
||||
if name, ok := isSyntheticSelector(matchers); ok {
|
||||
list := h.synthetic[name]
|
||||
if sortSeries {
|
||||
sorted := make([]*series, len(list))
|
||||
copy(sorted, list)
|
||||
sort.Slice(sorted, func(i, j int) bool { return labels.Compare(sorted[i].lset, sorted[j].lset) < 0 })
|
||||
list = sorted
|
||||
}
|
||||
return newSeriesSet(list)
|
||||
}
|
||||
return h.querier.Select(ctx, sortSeries, hints, matchers...)
|
||||
}
|
||||
@@ -1,412 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
// experimental gate for the timeSeries*ToGrid aggregate functions; attached
|
||||
// as a SETTINGS clause so telemetrystore hooks cannot clobber it.
|
||||
const gridFunctionsSetting = "SETTINGS allow_experimental_ts_to_grid_aggregate_function = 1"
|
||||
|
||||
var aggForEach = map[string]string{
|
||||
"sum": "sumForEach",
|
||||
"min": "minForEach",
|
||||
"max": "maxForEach",
|
||||
"avg": "avgForEach",
|
||||
"count": "countForEach",
|
||||
}
|
||||
|
||||
// buildUnitSQL renders the single ClickHouse query evaluating a core unit
|
||||
// over the [startMs, endMs] / stepMs evaluation grid: per-series grids via a
|
||||
// timeSeries*ToGrid aggregate (or a windowed aggregation for *_over_time),
|
||||
// then spatial aggregation with -ForEach combinators grouped by a canonical
|
||||
// JSON key of the projected label pairs.
|
||||
//
|
||||
// The heavy level is shaped to run on the shards: the top-level FROM is the
|
||||
// distributed samples table and the group-key join partner is a subquery on
|
||||
// the shard-local time series table, so the shard rewrite executes the join
|
||||
// and the per-(fingerprint, group key) aggregation next to the data —
|
||||
// complete by fingerprint co-locality (see localTimeSeriesTable) — and the
|
||||
// initiator only merges the per-series states and applies the spatial
|
||||
// -ForEach step. Same layout as the telemetrymetrics statement builder.
|
||||
// The windowed *_over_time form shares the frame but aggregates per
|
||||
// (series, group key, step bucket) instead of straight to grids
|
||||
// (see windowedInner).
|
||||
//
|
||||
// The selector's data window is offset-shifted; the resulting grid indices
|
||||
// map 1:1 onto the query grid (output ts = startMs + i*stepMs). Grid
|
||||
// parameters are rendered as literals — they are aggregate-function
|
||||
// parameters, not bindable values.
|
||||
//
|
||||
// Statements nest builder-rendered SQL as text, so the returned args must be
|
||||
// ordered by where each fragment lands in the final statement: ClickHouse
|
||||
// binds ? placeholders by position. A JOIN renders before WHERE, so a joined
|
||||
// subquery's args precede the outer query's own condition args.
|
||||
//
|
||||
// Row shape: the group-key columns (see groupKeyColumns) followed by
|
||||
// grid Array(Nullable(Float64)); NULL grid points are absent points (the
|
||||
// engine's "no value here"), which the -ForEach combinators preserve: an
|
||||
// index where every series is NULL aggregates to NULL, and countForEach's 0
|
||||
// is mapped back to NULL.
|
||||
func buildUnitSQL(unit *coreUnit, metricNames []string, dataStart, dataEnd int64, startMs, endMs, stepMs, lookbackMs int64) (string, []any, error) {
|
||||
selStart := startMs - unit.offsetMs
|
||||
selEnd := endMs - unit.offsetMs
|
||||
stepSec := stepMs / 1000
|
||||
if stepSec == 0 {
|
||||
// Instant query: start == end, so the grid has one point for any
|
||||
// positive step.
|
||||
stepSec = 1
|
||||
}
|
||||
windowMs := unit.rangeMs
|
||||
if unit.kind == unitInstant {
|
||||
windowMs = lookbackMs
|
||||
}
|
||||
windowSec := windowMs / 1000
|
||||
|
||||
adjustedTsStart, tsTable := timeSeriesTableFor(dataStart, dataEnd)
|
||||
keyNames := groupKeyColumns(unit)
|
||||
|
||||
// seriesSub computes fingerprint -> group key columns. It reads the
|
||||
// local series table when it rides inside the shard-rewritten samples
|
||||
// query, and the distributed one when it joins at the initiator
|
||||
// (windowed form).
|
||||
seriesSub := func(table string) (string, []any, error) {
|
||||
sub := sqlbuilder.NewSelectBuilder()
|
||||
selects := []string{"fingerprint"}
|
||||
if keyNames == nil {
|
||||
selects = append(selects, groupKeyExpr(unit)+" AS gkey")
|
||||
} else {
|
||||
// by (...) grouping extracts exactly the listed labels as plain
|
||||
// columns: no reason to build, sort and stringify every label
|
||||
// pair per row when the projection is a known short list and
|
||||
// the label names live in Go anyway.
|
||||
for i, name := range keyNames {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, %s) AS g%d", sub.Var(name), i))
|
||||
}
|
||||
}
|
||||
sub.Select(selects...)
|
||||
sub.From(fmt.Sprintf("%s.%s", databaseName, table))
|
||||
if err := applySeriesConditions(sub, adjustedTsStart, dataEnd, unit.matchers); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
sub.GroupBy(append([]string{"fingerprint"}, keyColumnAliases(keyNames)...)...)
|
||||
q, args := sub.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return q, args, nil
|
||||
}
|
||||
|
||||
// samplesConditions adds the samples-side WHERE. The group-key join
|
||||
// restricts to the matched series; no fingerprint condition is added
|
||||
// here.
|
||||
samplesConditions := func(sb *sqlbuilder.SelectBuilder, excludeStale bool) {
|
||||
switch len(metricNames) {
|
||||
case 0:
|
||||
// No name constraint derivable; correct but unable to use the
|
||||
// metric_name primary-key prefix.
|
||||
case 1:
|
||||
sb.Where(sb.EQ("metric_name", metricNames[0]))
|
||||
default:
|
||||
sb.Where(sb.In("metric_name", sqlbuilder.List(metricNames)))
|
||||
}
|
||||
// temporality precedes metric_name in the samples primary key; the
|
||||
// fingerprints already come from these temporalities, so this only
|
||||
// helps granule pruning.
|
||||
sb.Where("temporality IN ['Cumulative', 'Unspecified']")
|
||||
// When the window is narrower than the step, the grid windows
|
||||
// (t_k − window, t_k] cover only window/step of the selector's
|
||||
// timeline; a sample in a gap belongs to no window and cannot move
|
||||
// any grid point, but the grid aggregate buffers every row it is
|
||||
// fed. Keeping only in-window rows cut a 36k-series 1w rate from
|
||||
// 74s/28GiB to 16s/4.3GiB on fleet data — the read stays the same,
|
||||
// the aggregate input shrinks by the coverage ratio. The lattice
|
||||
// anchors at selStart (end may sit off-lattice on unaligned grids),
|
||||
// positiveModulo because samples above selStart make the dividend
|
||||
// negative, and the upper bound tightens to the last grid point —
|
||||
// rows past it are equally windowless. window >= step tiles the
|
||||
// timeline and keeps today's plain bounds.
|
||||
sliver := stepMs > 0 && windowMs > 0 && windowMs < stepMs
|
||||
upper := selEnd
|
||||
if sliver {
|
||||
upper = selStart + (selEnd-selStart)/stepMs*stepMs
|
||||
}
|
||||
// Left-open window: a sample exactly at the window's lower boundary
|
||||
// is never used (range selectors and lookback are both left-open).
|
||||
sb.Where(sb.GT("unix_milli", selStart-windowMs), sb.LTE("unix_milli", upper))
|
||||
if sliver {
|
||||
sb.Where(fmt.Sprintf("positiveModulo(%s - unix_milli, %s) < %s",
|
||||
sb.Var(selStart), sb.Var(stepMs), sb.Var(windowMs)))
|
||||
}
|
||||
if excludeStale {
|
||||
// PromQL excludes stale markers from range vectors. Instant
|
||||
// selectors need the stale rows for shadowing instead.
|
||||
sb.Where("bitAnd(flags, 1) = 0")
|
||||
}
|
||||
}
|
||||
|
||||
keyCols := keyColumnAliases(keyNames)
|
||||
|
||||
// joinedInner builds the shard-side SELECT for the single-pass kinds:
|
||||
// grid expression per (fingerprint, group key), group-key join against
|
||||
// the local series table.
|
||||
joinedInner := func(gridExpr string, excludeStale bool) (string, []any, error) {
|
||||
seriesSQL, seriesArgs, err := seriesSub(localTimeSeriesTable(tsTable))
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
selects := make([]string, 0, len(keyCols)+1)
|
||||
// A fingerprint is the hash of one labelset, so every group-key
|
||||
// column is functionally dependent on it: any() is exact, and
|
||||
// grouping by the fingerprint alone spares hashing the joined
|
||||
// string per sample row — measured -10-13% on a 1.9B-row rate.
|
||||
for _, col := range keyCols {
|
||||
selects = append(selects, fmt.Sprintf("any(series.%s) AS %s", col, col))
|
||||
}
|
||||
sb.Select(append(selects, gridExpr+" AS grid")...)
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", databaseName, distributedSamplesV4))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, fmt.Sprintf("(%s) AS series", seriesSQL), "points.fingerprint = series.fingerprint")
|
||||
samplesConditions(sb, excludeStale)
|
||||
sb.GroupBy("points.fingerprint")
|
||||
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
// The join text renders before WHERE: its args come first.
|
||||
return q, append(seriesArgs, args...), nil
|
||||
}
|
||||
|
||||
var inner string
|
||||
var innerArgs []any
|
||||
var err error
|
||||
switch unit.kind {
|
||||
case unitInstant:
|
||||
// Instant selection with stale shadowing: the grid value is the last
|
||||
// non-stale sample in (t-lookback, t], absent when the overall last
|
||||
// sample in that window is a stale marker (verified semantics: the
|
||||
// -If combinator applies to the grid aggregates, and NULL comparisons
|
||||
// make a stale-latest point absent).
|
||||
gridParams := fmt.Sprintf("(fromUnixTimestamp64Milli(%d), fromUnixTimestamp64Milli(%d), %d, %d)", selStart, selEnd, stepSec, windowSec)
|
||||
gridExpr := fmt.Sprintf(
|
||||
"arrayMap((tall, tok, vok) -> if(tall IS NULL OR tok IS NULL OR tall != tok, NULL, vok), timeSeriesLastToGrid%s(fromUnixTimestamp64Milli(unix_milli), toFloat64(unix_milli)), timeSeriesLastToGridIf%s(fromUnixTimestamp64Milli(unix_milli), toFloat64(unix_milli), bitAnd(flags, 1) = 0), timeSeriesLastToGridIf%s(fromUnixTimestamp64Milli(unix_milli), value, bitAnd(flags, 1) = 0))",
|
||||
gridParams, gridParams, gridParams,
|
||||
)
|
||||
inner, innerArgs, err = joinedInner(gridExpr, false)
|
||||
case unitOverTime:
|
||||
if unit.overFn == "last" {
|
||||
// last_over_time == last non-stale sample in the window: the
|
||||
// stale rows are already excluded in WHERE.
|
||||
gridExpr := fmt.Sprintf(
|
||||
"timeSeriesLastToGrid(fromUnixTimestamp64Milli(%d), fromUnixTimestamp64Milli(%d), %d, %d)(fromUnixTimestamp64Milli(unix_milli), value)",
|
||||
selStart, selEnd, stepSec, windowSec,
|
||||
)
|
||||
inner, innerArgs, err = joinedInner(gridExpr, true)
|
||||
break
|
||||
}
|
||||
inner, innerArgs, err = windowedInner(unit, samplesConditions, seriesSub, keyCols, localTimeSeriesTable(tsTable), selStart, selEnd, stepMs, windowMs)
|
||||
default: // unitRange
|
||||
gridExpr := fmt.Sprintf(
|
||||
"%s(fromUnixTimestamp64Milli(%d), fromUnixTimestamp64Milli(%d), %d, %d)(fromUnixTimestamp64Milli(unix_milli), value)",
|
||||
gridFunction[unit.fn], selStart, selEnd, stepSec, windowSec,
|
||||
)
|
||||
if unit.fn == fnIncrease {
|
||||
// increase == rate * range-seconds, exactly: extrapolatedRate
|
||||
// divides by the range only when isRate.
|
||||
gridExpr = fmt.Sprintf("arrayMap(x -> x * %d, %s)", windowSec, gridExpr)
|
||||
}
|
||||
inner, innerArgs, err = joinedInner(gridExpr, true)
|
||||
}
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
spatial := "maxForEach(grid)"
|
||||
switch {
|
||||
case !unit.hasAgg:
|
||||
// Per-series output: one row per (labels-minus-__name__) group.
|
||||
// Distinct fingerprints can collapse onto the same projected label
|
||||
// set only via a regex __name__ selector over metrics with identical
|
||||
// other labels; maxForEach is a deterministic NULL-skipping merge and
|
||||
// the identity for the overwhelmingly common one-fingerprint group.
|
||||
case unit.aggOp.String() == "count":
|
||||
// count over an all-absent index is an absent point, not 0.
|
||||
spatial = "arrayMap(c -> if(c = 0, NULL, toFloat64(c)), countForEach(grid))"
|
||||
default:
|
||||
spatial = fmt.Sprintf("%s(grid)", aggForEach[unit.aggOp.String()])
|
||||
}
|
||||
|
||||
keyList := strings.Join(keyCols, ", ")
|
||||
query := fmt.Sprintf("SELECT %s, %s AS grid FROM (%s) GROUP BY %s %s", keyList, spatial, inner, keyList, gridFunctionsSetting)
|
||||
return query, innerArgs, nil
|
||||
}
|
||||
|
||||
// groupKeyColumns returns the label names to extract as plain group-key
|
||||
// columns, or nil when the unit needs the canonical JSON key instead. Only
|
||||
// by (...) grouping qualifies: its projection is a known short list, so
|
||||
// extracting each label directly beats building, sorting and stringifying
|
||||
// every label pair per row. without and no-aggregation project a label SET
|
||||
// that varies per series — there the sorted-JSON key is load-bearing: the
|
||||
// sort is what makes two fingerprints with different stored JSON key order
|
||||
// land in one group, and the string carries the labels back out.
|
||||
func groupKeyColumns(unit *coreUnit) []string {
|
||||
if unit.hasAgg && unit.by && len(unit.grouping) > 0 {
|
||||
return unit.grouping
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keyColumnAliases names the group-key columns in every SELECT level: g0..gN
|
||||
// for direct extraction, the single canonical gkey otherwise.
|
||||
func keyColumnAliases(keyNames []string) []string {
|
||||
if keyNames == nil {
|
||||
return []string{"gkey"}
|
||||
}
|
||||
cols := make([]string, len(keyNames))
|
||||
for i := range keyNames {
|
||||
cols[i] = fmt.Sprintf("g%d", i)
|
||||
}
|
||||
return cols
|
||||
}
|
||||
|
||||
// windowedInner builds the avg/min/max/sum/count _over_time form without
|
||||
// fanning samples out. It runs only when the range is a whole multiple of
|
||||
// the step (see the transpile gate), because then the window
|
||||
// (t_k - range, t_k] is exactly the union of W = range/step step buckets —
|
||||
// both are left-open on the same boundaries — so bucket membership fully
|
||||
// determines window membership. Fanning each sample into all W windows it
|
||||
// covers (ARRAY JOIN) multiplies rows by W, which at long ranges over short
|
||||
// steps is a row explosion measured in billions.
|
||||
//
|
||||
// The bucketing itself is the -Resample combinator: one group per (series,
|
||||
// group key) whose state is a fixed array of per-bucket aggregates, updated
|
||||
// in place per sample. Grouping by (series, bucket) instead — measured on a
|
||||
// 100k-series x 371-bucket workload — creates a 37M-entry hash aggregation
|
||||
// whose per-thread partial tables scale memory WITH max_threads (12 -> 48
|
||||
// GiB from 2 to 8 threads, dead at 16) and ships one row per group to the
|
||||
// initiator; the Resample form carries the same numbers in 100k compact
|
||||
// array states, like every other unit kind.
|
||||
//
|
||||
// The wrapper level slides the window: slot k combines buckets k..k+W-1 by
|
||||
// direct aggregation over at most W partials — no prefix-sum tricks, so no
|
||||
// large-minus-large cancellation against the engine's directly-summed
|
||||
// windows. A slot with zero window count is absent, which also keeps
|
||||
// min/max honest: their slices filter on the bucket counts, so an empty
|
||||
// bucket's zero-fill can never be mistaken for a value (a real sample can
|
||||
// legitimately be 0 or +Inf).
|
||||
func windowedInner(unit *coreUnit, samplesConditions func(*sqlbuilder.SelectBuilder, bool), seriesSub func(string) (string, []any, error), keyCols []string, localSeriesTable string, selStart, selEnd, stepMs, windowMs int64) (string, []any, error) {
|
||||
effStepMs := stepMs
|
||||
if effStepMs == 0 {
|
||||
effStepMs = 1000
|
||||
}
|
||||
lastIdx := (selEnd - selStart) / effStepMs
|
||||
gridLen := lastIdx + 1
|
||||
w := windowMs / effStepMs
|
||||
bucketLen := gridLen + w
|
||||
|
||||
// A window narrower than the step makes the windows (t_k - range, t_k]
|
||||
// pairwise disjoint: there is nothing to slide, each slot reads exactly
|
||||
// its own window's aggregate. This is exact ONLY over sliver-filtered
|
||||
// rows (samplesConditions adds the window<step predicate): the index
|
||||
// below assigns every gap sample to the window above it, and the filter
|
||||
// is what removes them. Requires a real step — instant queries carry no
|
||||
// sliver filter, so they keep the tiled form and its gates.
|
||||
disjoint := stepMs > 0 && windowMs < stepMs
|
||||
if disjoint {
|
||||
w = 1
|
||||
bucketLen = gridLen
|
||||
}
|
||||
|
||||
seriesSQL, seriesArgs, err := seriesSub(localSeriesTable)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
// Bucket index, shifted so the earliest in-window sample lands at 0:
|
||||
// jj = ceil((ts - selStart)/step) + W - 1, folded into one intDiv. Slot
|
||||
// k's window is then buckets jj in [k, k+W-1]. In the disjoint form the
|
||||
// same ceil lands each in-window sample directly on its slot (W = 1),
|
||||
// and the numerator stays positive: the fetch floor is
|
||||
// selStart - range > selStart - step.
|
||||
jjShift := windowMs
|
||||
if disjoint {
|
||||
jjShift = effStepMs
|
||||
}
|
||||
jj := fmt.Sprintf("intDiv(unix_milli - %d + %d - 1, %d)", selStart, jjShift, effStepMs)
|
||||
buckets := sqlbuilder.NewSelectBuilder()
|
||||
selects := make([]string, 0, len(keyCols)+2)
|
||||
// any() over the group key: exact because the key is functionally
|
||||
// dependent on the fingerprint (see joinedInner).
|
||||
for _, col := range keyCols {
|
||||
selects = append(selects, fmt.Sprintf("any(series.%s) AS %s", col, col))
|
||||
}
|
||||
selects = append(selects, fmt.Sprintf("countResample(0, %d, 1)(value, %s) AS cnts", bucketLen, jj))
|
||||
if unit.overFn != "count" {
|
||||
selects = append(selects, fmt.Sprintf("%sResample(0, %d, 1)(value, %s) AS vals", map[string]string{
|
||||
"avg": "sum",
|
||||
"sum": "sum",
|
||||
"min": "min",
|
||||
"max": "max",
|
||||
}[unit.overFn], bucketLen, jj))
|
||||
}
|
||||
buckets.Select(selects...)
|
||||
buckets.From(fmt.Sprintf("%s.%s AS points", databaseName, distributedSamplesV4))
|
||||
buckets.JoinWithOption(sqlbuilder.InnerJoin, fmt.Sprintf("(%s) AS series", seriesSQL), "points.fingerprint = series.fingerprint")
|
||||
samplesConditions(buckets, true)
|
||||
buckets.GroupBy("points.fingerprint")
|
||||
bucketsSQL, bucketsArgs := buckets.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
windowCnt := fmt.Sprintf("arraySum(arraySlice(cnts, k + 1, %d))", w)
|
||||
var slot string
|
||||
switch unit.overFn {
|
||||
case "count":
|
||||
slot = fmt.Sprintf("if(%s = 0, NULL, toFloat64(%s))", windowCnt, windowCnt)
|
||||
case "sum":
|
||||
slot = fmt.Sprintf("if(%s = 0, NULL, arraySum(arraySlice(vals, k + 1, %d)))", windowCnt, w)
|
||||
case "avg":
|
||||
slot = fmt.Sprintf("if(%s = 0, NULL, arraySum(arraySlice(vals, k + 1, %d)) / %s)", windowCnt, w, windowCnt)
|
||||
case "min":
|
||||
slot = fmt.Sprintf("if(%s = 0, NULL, arrayMin(arrayFilter((v, c) -> c > 0, arraySlice(vals, k + 1, %d), arraySlice(cnts, k + 1, %d))))", windowCnt, w, w)
|
||||
case "max":
|
||||
slot = fmt.Sprintf("if(%s = 0, NULL, arrayMax(arrayFilter((v, c) -> c > 0, arraySlice(vals, k + 1, %d), arraySlice(cnts, k + 1, %d))))", windowCnt, w, w)
|
||||
}
|
||||
|
||||
keyList := strings.Join(keyCols, ", ")
|
||||
inner := fmt.Sprintf(
|
||||
"SELECT %s, arrayMap(k -> %s, range(toUInt64(%d))) AS grid FROM (%s)",
|
||||
keyList, slot, gridLen, bucketsSQL,
|
||||
)
|
||||
return inner, append(seriesArgs, bucketsArgs...), nil
|
||||
}
|
||||
|
||||
// groupKeyExpr renders the canonical JSON group key for the units whose
|
||||
// projected label SET varies per series (see groupKeyColumns): the sorted
|
||||
// [key, value] pairs of the projected labels, JSON-encoded.
|
||||
// - by () with no labels: one constant group;
|
||||
// - without (a, b): keep everything except the listed labels and __name__;
|
||||
// - no aggregation: keep everything including __name__ — even when the
|
||||
// unit drops the name from its OUTPUT, the key must keep it so distinct
|
||||
// metrics never merge in SQL; executeUnit strips the name afterwards and
|
||||
// turns a post-strip collision into the engine's duplicate-labelset
|
||||
// error instead of a silently invented merge.
|
||||
func groupKeyExpr(unit *coreUnit) string {
|
||||
// An empty label value means "label absent" in Prometheus; the stored
|
||||
// labels JSON can carry empty attribute values, which must not become
|
||||
// output labels or group keys.
|
||||
pairs := "arraySort(JSONExtractKeysAndValues(labels, 'String'))"
|
||||
if !unit.hasAgg {
|
||||
return fmt.Sprintf("toJSONString(arrayFilter(p -> p.2 != '', %s))", pairs)
|
||||
}
|
||||
if unit.by {
|
||||
// Non-empty by (...) never reaches here; groupKeyColumns extracts
|
||||
// those labels as plain columns instead.
|
||||
return "'[]'"
|
||||
}
|
||||
excluded := append([]string{metricNameLabel}, unit.grouping...)
|
||||
return fmt.Sprintf("toJSONString(arrayFilter(p -> p.2 != '' AND p.1 NOT IN (%s), %s))", quotedList(excluded), pairs)
|
||||
}
|
||||
|
||||
func quotedList(items []string) string {
|
||||
quoted := make([]string, len(items))
|
||||
for i, s := range items {
|
||||
quoted[i] = "'" + strings.ReplaceAll(s, "'", "\\'") + "'"
|
||||
}
|
||||
return strings.Join(quoted, ", ")
|
||||
}
|
||||
@@ -1,751 +0,0 @@
|
||||
package clickhouseprometheusv2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
cmock "github.com/SigNoz/clickhouse-go-mock"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestClient(t *testing.T) (*client, *telemetrystoretest.Provider) {
|
||||
t.Helper()
|
||||
store := telemetrystoretest.New(telemetrystore.Config{Provider: "clickhouse"}, sqlmock.QueryMatcherRegexp)
|
||||
settings := factory.NewScopedProviderSettings(instrumentationtest.New().ToProviderSettings(), "clickhouseprometheusv2_test")
|
||||
return newClient(settings, store, prometheus.Config{}), store
|
||||
}
|
||||
|
||||
var seriesCols = []cmock.ColumnType{
|
||||
{Name: "fingerprint", Type: "UInt64"},
|
||||
{Name: "labels", Type: "String"},
|
||||
}
|
||||
|
||||
func parse(t *testing.T, q string) parser.Expr {
|
||||
t.Helper()
|
||||
expr, err := parser.NewParser(parser.Options{}).ParseExpr(q)
|
||||
require.NoError(t, err)
|
||||
return expr
|
||||
}
|
||||
|
||||
func TestClassifyFullShapes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
check func(t *testing.T, u *coreUnit)
|
||||
}{
|
||||
{
|
||||
name: "sum by rate",
|
||||
query: `sum by (pod) (rate(http_requests_total{job="api"}[5m]))`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, fnRate, u.fn)
|
||||
assert.Equal(t, int64(300_000), u.rangeMs)
|
||||
assert.True(t, u.hasAgg)
|
||||
assert.True(t, u.by)
|
||||
assert.Equal(t, []string{"pod"}, u.grouping)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bare increase with offset",
|
||||
query: `increase(errors_total[10m] offset 30m)`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, fnIncrease, u.fn)
|
||||
assert.Equal(t, int64(1_800_000), u.offsetMs)
|
||||
assert.False(t, u.hasAgg)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "avg without over delta",
|
||||
query: `avg without (instance) (delta(gauge_metric[15m]))`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, fnDelta, u.fn)
|
||||
assert.True(t, u.hasAgg)
|
||||
assert.False(t, u.by)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scalar pipeline with comparison",
|
||||
query: `sum(rate(x[5m])) * 100 > 5`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
require.Len(t, u.ops, 2)
|
||||
assert.Equal(t, parser.ItemType(parser.MUL), u.ops[0].op)
|
||||
assert.Equal(t, 100.0, u.ops[0].scalar)
|
||||
assert.Equal(t, parser.ItemType(parser.GTR), u.ops[1].op)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "scalar on left with unary minus",
|
||||
query: `-1 * sum(rate(x[5m]))`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
require.Len(t, u.ops, 1)
|
||||
assert.True(t, u.ops[0].scalarOnLeft)
|
||||
assert.Equal(t, -1.0, u.ops[0].scalar)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bool comparison",
|
||||
query: `sum(rate(x[5m])) >= bool 0.5`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
require.Len(t, u.ops, 1)
|
||||
assert.True(t, u.ops[0].returnBool)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "irate utf8 name",
|
||||
query: `sum by ("k8s.pod.name") (irate({"k8s.container.cpu.time"}[2m]))`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, fnIRate, u.fn)
|
||||
assert.Equal(t, []string{"k8s.pod.name"}, u.grouping)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bare instant selector keeps name",
|
||||
query: `up{job="api"}`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, unitInstant, u.kind)
|
||||
assert.True(t, u.keepsName())
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gauge aggregation",
|
||||
query: `sum by (pod) (container_memory offset 5m)`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, unitInstant, u.kind)
|
||||
assert.Equal(t, int64(300_000), u.offsetMs)
|
||||
assert.True(t, u.hasAgg)
|
||||
assert.False(t, u.keepsName())
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gauge comparison keeps name",
|
||||
query: `container_memory > 100`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, unitInstant, u.kind)
|
||||
assert.True(t, u.keepsName())
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "gauge arithmetic drops name",
|
||||
query: `container_memory / 1024`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, unitInstant, u.kind)
|
||||
assert.False(t, u.keepsName())
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "avg_over_time",
|
||||
query: `max by (node) (avg_over_time(load1[10m]))`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, unitOverTime, u.kind)
|
||||
assert.Equal(t, "avg", u.overFn)
|
||||
assert.Equal(t, int64(600_000), u.rangeMs)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "last_over_time keeps name",
|
||||
query: `last_over_time(load1[10m])`,
|
||||
check: func(t *testing.T, u *coreUnit) {
|
||||
assert.Equal(t, unitOverTime, u.kind)
|
||||
assert.Equal(t, "last", u.overFn)
|
||||
assert.True(t, u.keepsName())
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
plan, ok := classify(parse(t, tt.query), testGrid(60_000))
|
||||
require.True(t, ok, "expected transpilable")
|
||||
require.True(t, plan.full, "expected full compilation")
|
||||
require.Len(t, plan.units, 1)
|
||||
tt.check(t, &plan.units[0].core)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyFallbackShapes(t *testing.T) {
|
||||
queries := []struct {
|
||||
name string
|
||||
query string
|
||||
step int64
|
||||
}{
|
||||
{"default-resolution subquery", `max_over_time(rate(x[5m])[30m:])`, 60_000},
|
||||
{"at modifier", `sum(rate(x[5m] @ 1609746000))`, 60_000},
|
||||
{"at modifier on gauge", `sum(container_memory @ 1609746000)`, 60_000},
|
||||
{"sub-second step", `sum(rate(x[5m]))`, 500},
|
||||
{"sub-second range", `sum(rate(x[1500ms]))`, 60_000},
|
||||
{"by __name__ full", `sum by (__name__) (rate({__name__=~"a|b"}[5m]))`, 60_000},
|
||||
{"quantile_over_time unsupported", `quantile_over_time(0.9, load1[10m])`, 60_000},
|
||||
// Duration expressions resolve into the selectors' static fields only
|
||||
// at evaluation time; classification reads those fields as zero, so
|
||||
// transpiling would silently use the wrong offset (caught by the
|
||||
// conformance corpus' duration_expression.test cases). Offset
|
||||
// expressions parse without the experimental-parser flag, so they do
|
||||
// reach the transpiler; range-position expressions are rejected at
|
||||
// parse (the RangeExpr/StepExpr guards are defense-in-depth).
|
||||
{"duration expression offset on instant", `x offset step()`, 60_000},
|
||||
{"duration expression offset arithmetic", `x offset -step()*2`, 60_000},
|
||||
{"duration expression offset on range", `sum(rate(x[5m] offset max(3s, step())))`, 60_000},
|
||||
{"duration expression subquery step", `max_over_time(rate(x[5m])[30m:step()])`, 60_000},
|
||||
}
|
||||
for _, tt := range queries {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, ok := classify(parse(t, tt.query), testGrid(tt.step))
|
||||
assert.False(t, ok, "expected fallback for %s", tt.query)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyHybridShapes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantUnits int
|
||||
wantRewritten string
|
||||
}{
|
||||
{
|
||||
name: "histogram quantile",
|
||||
query: `histogram_quantile(0.95, sum by (le) (rate(http_bucket[5m])))`,
|
||||
wantUnits: 1,
|
||||
wantRewritten: `histogram_quantile(0.95, __signoz_transpiled_0__)`,
|
||||
},
|
||||
{
|
||||
name: "topk over compiled",
|
||||
query: `topk(5, sum by (pod) (rate(x[5m])))`,
|
||||
wantUnits: 1,
|
||||
wantRewritten: `topk(5, __signoz_transpiled_0__)`,
|
||||
},
|
||||
{
|
||||
name: "ratio of compiled units",
|
||||
query: `sum(rate(a[5m])) / sum(rate(b[5m]))`,
|
||||
wantUnits: 2,
|
||||
wantRewritten: `__signoz_transpiled_0__ / __signoz_transpiled_1__`,
|
||||
},
|
||||
{
|
||||
name: "or vector zero",
|
||||
query: `sum(rate(a[5m])) or vector(0)`,
|
||||
wantUnits: 1,
|
||||
wantRewritten: `__signoz_transpiled_0__ or vector(0)`,
|
||||
},
|
||||
{
|
||||
name: "quantile agg over compiled rate",
|
||||
query: `quantile(0.9, rate(x[5m]))`,
|
||||
wantUnits: 1,
|
||||
wantRewritten: `quantile(0.9, __signoz_transpiled_0__)`,
|
||||
},
|
||||
{
|
||||
name: "non-literal scalar side stays engine-side",
|
||||
query: `sum(rate(x[5m])) * scalar(y)`,
|
||||
wantUnits: 1,
|
||||
wantRewritten: `__signoz_transpiled_0__ * scalar(y)`,
|
||||
},
|
||||
{
|
||||
name: "compiled mixed with raw selector",
|
||||
query: `sum by (pod) (rate(a[5m])) / on (pod) group_left () b`,
|
||||
wantUnits: 1,
|
||||
wantRewritten: `__signoz_transpiled_0__ / on (pod) group_left () b`,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
plan, ok := classify(parse(t, tt.query), testGrid(60_000))
|
||||
require.True(t, ok)
|
||||
assert.False(t, plan.full)
|
||||
assert.Len(t, plan.units, tt.wantUnits)
|
||||
assert.Equal(t, tt.wantRewritten, plan.rewritten)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyHybridGuards(t *testing.T) {
|
||||
t.Run("no substitution under on(__name__)", func(t *testing.T) {
|
||||
plan, ok := classify(parse(t, `sum(rate(a[5m])) * on (__name__) b`), testGrid(60_000))
|
||||
_ = plan
|
||||
assert.False(t, ok, "matching on __name__ must not see synthetic names")
|
||||
})
|
||||
t.Run("no substitution inside @-pinned subquery", func(t *testing.T) {
|
||||
_, ok := classify(parse(t, `max_over_time(rate(x[5m])[30m:1m] @ 1609746000)`), testGrid(60_000))
|
||||
assert.False(t, ok)
|
||||
})
|
||||
}
|
||||
|
||||
// The alert-smoothing idiom: units inside a fixed-resolution subquery
|
||||
// evaluate on the subquery grid — epoch-aligned multiples of the resolution,
|
||||
// starting strictly after (outer start - range), exactly as the engine
|
||||
// derives it.
|
||||
func TestClassifySubqueryUnits(t *testing.T) {
|
||||
grid := gridContext{startMs: 1_700_000_030_000, endMs: 1_700_007_200_000, stepMs: 60_000}
|
||||
|
||||
plan, ok := classify(parse(t, `min_over_time((sum by (ns) (increase(x[5m])))[10m:5m]) > 0`), grid)
|
||||
require.True(t, ok)
|
||||
require.False(t, plan.full)
|
||||
require.Len(t, plan.units, 1)
|
||||
assert.Equal(t, `min_over_time(__signoz_transpiled_0__[10m:5m]) > 0`, plan.rewritten)
|
||||
|
||||
unit := plan.units[0]
|
||||
// lower bound = outer start - range = 1_699_999_430_000; first multiple
|
||||
// of 300_000 strictly greater is 1_699_999_500_000.
|
||||
assert.Equal(t, int64(1_699_999_500_000), unit.grid.startMs)
|
||||
assert.Equal(t, grid.endMs, unit.grid.endMs)
|
||||
assert.Equal(t, int64(300_000), unit.grid.stepMs)
|
||||
assert.Equal(t, fnIncrease, unit.core.fn)
|
||||
|
||||
t.Run("subquery offset shifts the grid", func(t *testing.T) {
|
||||
plan, ok := classify(parse(t, `max_over_time((sum(rate(x[5m])))[10m:5m] offset 30m)`), grid)
|
||||
require.True(t, ok)
|
||||
require.Len(t, plan.units, 1)
|
||||
// lower = start - offset - range = 1_699_997_630_000 -> first
|
||||
// multiple of 300_000 above = 1_699_997_700_000; end shifts too.
|
||||
assert.Equal(t, int64(1_699_997_700_000), plan.units[0].grid.startMs)
|
||||
assert.Equal(t, grid.endMs-1_800_000, plan.units[0].grid.endMs)
|
||||
})
|
||||
|
||||
t.Run("mollusk ratio-inside-subquery idiom", func(t *testing.T) {
|
||||
q := `min_over_time(((sum by (a) (rate(m1[5m]))) / (avg by (a) (m2)))[5m:1m])`
|
||||
plan, ok := classify(parse(t, q), grid)
|
||||
require.True(t, ok)
|
||||
// Both sides compile on the subquery grid: the rate side and the
|
||||
// gauge aggregation side; the engine joins them and smooths.
|
||||
require.Len(t, plan.units, 2)
|
||||
assert.Equal(t, int64(60_000), plan.units[0].grid.stepMs)
|
||||
assert.Equal(t, unitInstant, plan.units[1].core.kind)
|
||||
assert.Contains(t, plan.rewritten, `__signoz_transpiled_0__ / __signoz_transpiled_1__`)
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildUnitSQL(t *testing.T) {
|
||||
unit := &coreUnit{
|
||||
fn: fnRate,
|
||||
rangeMs: 300_000,
|
||||
hasAgg: true,
|
||||
aggOp: parser.SUM,
|
||||
by: true,
|
||||
grouping: []string{"pod"},
|
||||
matchers: []*labels.Matcher{mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total")},
|
||||
}
|
||||
sql, args, err := buildUnitSQL(unit, []string{"http_requests_total"}, 1_699_999_700_000, 1_700_003_600_000, 1_700_000_000_000, 1_700_003_600_000, 60_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Contains(t, sql, "timeSeriesRateToGrid(fromUnixTimestamp64Milli(1700000000000), fromUnixTimestamp64Milli(1700003600000), 60, 300)(fromUnixTimestamp64Milli(unix_milli), value)")
|
||||
assert.Contains(t, sql, "unix_milli > ? AND unix_milli <= ?")
|
||||
assert.Contains(t, sql, "bitAnd(flags, 1) = 0")
|
||||
assert.Contains(t, sql, "sumForEach(grid)")
|
||||
// The group-key join rides inside the shard query: distributed samples
|
||||
// at the top level, the local series table in the join subquery, the
|
||||
// grid aggregation grouped per (fingerprint, group key) shard-side.
|
||||
assert.Contains(t, sql, "FROM signoz_metrics.distributed_samples_v4 AS points INNER JOIN (SELECT fingerprint,")
|
||||
assert.Contains(t, sql, "FROM signoz_metrics.time_series_v4 WHERE")
|
||||
// The group key is functionally dependent on the fingerprint (one
|
||||
// labelset per fingerprint): any() is exact and the per-row hash key
|
||||
// shrinks to the fingerprint alone.
|
||||
assert.Contains(t, sql, "any(series.g0) AS g0")
|
||||
assert.Contains(t, sql, "GROUP BY points.fingerprint)")
|
||||
// No samples-side fingerprint condition: the group-key join restricts.
|
||||
assert.NotContains(t, sql, "points.fingerprint IN (")
|
||||
// by (pod) extracts the grouped label directly — no per-row JSON
|
||||
// build/sort/stringify for a known projection.
|
||||
assert.Contains(t, sql, "JSONExtractString(labels, ?) AS g0")
|
||||
assert.NotContains(t, sql, "toJSONString")
|
||||
assert.Contains(t, sql, "SETTINGS allow_experimental_ts_to_grid_aggregate_function = 1")
|
||||
// Args follow placeholder order: the joined series subquery renders
|
||||
// before the samples WHERE, and its select list ('pod') renders before
|
||||
// its own conditions.
|
||||
assert.Equal(t, []any{"pod", "http_requests_total", int64(1_699_999_200_000), int64(1_700_003_600_000), "http_requests_total", int64(1_699_999_700_000), int64(1_700_003_600_000)}, args)
|
||||
}
|
||||
|
||||
func TestBuildUnitSQLIncreaseAndOffset(t *testing.T) {
|
||||
unit := &coreUnit{
|
||||
fn: fnIncrease,
|
||||
rangeMs: 600_000,
|
||||
offsetMs: 1_800_000,
|
||||
matchers: []*labels.Matcher{mustMatcher(t, labels.MatchEqual, "__name__", "errors_total")},
|
||||
}
|
||||
sql, _, err := buildUnitSQL(unit, nil, 1_699_997_600_000, 1_700_001_800_000, 1_700_000_000_000, 1_700_003_600_000, 60_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Grid and window shift by the offset; increase multiplies rate by the
|
||||
// range in seconds.
|
||||
assert.Contains(t, sql, "fromUnixTimestamp64Milli(1699998200000), fromUnixTimestamp64Milli(1700001800000)")
|
||||
assert.Contains(t, sql, "arrayMap(x -> x * 600, timeSeriesRateToGrid")
|
||||
assert.Contains(t, sql, "maxForEach(grid)")
|
||||
}
|
||||
|
||||
func TestBuildUnitSQLOverLimitJoinOnly(t *testing.T) {
|
||||
// Past the inline limit no fingerprint filter is rendered: the series
|
||||
// join restricts to the matched fingerprints on its own.
|
||||
unit := &coreUnit{
|
||||
fn: fnRate,
|
||||
rangeMs: 300_000,
|
||||
hasAgg: true,
|
||||
aggOp: parser.SUM,
|
||||
by: true,
|
||||
matchers: []*labels.Matcher{mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total")},
|
||||
}
|
||||
sql, _, err := buildUnitSQL(unit, []string{"http_requests_total"}, 1_699_999_700_000, 1_700_003_600_000, 1_700_000_000_000, 1_700_003_600_000, 60_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotContains(t, sql, "points.fingerprint IN")
|
||||
assert.Contains(t, sql, "INNER JOIN (SELECT fingerprint,")
|
||||
assert.Contains(t, sql, "FROM signoz_metrics.time_series_v4 WHERE")
|
||||
}
|
||||
|
||||
func TestBuildUnitSQLWindowSliver(t *testing.T) {
|
||||
// rate[5m] on a 30m grid evaluates only a 5m sliver before each grid
|
||||
// point — samples in the gaps belong to no window and would only be
|
||||
// buffered by the grid aggregate. The WHERE must keep exactly the
|
||||
// in-window rows: positiveModulo anchored at the selector start (end
|
||||
// can sit off-lattice on unaligned grids, and samples above the start
|
||||
// make the plain modulo dividend negative), and the scan capped at the
|
||||
// last grid point — rows past it are equally windowless.
|
||||
unit := &coreUnit{
|
||||
fn: fnRate,
|
||||
rangeMs: 300_000,
|
||||
hasAgg: true,
|
||||
aggOp: parser.SUM,
|
||||
by: true,
|
||||
grouping: []string{"pod"},
|
||||
matchers: []*labels.Matcher{mustMatcher(t, labels.MatchEqual, "__name__", "http_requests_total")},
|
||||
}
|
||||
sql, args, err := buildUnitSQL(unit, []string{"http_requests_total"}, 1_699_999_700_000, 1_700_003_600_000, 1_700_000_000_000, 1_700_003_600_000, 1_800_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Contains(t, sql, "positiveModulo(? - unix_milli, ?) < ?")
|
||||
assert.Equal(t, []any{"pod", "http_requests_total", int64(1_699_999_200_000), int64(1_700_003_600_000), "http_requests_total", int64(1_699_999_700_000), int64(1_700_003_600_000), int64(1_700_000_000_000), int64(1_800_000), int64(300_000)}, args)
|
||||
|
||||
t.Run("off-lattice end caps the scan at the last grid point", func(t *testing.T) {
|
||||
// end - start = 50m at a 30m step: the only grid points are start
|
||||
// and start+30m; samples in the trailing 20m serve no window.
|
||||
_, args, err := buildUnitSQL(unit, []string{"http_requests_total"}, 1_699_999_700_000, 1_700_003_000_000, 1_700_000_000_000, 1_700_003_000_000, 1_800_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, args, int64(1_700_001_800_000))
|
||||
})
|
||||
|
||||
t.Run("window covering the step keeps plain bounds", func(t *testing.T) {
|
||||
sql, _, err := buildUnitSQL(unit, []string{"http_requests_total"}, 1_699_999_700_000, 1_700_003_600_000, 1_700_000_000_000, 1_700_003_600_000, 60_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, sql, "positiveModulo")
|
||||
})
|
||||
}
|
||||
|
||||
func TestBuildUnitSQLWindowedBucketsWithoutFanOut(t *testing.T) {
|
||||
// The window is W = range/step whole buckets, so each sample lands in
|
||||
// exactly one bucket via GROUP BY and the window slides over bucket
|
||||
// partials — fanning samples into every covered window (ARRAY JOIN)
|
||||
// multiplies rows by W, a row explosion at long ranges.
|
||||
unit := &coreUnit{
|
||||
kind: unitOverTime,
|
||||
overFn: "avg",
|
||||
rangeMs: 600_000,
|
||||
matchers: []*labels.Matcher{mustMatcher(t, labels.MatchEqual, "__name__", "node_load1")},
|
||||
}
|
||||
sql, _, err := buildUnitSQL(unit, []string{"node_load1"}, 1_699_999_400_000, 1_700_003_600_000, 1_700_000_000_000, 1_700_003_600_000, 60_000, 600_000)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotContains(t, sql, "ARRAY JOIN")
|
||||
// One group per series with fixed per-bucket arrays (-Resample); the
|
||||
// bucket index jj = ceil((ts - start)/step) + W - 1 folded into a single
|
||||
// intDiv. Grouping by (series, bucket) instead measured 37M hash groups
|
||||
// whose per-thread partials scale memory with max_threads.
|
||||
assert.Contains(t, sql, "countResample(0, 71, 1)(value, intDiv(unix_milli - 1700000000000 + 600000 - 1, 60000)) AS cnts")
|
||||
assert.Contains(t, sql, "sumResample(0, 71, 1)(value, intDiv(unix_milli - 1700000000000 + 600000 - 1, 60000)) AS vals")
|
||||
assert.Contains(t, sql, "any(series.gkey) AS gkey")
|
||||
assert.Contains(t, sql, "GROUP BY points.fingerprint)")
|
||||
assert.NotContains(t, sql, "jj) AS jj")
|
||||
assert.Contains(t, sql, "INNER JOIN (SELECT fingerprint,")
|
||||
assert.Contains(t, sql, "FROM signoz_metrics.time_series_v4 WHERE")
|
||||
// Slide: W = 10 buckets per slot, absent when the window count is 0.
|
||||
assert.Contains(t, sql, "arraySum(arraySlice(cnts, k + 1, 10))")
|
||||
assert.Contains(t, sql, "arraySum(arraySlice(vals, k + 1, 10))")
|
||||
}
|
||||
|
||||
func TestBuildUnitSQLDisjointOverTime(t *testing.T) {
|
||||
// avg_over_time[5m] on a 30m grid: the windows are pairwise disjoint,
|
||||
// so there is no slide — one Resample bucket per grid slot, read
|
||||
// directly. Exact only together with the window-sliver predicate, which
|
||||
// removes the gap samples the ceil index would otherwise assign to the
|
||||
// window above them.
|
||||
unit := &coreUnit{
|
||||
kind: unitOverTime,
|
||||
overFn: "avg",
|
||||
rangeMs: 300_000,
|
||||
matchers: []*labels.Matcher{mustMatcher(t, labels.MatchEqual, "__name__", "node_load1")},
|
||||
}
|
||||
sql, _, err := buildUnitSQL(unit, []string{"node_load1"}, 1_699_999_700_000, 1_700_003_600_000, 1_700_000_000_000, 1_700_003_600_000, 1_800_000, 300_000)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.NotContains(t, sql, "ARRAY JOIN")
|
||||
// gridLen = 3 slots, bucket array the same length — no W tail.
|
||||
assert.Contains(t, sql, "countResample(0, 3, 1)(value, intDiv(unix_milli - 1700000000000 + 1800000 - 1, 1800000)) AS cnts")
|
||||
assert.Contains(t, sql, "sumResample(0, 3, 1)(value, intDiv(unix_milli - 1700000000000 + 1800000 - 1, 1800000)) AS vals")
|
||||
// Single-bucket window: the slide degenerates to reading one slot.
|
||||
assert.Contains(t, sql, "arraySum(arraySlice(cnts, k + 1, 1))")
|
||||
// The sliver predicate is the correctness precondition of this form.
|
||||
assert.Contains(t, sql, "positiveModulo(? - unix_milli, ?) < ?")
|
||||
}
|
||||
|
||||
// TestDisjointWindowLattice brute-forces the disjoint-form arithmetic: a
|
||||
// sample survives the sliver predicate exactly when some grid window
|
||||
// contains it, and the ceil bucket index then lands it on that window's
|
||||
// slot. This is the pure-Go mirror of the SQL expressions — the predicate
|
||||
// in samplesConditions and jj in windowedInner — over random lattices,
|
||||
// including off-lattice ends and samples beyond the last grid point.
|
||||
func TestDisjointWindowLattice(t *testing.T) {
|
||||
rng := func(seed *uint64) int64 {
|
||||
*seed = *seed*6364136223846793005 + 1442695040888963407
|
||||
return int64(*seed >> 33)
|
||||
}
|
||||
seed := uint64(42)
|
||||
for trial := 0; trial < 2000; trial++ {
|
||||
stepMs := 1_000 * (1 + rng(&seed)%3600)
|
||||
windowMs := 1 + rng(&seed)%(stepMs-1) // strictly below the step
|
||||
selStart := 1_700_000_000_000 + rng(&seed)%1_000_000
|
||||
selEnd := selStart + rng(&seed)%(50*stepMs) // end may sit off-lattice
|
||||
lastIdx := (selEnd - selStart) / stepMs
|
||||
upper := selStart + lastIdx*stepMs
|
||||
|
||||
for i := 0; i < 50; i++ {
|
||||
u := selStart - windowMs - stepMs + rng(&seed)%(selEnd-selStart+3*stepMs)
|
||||
|
||||
// Oracle: is u inside any window (t_k - window, t_k]?
|
||||
inWindow := false
|
||||
var slot int64 = -1
|
||||
for k := int64(0); k <= lastIdx; k++ {
|
||||
tk := selStart + k*stepMs
|
||||
if u > tk-windowMs && u <= tk {
|
||||
inWindow = true
|
||||
slot = k
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// The SQL: fetch bounds, then the sliver predicate
|
||||
// positiveModulo(selStart - u, step) < window.
|
||||
kept := u > selStart-windowMs && u <= upper
|
||||
if kept {
|
||||
pmod := (selStart - u) % stepMs
|
||||
if pmod < 0 {
|
||||
pmod += stepMs
|
||||
}
|
||||
kept = pmod < windowMs
|
||||
}
|
||||
|
||||
require.Equal(t, inWindow, kept,
|
||||
"sliver keep mismatch: u=%d selStart=%d step=%d window=%d", u, selStart, stepMs, windowMs)
|
||||
if !kept {
|
||||
continue
|
||||
}
|
||||
// jj = ceil((u - selStart)/step) via one intDiv; numerator is
|
||||
// positive because u > selStart - window > selStart - step.
|
||||
jj := (u - selStart + stepMs - 1) / stepMs
|
||||
require.Equal(t, slot, jj,
|
||||
"slot mismatch: u=%d selStart=%d step=%d window=%d", u, selStart, stepMs, windowMs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryExecuteRange_WindowedGateFallsBack(t *testing.T) {
|
||||
c, store := newTestClient(t)
|
||||
e := &executor{client: c, parser: prometheus.NewParser()}
|
||||
|
||||
start := time.UnixMilli(1_700_000_000_000)
|
||||
end := time.UnixMilli(1_700_003_600_000)
|
||||
|
||||
// 10m range at 90s step: the window is not a whole number of buckets.
|
||||
_, ok, err := e.TryExecuteRange(context.Background(), `avg_over_time(up[10m])`, start, end, 90*time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok, "range not divisible by step must not transpile")
|
||||
|
||||
// 1d range at 60s step: 1440 bucket combines per slot, over the cap.
|
||||
_, ok, err = e.TryExecuteRange(context.Background(), `avg_over_time(up[1d])`, start, end, time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ok, "range/step above maxWindowBuckets must not transpile")
|
||||
|
||||
// 1m range at 5m step: the windows are disjoint slivers — no
|
||||
// divisibility or width requirement, so this transpiles.
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("up", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{}))
|
||||
_, ok, err = e.TryExecuteRange(context.Background(), `avg_over_time(up[1m])`, start, end, 5*time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok, "range below step is the disjoint form and must transpile")
|
||||
}
|
||||
|
||||
func TestApplyScalarOps(t *testing.T) {
|
||||
f := func(v float64) *float64 { return &v }
|
||||
|
||||
t.Run("arithmetic chain", func(t *testing.T) {
|
||||
values := []*float64{f(2), nil, f(4)}
|
||||
applyScalarOps([]scalarOp{{op: parser.MUL, scalar: 100}, {op: parser.ADD, scalar: 1}}, values)
|
||||
require.NotNil(t, values[0])
|
||||
assert.Equal(t, 201.0, *values[0])
|
||||
assert.Nil(t, values[1])
|
||||
assert.Equal(t, 401.0, *values[2])
|
||||
})
|
||||
|
||||
t.Run("comparison filters points", func(t *testing.T) {
|
||||
values := []*float64{f(1), f(10)}
|
||||
applyScalarOps([]scalarOp{{op: parser.GTR, scalar: 5}}, values)
|
||||
assert.Nil(t, values[0])
|
||||
require.NotNil(t, values[1])
|
||||
assert.Equal(t, 10.0, *values[1], "filter comparisons keep the original value")
|
||||
})
|
||||
|
||||
t.Run("bool comparison emits 0/1", func(t *testing.T) {
|
||||
values := []*float64{f(1), f(10)}
|
||||
applyScalarOps([]scalarOp{{op: parser.GTR, scalar: 5, returnBool: true}}, values)
|
||||
assert.Equal(t, 0.0, *values[0])
|
||||
assert.Equal(t, 1.0, *values[1])
|
||||
})
|
||||
|
||||
t.Run("scalar on left division", func(t *testing.T) {
|
||||
values := []*float64{f(4)}
|
||||
applyScalarOps([]scalarOp{{op: parser.DIV, scalar: 100, scalarOnLeft: true}}, values)
|
||||
assert.Equal(t, 25.0, *values[0])
|
||||
})
|
||||
}
|
||||
|
||||
func TestLabelsFromGroupKey(t *testing.T) {
|
||||
lset, err := labelsFromGroupKey(`[["pod","api-0"],["ns","prod"]]`)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "api-0", lset.Get("pod"))
|
||||
assert.Equal(t, "prod", lset.Get("ns"))
|
||||
|
||||
empty, err := labelsFromGroupKey(`[]`)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, empty.IsEmpty())
|
||||
}
|
||||
|
||||
// testGrid is a 2h query grid ending on a round timestamp.
|
||||
func testGrid(stepMs int64) gridContext {
|
||||
return gridContext{startMs: 1_700_000_000_000, endMs: 1_700_007_200_000, stepMs: stepMs}
|
||||
}
|
||||
|
||||
// A bool comparison returns 0/1, not the sample, so the engine drops
|
||||
// __name__; keeping it would change downstream vector matching.
|
||||
func TestKeepsName_BoolComparisonDropsName(t *testing.T) {
|
||||
plan, ok := classify(parse(t, `up > bool 0`), testGrid(60_000))
|
||||
require.True(t, ok)
|
||||
assert.False(t, plan.units[0].core.keepsName())
|
||||
|
||||
plan, ok = classify(parse(t, `up > 0`), testGrid(60_000))
|
||||
require.True(t, ok)
|
||||
assert.True(t, plan.units[0].core.keepsName())
|
||||
}
|
||||
|
||||
// timeSeriesLastToGrid widens its window to max(window, step) — probed on
|
||||
// 25.12 — so Last-style units at window < step must fall back or they would
|
||||
// resurrect samples the engine's lookback already dropped.
|
||||
func TestTryExecuteRange_LastStyleWindowBelowStepTranspiles(t *testing.T) {
|
||||
// These used to fall back because timeSeriesLastToGrid widens its window
|
||||
// to max(window, step). Over sliver-filtered rows the widening is
|
||||
// harmless — the widened window intersected with the data IS the
|
||||
// lookback window — so the gate is gone and both shapes transpile. The
|
||||
// mock returns no series: the point here is the routing, the value
|
||||
// semantics are the parity suite's job.
|
||||
c, store := newTestClient(t)
|
||||
e := &executor{client: c, parser: prometheus.NewParser()}
|
||||
|
||||
start := time.UnixMilli(1_700_000_000_000)
|
||||
end := time.UnixMilli(1_700_003_600_000)
|
||||
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("up", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{}))
|
||||
_, ok, err := e.TryExecuteRange(context.Background(), `sum by (pod) (up)`, start, end, time.Hour)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok, "instant selection at step > lookback must transpile")
|
||||
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("up", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{}))
|
||||
_, ok, err = e.TryExecuteRange(context.Background(), `last_over_time(up[10m])`, start, end, time.Hour)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ok, "last_over_time at range < step must transpile")
|
||||
}
|
||||
|
||||
// Two metrics collapsing onto one labelset after the name drop, with values
|
||||
// on the same grid slot, is the engine's duplicate-labelset error; merging
|
||||
// them would invent a series no engine would produce. (Temporally disjoint
|
||||
// twins merge instead — see TestMergeSameLabelsetSeries.)
|
||||
func TestExecuteUnit_NameCollisionErrors(t *testing.T) {
|
||||
c, store := newTestClient(t)
|
||||
e := &executor{client: c, parser: prometheus.NewParser()}
|
||||
|
||||
store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("^(?:a|b)$", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{
|
||||
{uint64(1), `{"__name__":"a","job":"x"}`},
|
||||
{uint64(2), `{"__name__":"b","job":"x"}`},
|
||||
}))
|
||||
store.Mock().ExpectQuery("SELECT gkey").
|
||||
WithArgs("^(?:a|b)$", int64(1_699_999_200_000), int64(1_700_003_600_000), "a", "b", int64(1_699_999_700_000), int64(1_700_003_600_000)).
|
||||
WillReturnRows(cmock.NewRows(gkeyCols, [][]any{
|
||||
{`[["__name__","a"],["job","x"]]`, []*float64{f64(1)}},
|
||||
{`[["__name__","b"],["job","x"]]`, []*float64{f64(2)}},
|
||||
}))
|
||||
|
||||
plan, ok := classify(parse(t, `rate({__name__=~"a|b"}[5m])`), gridContext{startMs: 1_700_000_000_000, endMs: 1_700_003_600_000, stepMs: 60_000})
|
||||
require.True(t, ok)
|
||||
|
||||
_, err := e.executeUnit(context.Background(), &plan.units[0].core, plan.units[0].grid)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "vector cannot contain metrics with the same labelset")
|
||||
}
|
||||
|
||||
var gkeyCols = []cmock.ColumnType{
|
||||
{Name: "gkey", Type: "String"},
|
||||
{Name: "grid", Type: "Array(Nullable(Float64))"},
|
||||
}
|
||||
|
||||
func f64(v float64) *float64 { return &v }
|
||||
|
||||
// A nameless selector can span metrics whose series alternate in time (one
|
||||
// dies inside the lookback before the other appears); after the name drop
|
||||
// the engine merges them into ONE series and errors only when two samples
|
||||
// share an evaluation timestamp. Pinned by conformance cases
|
||||
// operators.test:994/997 (-{job="api"} over http_requests/http_errors).
|
||||
func TestMergeSameLabelsetSeries(t *testing.T) {
|
||||
f := func(v float64) *float64 { return &v }
|
||||
api := labels.FromStrings("job", "api")
|
||||
|
||||
out, err := mergeSameLabelsetSeries([]transpiledSeries{
|
||||
{lset: api, values: []*float64{f(-2), nil}},
|
||||
{lset: api, values: []*float64{nil, f(-4)}},
|
||||
{lset: labels.FromStrings("job", "web"), values: []*float64{f(7), nil}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out, 2)
|
||||
assert.Equal(t, []*float64{f(-2), f(-4)}, out[0].values, "temporally disjoint twins must merge into one series")
|
||||
|
||||
_, err = mergeSameLabelsetSeries([]transpiledSeries{
|
||||
{lset: api, values: []*float64{f(1), nil}},
|
||||
{lset: api, values: []*float64{f(2), nil}},
|
||||
})
|
||||
require.Error(t, err, "two values on one evaluation timestamp is the engine's duplicate error")
|
||||
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
|
||||
}
|
||||
|
||||
// Hybrid twin case: stripping the synthetic __name__ can leave two engine
|
||||
// output series distinguishable only by those names (-metric_a or -metric_b:
|
||||
// both {} once real names are dropped). Pinned by conformance cases
|
||||
// name_label_dropping.test:137 and operators.test:1016.
|
||||
func TestMergeMatrixByLabelset(t *testing.T) {
|
||||
empty := labels.EmptyLabels()
|
||||
|
||||
out, err := mergeMatrixByLabelset(promql.Matrix{
|
||||
{Metric: empty, Floats: []promql.FPoint{{T: 0, F: -1}}},
|
||||
{Metric: empty, Floats: []promql.FPoint{{T: 600_000, F: -4}}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, out, 1)
|
||||
assert.Equal(t, []promql.FPoint{{T: 0, F: -1}, {T: 600_000, F: -4}}, out[0].Floats)
|
||||
|
||||
_, err = mergeMatrixByLabelset(promql.Matrix{
|
||||
{Metric: empty, Floats: []promql.FPoint{{T: 0, F: -1}}},
|
||||
{Metric: empty, Floats: []promql.FPoint{{T: 0, F: -3}}},
|
||||
})
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
|
||||
}
|
||||
@@ -24,10 +24,6 @@ type Config struct {
|
||||
|
||||
// Timeout is the maximum time a query is allowed to run before being aborted.
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
|
||||
// ProviderName selects the storage provider: "clickhouse" (default) or
|
||||
// "clickhousev2".
|
||||
ProviderName string `mapstructure:"provider"`
|
||||
}
|
||||
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
@@ -41,8 +37,7 @@ func newConfig() factory.Config {
|
||||
Path: "",
|
||||
MaxConcurrent: 20,
|
||||
},
|
||||
Timeout: 2 * time.Minute,
|
||||
ProviderName: "clickhouse",
|
||||
Timeout: 2 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,15 +45,9 @@ func (c Config) Validate() error {
|
||||
if c.Timeout <= 0 {
|
||||
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
|
||||
}
|
||||
if c.ProviderName != "" && c.ProviderName != "clickhouse" && c.ProviderName != "clickhousev2" {
|
||||
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::provider must be one of [clickhouse, clickhousev2], got %q", c.ProviderName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c Config) Provider() string {
|
||||
if c.ProviderName == "" {
|
||||
return "clickhouse"
|
||||
}
|
||||
return c.ProviderName
|
||||
return "clickhouse"
|
||||
}
|
||||
|
||||
@@ -1,233 +0,0 @@
|
||||
// Package promapi serves the Prometheus HTTP query API over a
|
||||
// prometheus.Prometheus provider: /query and /query_range in the shape of
|
||||
// Prometheus' /api/v1 endpoints (https://prometheus.io/docs/prometheus/latest/querying/api/),
|
||||
// intended to be mounted under a distinguishing prefix (/prometheus/api/v1)
|
||||
// so PromQL-only endpoints are separate from the SigNoz query APIs. The
|
||||
// request and response contracts follow Prometheus: form-encoded GET/POST
|
||||
// params, {"status":"success","data":{resultType,result}} on success and
|
||||
// {"status":"error","errorType","error"} with Prometheus' status codes on
|
||||
// failure — so Prometheus-compatible clients can point at the prefix.
|
||||
package promapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
promModel "github.com/prometheus/common/model"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/util/stats"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
)
|
||||
|
||||
// Handler serves the Prometheus query API over the configured provider.
|
||||
type Handler struct {
|
||||
logger *slog.Logger
|
||||
prom prometheus.Prometheus
|
||||
}
|
||||
|
||||
func NewHandler(logger *slog.Logger, prom prometheus.Prometheus) *Handler {
|
||||
return &Handler{logger: logger, prom: prom}
|
||||
}
|
||||
|
||||
type errorType string
|
||||
|
||||
const (
|
||||
errBadData errorType = "bad_data"
|
||||
errExec errorType = "execution"
|
||||
errCanceled errorType = "canceled"
|
||||
errTimeout errorType = "timeout"
|
||||
errInternal errorType = "internal"
|
||||
)
|
||||
|
||||
type queryData struct {
|
||||
ResultType parser.ValueType `json:"resultType"`
|
||||
Result parser.Value `json:"result"`
|
||||
Stats stats.QueryStats `json:"stats,omitempty"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Status string `json:"status"`
|
||||
Data *queryData `json:"data,omitempty"`
|
||||
ErrorType errorType `json:"errorType,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// QueryRange evaluates an expression over a grid: query, start, end, step,
|
||||
// and optional timeout/stats params, all in Prometheus' formats.
|
||||
func (h *Handler) QueryRange(w http.ResponseWriter, r *http.Request) {
|
||||
start, err := parseTime(r.FormValue("start"))
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
end, err := parseTime(r.FormValue("end"))
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
if end.Before(start) {
|
||||
h.respondError(r.Context(), w, errBadData, errors.NewInvalidInputf(errors.CodeInvalidInput, "end timestamp must not be before start time"))
|
||||
return
|
||||
}
|
||||
step, err := parseDuration(r.FormValue("step"))
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
if step <= 0 {
|
||||
h.respondError(r.Context(), w, errBadData, errors.NewInvalidInputf(errors.CodeInvalidInput, "zero or negative query resolution step widths are not accepted. Try a positive integer"))
|
||||
return
|
||||
}
|
||||
// The engine materializes every point of every series; an unbounded
|
||||
// grid is an unbounded allocation. 11,000 points covers 60s resolution
|
||||
// for a week or 1h resolution for a year.
|
||||
if end.Sub(start)/step > 11000 {
|
||||
h.respondError(r.Context(), w, errBadData, errors.NewInvalidInputf(errors.CodeInvalidInput, "exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)"))
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel, err := h.contextWithTimeout(r)
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
qry, err := h.prom.Engine().NewRangeQuery(ctx, h.prom.Storage(), nil, r.FormValue("query"), start, end, step)
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
h.exec(ctx, w, r, qry)
|
||||
}
|
||||
|
||||
// Query evaluates an expression at a single instant: query and optional
|
||||
// time/timeout/stats params. A missing time evaluates at the server's now,
|
||||
// as in Prometheus.
|
||||
func (h *Handler) Query(w http.ResponseWriter, r *http.Request) {
|
||||
ts := time.Now()
|
||||
if t := r.FormValue("time"); t != "" {
|
||||
var err error
|
||||
ts, err = parseTime(t)
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel, err := h.contextWithTimeout(r)
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
qry, err := h.prom.Engine().NewInstantQuery(ctx, h.prom.Storage(), nil, r.FormValue("query"), ts)
|
||||
if err != nil {
|
||||
h.respondError(r.Context(), w, errBadData, err)
|
||||
return
|
||||
}
|
||||
h.exec(ctx, w, r, qry)
|
||||
}
|
||||
|
||||
func (h *Handler) exec(ctx context.Context, w http.ResponseWriter, r *http.Request, qry promql.Query) {
|
||||
defer qry.Close()
|
||||
res := qry.Exec(ctx)
|
||||
if res.Err != nil {
|
||||
h.logger.ErrorContext(ctx, "error evaluating promql query", errors.Attr(res.Err))
|
||||
switch res.Err.(type) {
|
||||
case promql.ErrQueryCanceled:
|
||||
h.respondError(ctx, w, errCanceled, res.Err)
|
||||
case promql.ErrQueryTimeout:
|
||||
h.respondError(ctx, w, errTimeout, res.Err)
|
||||
case promql.ErrStorage:
|
||||
h.respondError(ctx, w, errInternal, res.Err)
|
||||
default:
|
||||
h.respondError(ctx, w, errExec, res.Err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
data := &queryData{ResultType: res.Value.Type(), Result: res.Value}
|
||||
if r.FormValue("stats") != "" {
|
||||
data.Stats = stats.NewQueryStats(qry.Stats())
|
||||
}
|
||||
h.respond(ctx, w, data)
|
||||
}
|
||||
|
||||
func (h *Handler) contextWithTimeout(r *http.Request) (context.Context, context.CancelFunc, error) {
|
||||
ctx := r.Context()
|
||||
if to := r.FormValue("timeout"); to != "" {
|
||||
timeout, err := parseDuration(to)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
return ctx, cancel, nil
|
||||
}
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
return ctx, cancel, nil
|
||||
}
|
||||
|
||||
func (h *Handler) respond(ctx context.Context, w http.ResponseWriter, data *queryData) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if err := json.NewEncoder(w).Encode(&response{Status: "success", Data: data}); err != nil {
|
||||
h.logger.ErrorContext(ctx, "error writing prometheus api response", errors.Attr(err))
|
||||
}
|
||||
}
|
||||
|
||||
// respondError follows Prometheus' status-code mapping: bad_data 400,
|
||||
// execution 422, canceled/timeout 503, internal 500.
|
||||
func (h *Handler) respondError(ctx context.Context, w http.ResponseWriter, typ errorType, err error) {
|
||||
code := http.StatusInternalServerError
|
||||
switch typ {
|
||||
case errBadData:
|
||||
code = http.StatusBadRequest
|
||||
case errExec:
|
||||
code = http.StatusUnprocessableEntity
|
||||
case errCanceled, errTimeout:
|
||||
code = http.StatusServiceUnavailable
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
if encErr := json.NewEncoder(w).Encode(&response{Status: "error", ErrorType: typ, Error: err.Error()}); encErr != nil {
|
||||
h.logger.ErrorContext(ctx, "error writing prometheus api error response", errors.Attr(encErr))
|
||||
}
|
||||
}
|
||||
|
||||
// parseTime accepts Prometheus' time formats: float unix seconds or RFC3339.
|
||||
func parseTime(s string) (time.Time, error) {
|
||||
if t, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
sec, ns := math.Modf(t)
|
||||
return time.Unix(int64(sec), int64(ns*float64(time.Second))), nil
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
return time.Time{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "cannot parse %q to a valid timestamp", s)
|
||||
}
|
||||
|
||||
// parseDuration accepts Prometheus' duration formats: float seconds or a
|
||||
// duration string like 5m.
|
||||
func parseDuration(s string) (time.Duration, error) {
|
||||
if d, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
ts := d * float64(time.Second)
|
||||
if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) {
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "cannot parse %q to a valid duration. It overflows int64", s)
|
||||
}
|
||||
return time.Duration(ts), nil
|
||||
}
|
||||
if d, err := promModel.ParseDuration(s); err == nil {
|
||||
return time.Duration(d), nil
|
||||
}
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "cannot parse %q to a valid duration", s)
|
||||
}
|
||||
@@ -35,9 +35,3 @@ type StatementRecorder interface {
|
||||
type StatementCapturer interface {
|
||||
CapturingStorage() (storage.Queryable, StatementRecorder)
|
||||
}
|
||||
|
||||
// ProviderClickhouseV2 is the clickhousev2 provider name: the factory
|
||||
// registration, the prometheus::provider config value and the
|
||||
// X-SigNoz-PromQL-Provider request header all use it, so they cannot drift
|
||||
// apart.
|
||||
const ProviderClickhouseV2 = "clickhousev2"
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
)
|
||||
|
||||
type queryTraitsKey struct{}
|
||||
|
||||
// QueryTraits carries per-query facts a storage implementation cannot derive
|
||||
// from SelectHints alone. Call sites that parse the PromQL expression attach
|
||||
// traits to the context before handing it to the engine; storages treat a
|
||||
// missing traits value as "unknown" and stay conservative.
|
||||
type QueryTraits struct {
|
||||
// SubqueryFree is true when the query contains no subquery expression.
|
||||
// Subquery selectors are evaluated at the subquery's own step, but
|
||||
// SelectHints.Step always carries the top-level step, so step-aligned
|
||||
// storage optimizations (e.g. keeping only the last sample per step
|
||||
// bucket) are safe only when this is true.
|
||||
SubqueryFree bool
|
||||
}
|
||||
|
||||
// DetectQueryTraits derives QueryTraits from a parsed PromQL expression.
|
||||
func DetectQueryTraits(expr parser.Expr) QueryTraits {
|
||||
subqueryFree := true
|
||||
parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
|
||||
if _, ok := node.(*parser.SubqueryExpr); ok {
|
||||
subqueryFree = false
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return QueryTraits{SubqueryFree: subqueryFree}
|
||||
}
|
||||
|
||||
// NewContextWithQueryTraits returns a context carrying the given traits.
|
||||
func NewContextWithQueryTraits(ctx context.Context, traits QueryTraits) context.Context {
|
||||
return context.WithValue(ctx, queryTraitsKey{}, traits)
|
||||
}
|
||||
|
||||
// QueryTraitsFromContext returns the traits attached to ctx, if any.
|
||||
//
|
||||
// Context is used here, unlike for backend selection, because traits must
|
||||
// cross the promql engine to reach storage.Querier.Select, and the engine's
|
||||
// interfaces offer no other channel; the alternative is a Prometheus fork.
|
||||
func QueryTraitsFromContext(ctx context.Context) (QueryTraits, bool) {
|
||||
traits, ok := ctx.Value(queryTraitsKey{}).(QueryTraits)
|
||||
return traits, ok
|
||||
}
|
||||
@@ -50,7 +50,6 @@ func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
queryRangeRequest.PromQLProvider = req.Header.Get("X-SigNoz-PromQL-Provider")
|
||||
|
||||
// Validate the query request
|
||||
if err := queryRangeRequest.Validate(); err != nil {
|
||||
|
||||
@@ -100,20 +100,9 @@ func (q *chSQLQuery) renderVars(query string, vars map[string]qbtypes.VariableIt
|
||||
return newQuery.String(), nil
|
||||
}
|
||||
|
||||
func (q *chSQLQuery) render(ctx context.Context) (string, error) {
|
||||
rendered, err := q.renderVars(q.query.Query, q.vars, q.fromMS, q.toMS)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
querybuilder.LogIfStatementIsNotValid(ctx, q.logger, rendered)
|
||||
|
||||
return rendered, nil
|
||||
}
|
||||
|
||||
// Statement renders the SQL without executing it, for the preview path.
|
||||
func (q *chSQLQuery) Statement(ctx context.Context) (*qbtypes.Statement, error) {
|
||||
rendered, err := q.render(ctx)
|
||||
func (q *chSQLQuery) Statement(_ context.Context) (*qbtypes.Statement, error) {
|
||||
rendered, err := q.renderVars(q.query.Query, q.vars, q.fromMS, q.toMS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -135,7 +124,7 @@ func (q *chSQLQuery) Execute(ctx context.Context) (*qbtypes.Result, error) {
|
||||
elapsed += p.Elapsed
|
||||
}))
|
||||
|
||||
query, err := q.render(ctx)
|
||||
query, err := q.renderVars(q.query.Query, q.vars, q.fromMS, q.toMS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -146,11 +135,11 @@ func (q *chSQLQuery) Execute(ctx context.Context) (*qbtypes.Result, error) {
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// TODO: map the errors from ClickHouse to our error types
|
||||
payload, err := consume(rows, q.kind, nil, qbtypes.Step{}, q.query.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &qbtypes.Result{
|
||||
Type: q.kind,
|
||||
Value: payload,
|
||||
|
||||
@@ -231,7 +231,7 @@ func (q *querier) buildPreviewProviders(
|
||||
sub.CompositeQuery = qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{query}}
|
||||
}
|
||||
|
||||
built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event, promqlOptions{})
|
||||
built, _, bErr := q.buildQueries(orgID, &sub, deps, missingMetricQuerySet, event)
|
||||
if bErr != nil {
|
||||
errs[name] = bErr
|
||||
continue
|
||||
|
||||
@@ -8,19 +8,15 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -102,24 +98,6 @@ type promqlQuery struct {
|
||||
tr qbv5.TimeRange
|
||||
requestType qbv5.RequestType
|
||||
vars map[string]qbv5.VariableItem
|
||||
opts promqlOptions
|
||||
}
|
||||
|
||||
// promqlOptions is how a PromQL query relates to the clickhousev2 provider
|
||||
// (see querier.promqlOptions for where the fields come from and why they are
|
||||
// flag-gated). Both providers are nil for a plain request, so a plain
|
||||
// request costs nothing extra.
|
||||
type promqlOptions struct {
|
||||
// shadow, when set, runs the query on this provider after serving and
|
||||
// logs any result difference; the response is never affected.
|
||||
shadow *clickhouseprometheusv2.Provider
|
||||
// shadowSlots is the querier-wide admission for shadow runs, shared by
|
||||
// every query so the bound holds per process.
|
||||
shadowSlots chan struct{}
|
||||
// serve, when set, serves the response from this provider instead of the
|
||||
// default path. Comparison callers fetch the default and the pinned
|
||||
// result as two API calls and diff them.
|
||||
serve *clickhouseprometheusv2.Provider
|
||||
}
|
||||
|
||||
var _ qbv5.Query = (*promqlQuery)(nil)
|
||||
@@ -132,7 +110,6 @@ func newPromqlQuery(
|
||||
tr qbv5.TimeRange,
|
||||
requestType qbv5.RequestType,
|
||||
variables map[string]qbv5.VariableItem,
|
||||
opts promqlOptions,
|
||||
) *promqlQuery {
|
||||
return &promqlQuery{
|
||||
logger: logger,
|
||||
@@ -142,19 +119,10 @@ func newPromqlQuery(
|
||||
tr: tr,
|
||||
requestType: requestType,
|
||||
vars: variables,
|
||||
opts: opts,
|
||||
}
|
||||
}
|
||||
|
||||
func (q *promqlQuery) Fingerprint() string {
|
||||
// A pinned request must not share cache entries with default serving: a
|
||||
// cached default result would satisfy the pin without running the pinned
|
||||
// provider, and a pinned result would poison normal serving. No
|
||||
// fingerprint means no caching at all — the pin exists to observe a
|
||||
// provider, so a cache in front of it defeats the point.
|
||||
if q.opts.serve != nil {
|
||||
return ""
|
||||
}
|
||||
if q.requestType != qbv5.RequestTypeTimeSeries {
|
||||
return ""
|
||||
}
|
||||
@@ -284,16 +252,7 @@ func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.Captu
|
||||
start := int64(querybuilder.ToNanoSecs(q.tr.From))
|
||||
end := int64(querybuilder.ToNanoSecs(q.tr.To))
|
||||
|
||||
// Attach the same query traits as Execute so the captured statements
|
||||
// match what the live path would run.
|
||||
if expr, parseErr := q.parser.ParseExpr(rendered); parseErr == nil {
|
||||
ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
|
||||
}
|
||||
|
||||
capStorage, recorder := storer.CapturingStorage()
|
||||
if capStorage == nil {
|
||||
return nil, nil
|
||||
}
|
||||
qry, err := q.promEngine.Engine().NewRangeQuery(
|
||||
ctx,
|
||||
capStorage,
|
||||
@@ -337,58 +296,6 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Attach query traits so the storage can prove step-aligned optimizations
|
||||
// safe (see prometheus.QueryTraits). A parse failure surfaces below via
|
||||
// the engine with the enhanced error message.
|
||||
if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
|
||||
ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
|
||||
}
|
||||
|
||||
// Accumulate ClickHouse-side scan stats across every storage query this
|
||||
// evaluation issues (engine selectors or the compiled executor): progress
|
||||
// options propagate to each ClickHouse query through the context.
|
||||
var statsMu sync.Mutex
|
||||
var rowsScanned, bytesScanned uint64
|
||||
ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(p *clickhouse.Progress) {
|
||||
statsMu.Lock()
|
||||
rowsScanned += p.Rows
|
||||
bytesScanned += p.Bytes
|
||||
statsMu.Unlock()
|
||||
}))
|
||||
|
||||
began := time.Now()
|
||||
|
||||
// A pinned provider serves directly from it: comparison callers fetch
|
||||
// the default result and the pinned result as two API calls and diff
|
||||
// them.
|
||||
if q.opts.serve != nil {
|
||||
matrix, err := q.serveFromProvider(ctx, query, start, end)
|
||||
if err != nil {
|
||||
if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
|
||||
return nil, enhanced
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned), nil
|
||||
}
|
||||
|
||||
// When the serving provider itself is clickhousev2
|
||||
// (prometheus::provider: clickhousev2), serve the way the provider is
|
||||
// designed to serve: transpiled when the shape allows. Without this the
|
||||
// override would silently run the engine path only.
|
||||
if prov, ok := q.promEngine.(*clickhouseprometheusv2.Provider); ok {
|
||||
matrix, served, err := prov.TryExecuteRange(ctx, query, time.Unix(0, start), time.Unix(0, end), q.query.Step.Duration)
|
||||
if err != nil {
|
||||
if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
|
||||
return nil, enhanced
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if served {
|
||||
return q.toResult(matrix, nil, began, &statsMu, &rowsScanned, &bytesScanned), nil
|
||||
}
|
||||
}
|
||||
|
||||
qry, err := q.promEngine.Engine().NewRangeQuery(
|
||||
ctx,
|
||||
q.promEngine.Storage(),
|
||||
@@ -424,42 +331,11 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
|
||||
return nil, errors.WrapInternalf(promErr, errors.CodeInternal, "error getting matrix from promql query %q", query)
|
||||
}
|
||||
|
||||
if q.opts.shadow != nil {
|
||||
// Shadows detach from the request, so without admission a dashboard
|
||||
// burst would stack unbounded ClickHouse work for up to the shadow
|
||||
// timeout — the concurrency pattern behind the original outages.
|
||||
// Non-blocking: at the cap the comparison is skipped, not queued;
|
||||
// a sampled shadow stream is exactly as useful for rollout evidence.
|
||||
select {
|
||||
case q.opts.shadowSlots <- struct{}{}:
|
||||
// The engine pools the result's sample slices on Close; the
|
||||
// shadow comparison needs a stable copy of what was served.
|
||||
served := copyMatrix(matrix)
|
||||
servedIn := time.Since(began)
|
||||
go func() {
|
||||
defer func() { <-q.opts.shadowSlots }()
|
||||
q.runShadowCompare(context.WithoutCancel(ctx), query, start, end, served, servedIn)
|
||||
}()
|
||||
default:
|
||||
q.logger.DebugContext(ctx, "promql shadow skipped: at concurrency cap", slog.String("query", query))
|
||||
}
|
||||
}
|
||||
|
||||
warnings, _ := res.Warnings.AsStrings(query, 10, 0)
|
||||
return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned), nil
|
||||
}
|
||||
|
||||
// toResult converts an evaluated matrix into the v5 result shape, attaching
|
||||
// the ClickHouse scan stats accumulated during evaluation.
|
||||
func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
|
||||
// Hide only known SigNoz storage keys: label names are user data and may
|
||||
// legitimately start with "__" (e.g. __address__), so a blanket dunder
|
||||
// strip mangles user labelsets. The __scope./__resource. prefixes cover
|
||||
// every exporter version's keys.
|
||||
excludeLabel := func(labelName string) bool {
|
||||
return labelName == "__temporality__" ||
|
||||
strings.HasPrefix(labelName, "__scope.") ||
|
||||
strings.HasPrefix(labelName, "__resource.")
|
||||
if labelName == "__name__" {
|
||||
return false
|
||||
}
|
||||
return strings.HasPrefix(labelName, "__") || labelName == "fingerprint"
|
||||
}
|
||||
|
||||
var series []*qbv5.TimeSeries
|
||||
@@ -487,13 +363,7 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
|
||||
series = append(series, &s)
|
||||
}
|
||||
|
||||
statsMu.Lock()
|
||||
stats := qbv5.ExecStats{
|
||||
RowsScanned: *rowsScanned,
|
||||
BytesScanned: *bytesScanned,
|
||||
DurationMS: uint64(time.Since(began).Milliseconds()),
|
||||
}
|
||||
statsMu.Unlock()
|
||||
warnings, _ := res.Warnings.AsStrings(query, 10, 0)
|
||||
|
||||
tsData := &qbv5.TimeSeriesData{
|
||||
QueryName: q.query.Name,
|
||||
@@ -527,6 +397,6 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti
|
||||
Type: q.requestType,
|
||||
Value: payload,
|
||||
Warnings: warnings,
|
||||
Stats: stats,
|
||||
}
|
||||
// TODO: map promql stats?
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
@@ -441,15 +440,3 @@ func TestQuotedMetricOutsideBracesPattern(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A pinned request must not share cache entries with default serving: a
|
||||
// cached default result would satisfy the pin without running the pinned
|
||||
// provider.
|
||||
func TestFingerprint_PinnedProviderBypassesCache(t *testing.T) {
|
||||
q := &promqlQuery{
|
||||
logger: slog.Default(),
|
||||
query: qbv5.PromQuery{Query: "up"},
|
||||
opts: promqlOptions{serve: &clickhouseprometheusv2.Provider{}},
|
||||
}
|
||||
assert.Empty(t, q.Fingerprint())
|
||||
}
|
||||
|
||||
@@ -1,186 +0,0 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
)
|
||||
|
||||
// shadowTimeout bounds a shadow evaluation; a shadow run must never outlive
|
||||
// the request by much or pile up.
|
||||
const shadowTimeout = 2 * time.Minute
|
||||
|
||||
// runShadowCompare executes the query on the clickhousev2 provider exactly
|
||||
// as it would serve (transpiled when the shape allows, engine over the v2
|
||||
// querier otherwise), compares against the served result and logs the
|
||||
// outcome. Serving is never affected: this runs after the response, off the
|
||||
// request context, and only logs. The mismatch and failure logs are the
|
||||
// rollout evidence — serving cuts over to v2 only after they stay clean.
|
||||
func (q *promqlQuery) runShadowCompare(ctx context.Context, query string, startNs, endNs int64, served promql.Matrix, servedIn time.Duration) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
q.logger.ErrorContext(ctx, "promql shadow comparison panicked", slog.Any("panic", r), slog.String("query", query))
|
||||
}
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, shadowTimeout)
|
||||
defer cancel()
|
||||
|
||||
// The request context carries the served response's scan-stats progress
|
||||
// callback; without replacing it the shadow's ClickHouse progress would
|
||||
// race into the served stats. The response itself was already sent.
|
||||
ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(*clickhouse.Progress) {}))
|
||||
|
||||
if expr, parseErr := q.parser.ParseExpr(query); parseErr == nil {
|
||||
ctx = prometheus.NewContextWithQueryTraits(ctx, prometheus.DetectQueryTraits(expr))
|
||||
}
|
||||
|
||||
start, end := time.Unix(0, startNs), time.Unix(0, endNs)
|
||||
began := time.Now()
|
||||
shadow, transpiled, err := executeOnProvider(ctx, q.opts.shadow, query, start, end, q.query.Step.Duration)
|
||||
shadowIn := time.Since(began)
|
||||
|
||||
logAttrs := []any{
|
||||
slog.String("query", query),
|
||||
slog.Int64("start_ms", startNs/int64(time.Millisecond)),
|
||||
slog.Int64("end_ms", endNs/int64(time.Millisecond)),
|
||||
slog.Duration("step", q.query.Step.Duration),
|
||||
slog.Bool("transpiled", transpiled),
|
||||
slog.Duration("served_in", servedIn),
|
||||
slog.Duration("shadow_in", shadowIn),
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// A shadow failure would be a serving failure after rollout; surface
|
||||
// it at the same level as a result mismatch.
|
||||
q.logger.WarnContext(ctx, "promql shadow execution failed", append(logAttrs, slog.Any("error", err))...)
|
||||
return
|
||||
}
|
||||
|
||||
servedNorm := normalizeShadowMatrix(served)
|
||||
shadowNorm := normalizeShadowMatrix(shadow)
|
||||
if diff := diffShadowMatrices(servedNorm, shadowNorm); diff != "" {
|
||||
q.logger.WarnContext(ctx, "promql shadow comparison mismatch", append(logAttrs,
|
||||
slog.String("diff", diff),
|
||||
slog.Int("served_series", len(servedNorm)),
|
||||
slog.Int("shadow_series", len(shadowNorm)),
|
||||
)...)
|
||||
return
|
||||
}
|
||||
// Matches log the timings: served_in vs shadow_in across the fleet is
|
||||
// the perf evidence for the cutover, gathered for free.
|
||||
q.logger.DebugContext(ctx, "promql shadow comparison matched", logAttrs...)
|
||||
}
|
||||
|
||||
// serveFromProvider evaluates the query the way the pinned provider would
|
||||
// serve it.
|
||||
func (q *promqlQuery) serveFromProvider(ctx context.Context, query string, startNs, endNs int64) (promql.Matrix, error) {
|
||||
matrix, _, err := executeOnProvider(ctx, q.opts.serve, query, time.Unix(0, startNs), time.Unix(0, endNs), q.query.Step.Duration)
|
||||
return matrix, err
|
||||
}
|
||||
|
||||
// executeOnProvider evaluates the query the way the provider would serve it:
|
||||
// transpiled in ClickHouse when the shape allows, the engine over the
|
||||
// provider's storage otherwise. The returned matrix is an owned copy.
|
||||
func executeOnProvider(ctx context.Context, prov *clickhouseprometheusv2.Provider, query string, start, end time.Time, step time.Duration) (promql.Matrix, bool, error) {
|
||||
matrix, ok, err := prov.TryExecuteRange(ctx, query, start, end, step)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
if ok {
|
||||
return matrix, true, nil
|
||||
}
|
||||
|
||||
qry, err := prov.Engine().NewRangeQuery(ctx, prov.Storage(), nil, query, start, end, step)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer qry.Close()
|
||||
|
||||
res := qry.Exec(ctx)
|
||||
if res.Err != nil {
|
||||
return nil, false, res.Err
|
||||
}
|
||||
matrix, err = res.Matrix()
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
// Close returns the result's sample slices to the engine pool.
|
||||
return copyMatrix(matrix), false, nil
|
||||
}
|
||||
|
||||
func copyMatrix(matrix promql.Matrix) promql.Matrix {
|
||||
out := make(promql.Matrix, 0, len(matrix))
|
||||
for _, s := range matrix {
|
||||
floats := make([]promql.FPoint, len(s.Floats))
|
||||
copy(floats, s.Floats)
|
||||
out = append(out, promql.Series{Metric: s.Metric.Copy(), Floats: floats})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// normalizeShadowMatrix sorts by label set for order-independent
|
||||
// comparison. Both providers now resolve series identity the same way
|
||||
// (empty-valued labels dropped at read, no synthetic fingerprint label
|
||||
// since the v1 series-identity fix), so labels need no normalization.
|
||||
func normalizeShadowMatrix(matrix promql.Matrix) promql.Matrix {
|
||||
out := make(promql.Matrix, 0, len(matrix))
|
||||
out = append(out, matrix...)
|
||||
sort.Slice(out, func(i, j int) bool { return labels.Compare(out[i].Metric, out[j].Metric) < 0 })
|
||||
return out
|
||||
}
|
||||
|
||||
// diffShadowMatrices returns a description of the first difference, or "".
|
||||
// Values compare with relative tolerance: spatial aggregations accumulate
|
||||
// floats in storage order, which differs between the providers in the last
|
||||
// ULP.
|
||||
func diffShadowMatrices(served, shadow promql.Matrix) string {
|
||||
const relTol = 1e-9
|
||||
if len(served) != len(shadow) {
|
||||
return fmt.Sprintf("series count: served=%d shadow=%d", len(served), len(shadow))
|
||||
}
|
||||
for i := range served {
|
||||
if labels.Compare(served[i].Metric, shadow[i].Metric) != 0 {
|
||||
return fmt.Sprintf("series %d labels: served=%s shadow=%s", i, served[i].Metric, shadow[i].Metric)
|
||||
}
|
||||
if len(served[i].Floats) != len(shadow[i].Floats) {
|
||||
return fmt.Sprintf("series %s points: served=%d shadow=%d", served[i].Metric, len(served[i].Floats), len(shadow[i].Floats))
|
||||
}
|
||||
for j := range served[i].Floats {
|
||||
a, b := served[i].Floats[j], shadow[i].Floats[j]
|
||||
if a.T != b.T {
|
||||
return fmt.Sprintf("series %s point %d ts: served=%d shadow=%d", served[i].Metric, j, a.T, b.T)
|
||||
}
|
||||
// NaN and infinities first: NaN != NaN and Inf-Inf arithmetic
|
||||
// would otherwise make one-sided NaN and Inf-vs-finite compare
|
||||
// as equal (NaN > x and Inf > Inf are both false).
|
||||
if math.IsNaN(a.F) || math.IsNaN(b.F) {
|
||||
if math.IsNaN(a.F) != math.IsNaN(b.F) {
|
||||
return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if math.IsInf(a.F, 0) || math.IsInf(b.F, 0) {
|
||||
if a.F != b.F {
|
||||
return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
|
||||
}
|
||||
continue
|
||||
}
|
||||
diff := math.Abs(a.F - b.F)
|
||||
scale := math.Max(math.Abs(a.F), math.Abs(b.F))
|
||||
if diff > relTol*math.Max(scale, 1e-300) && diff > 1e-12 {
|
||||
return fmt.Sprintf("series %s @%d value: served=%v shadow=%v", served[i].Metric, a.T, a.F, b.F)
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNormalizeShadowMatrix(t *testing.T) {
|
||||
matrix := promql.Matrix{
|
||||
{
|
||||
Metric: labels.FromStrings("__name__", "up", "job", "api"),
|
||||
Floats: []promql.FPoint{{T: 1000, F: 1}},
|
||||
},
|
||||
{
|
||||
Metric: labels.FromStrings("a", "1"),
|
||||
Floats: []promql.FPoint{{T: 1000, F: 2}},
|
||||
},
|
||||
}
|
||||
norm := normalizeShadowMatrix(matrix)
|
||||
// sorted by label set; labels pass through untouched — both providers
|
||||
// resolve series identity identically since the v1 series-identity fix
|
||||
assert.Equal(t, labels.FromStrings("__name__", "up", "job", "api"), norm[0].Metric)
|
||||
assert.Equal(t, labels.FromStrings("a", "1"), norm[1].Metric)
|
||||
}
|
||||
|
||||
func TestDiffShadowMatrices(t *testing.T) {
|
||||
series := func(v float64) promql.Matrix {
|
||||
return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
|
||||
}
|
||||
|
||||
assert.Empty(t, diffShadowMatrices(series(1.5), series(1.5)))
|
||||
// last-ULP differences from storage-order float accumulation are expected
|
||||
assert.Empty(t, diffShadowMatrices(series(0.08888888888888889), series(0.08888888888888888)))
|
||||
assert.Empty(t, diffShadowMatrices(series(math.NaN()), series(math.NaN())))
|
||||
|
||||
assert.Contains(t, diffShadowMatrices(series(1.5), series(1.6)), "value")
|
||||
assert.Contains(t, diffShadowMatrices(series(1.5), promql.Matrix{}), "series count")
|
||||
assert.Contains(t, diffShadowMatrices(
|
||||
series(1.5),
|
||||
promql.Matrix{{Metric: labels.FromStrings("a", "2"), Floats: []promql.FPoint{{T: 1000, F: 1.5}}}},
|
||||
), "labels")
|
||||
assert.Contains(t, diffShadowMatrices(
|
||||
series(1.5),
|
||||
promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 2000, F: 1.5}}}},
|
||||
), "ts")
|
||||
}
|
||||
|
||||
// One-sided NaN makes every float comparison false, and Inf-Inf arithmetic
|
||||
// yields Inf > Inf == false; without explicit handling both divergences log
|
||||
// as matched — a shadow comparator that cannot see them would green-light a
|
||||
// broken rollout.
|
||||
func TestDiffShadowMatrices_SpecialFloats(t *testing.T) {
|
||||
point := func(v float64) promql.Matrix {
|
||||
return promql.Matrix{{Metric: labels.FromStrings("a", "1"), Floats: []promql.FPoint{{T: 1000, F: v}}}}
|
||||
}
|
||||
|
||||
assert.NotEmpty(t, diffShadowMatrices(point(math.NaN()), point(1.5)), "one-sided NaN must diff")
|
||||
assert.NotEmpty(t, diffShadowMatrices(point(1.5), point(math.NaN())), "one-sided NaN must diff either way")
|
||||
assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(1.5)), "Inf vs finite must diff")
|
||||
assert.NotEmpty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(-1))), "opposite infinities must diff")
|
||||
assert.Empty(t, diffShadowMatrices(point(math.Inf(1)), point(math.Inf(1))), "equal infinities match")
|
||||
assert.Empty(t, diffShadowMatrices(point(math.NaN()), point(math.NaN())), "both NaN match")
|
||||
}
|
||||
@@ -19,12 +19,10 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -38,19 +36,11 @@ var (
|
||||
)
|
||||
|
||||
type querier struct {
|
||||
logger *slog.Logger
|
||||
fl flagger.Flagger
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
promEngine prometheus.Prometheus
|
||||
// promV2 is the clickhousev2 prometheus provider, wired only when the
|
||||
// serving provider is the default one (nil otherwise). It reads the same
|
||||
// ClickHouse data through a different implementation; PromQL queries
|
||||
// shadow-compare against it behind the use_prometheus_clickhouse_v2 flag
|
||||
// and can be pinned to it for a response (see promqlOptions). It never
|
||||
// serves by default — that cutover happens only after the shadow logs
|
||||
// stay clean.
|
||||
promV2 *clickhouseprometheusv2.Provider
|
||||
logger *slog.Logger
|
||||
fl flagger.Flagger
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
promEngine prometheus.Prometheus
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
|
||||
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
|
||||
@@ -61,16 +51,8 @@ type querier struct {
|
||||
liveDataRefresh time.Duration
|
||||
builderConfig builderConfig
|
||||
maxConcurrentQueries int
|
||||
// shadowSlots bounds concurrent shadow comparisons per process; shadows
|
||||
// detach from their requests, so nothing else limits how many pile up.
|
||||
shadowSlots chan struct{}
|
||||
}
|
||||
|
||||
// maxConcurrentShadows is deliberately small: a shadow is a full extra
|
||||
// ClickHouse evaluation, and a sampled stream of comparisons is exactly as
|
||||
// useful for rollout evidence as an exhaustive one under load.
|
||||
const maxConcurrentShadows = 8
|
||||
|
||||
var _ Querier = (*querier)(nil)
|
||||
|
||||
func New(
|
||||
@@ -78,7 +60,6 @@ func New(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
metadataStore telemetrytypes.MetadataStore,
|
||||
promEngine prometheus.Prometheus,
|
||||
promV2 *clickhouseprometheusv2.Provider,
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
|
||||
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
|
||||
@@ -100,7 +81,6 @@ func New(
|
||||
telemetryStore: telemetryStore,
|
||||
metadataStore: metadataStore,
|
||||
promEngine: promEngine,
|
||||
promV2: promV2,
|
||||
traceStmtBuilder: traceStmtBuilder,
|
||||
logStmtBuilder: logStmtBuilder,
|
||||
auditStmtBuilder: auditStmtBuilder,
|
||||
@@ -113,7 +93,6 @@ func New(
|
||||
logTraceIDWindowPaddingMS: uint64(logTraceIDWindowPadding.Milliseconds()),
|
||||
},
|
||||
maxConcurrentQueries: maxConcurrentQueries,
|
||||
shadowSlots: make(chan struct{}, maxConcurrentShadows),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -153,11 +132,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
|
||||
missingMetricQuerySet[name] = true
|
||||
}
|
||||
|
||||
promqlOpts, err := q.promqlOptions(ctx, orgID, req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event, promqlOpts)
|
||||
queries, steps, err := q.buildQueries(orgID, req, dependencyQueries, missingMetricQuerySet, event)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -200,41 +175,12 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
|
||||
return qbResp, qbErr
|
||||
}
|
||||
|
||||
// promqlOptions derives the PromQL execution options for a request. With the
|
||||
// org's use_prometheus_clickhouse_v2 flag on, queries are shadow-compared
|
||||
// against the clickhousev2 provider (serving unaffected, diffs logged; see
|
||||
// promql_shadow.go). The X-SigNoz-PromQL-Provider header may instead pin the
|
||||
// response to that provider — integration tests and support fetch both
|
||||
// results for comparison — so it is deliberately flag-gated too: without the
|
||||
// gate the header would be an unaudited switch onto a provider still under
|
||||
// validation.
|
||||
func (q *querier) promqlOptions(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (promqlOptions, error) {
|
||||
enabled := q.fl.BooleanOrEmpty(ctx, flagger.FeatureUsePrometheusClickhouseV2, featuretypes.NewFlaggerEvaluationContext(orgID))
|
||||
if req.PromQLProvider == "" {
|
||||
if enabled && q.promV2 != nil {
|
||||
return promqlOptions{shadow: q.promV2, shadowSlots: q.shadowSlots}, nil
|
||||
}
|
||||
return promqlOptions{}, nil
|
||||
}
|
||||
if req.PromQLProvider != prometheus.ProviderClickhouseV2 {
|
||||
return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "unknown promql provider %q", req.PromQLProvider)
|
||||
}
|
||||
if !enabled {
|
||||
return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q requires the use_prometheus_clickhouse_v2 flag", req.PromQLProvider)
|
||||
}
|
||||
if q.promV2 == nil {
|
||||
return promqlOptions{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "promql provider %q is not available", req.PromQLProvider)
|
||||
}
|
||||
return promqlOptions{serve: q.promV2}, nil
|
||||
}
|
||||
|
||||
func (q *querier) buildQueries(
|
||||
orgID valuer.UUID,
|
||||
req *qbtypes.QueryRangeRequest,
|
||||
dependencyQueries map[string]bool,
|
||||
missingMetricQuerySet map[string]bool,
|
||||
event *qbtypes.QBEvent,
|
||||
promqlOpts promqlOptions,
|
||||
) (map[string]qbtypes.Query, map[string]qbtypes.Step, error) {
|
||||
|
||||
tmplVars := req.Variables
|
||||
@@ -259,7 +205,7 @@ func (q *querier) buildQueries(
|
||||
if !ok {
|
||||
return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query spec %T", query.Spec)
|
||||
}
|
||||
promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars, promqlOpts)
|
||||
promqlQuery := newPromqlQuery(q.logger, q.promEngine, promQuery, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars)
|
||||
queries[promQuery.Name] = promqlQuery
|
||||
steps[promQuery.Name] = promQuery.Step
|
||||
case qbtypes.QueryTypeClickHouseSQL:
|
||||
@@ -902,7 +848,7 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
|
||||
switch qt := originalQuery.(type) {
|
||||
case *promqlQuery:
|
||||
queryCopy := qt.query.Copy()
|
||||
return newPromqlQuery(q.logger, qt.promEngine, queryCopy, timeRange, qt.requestType, qt.vars, qt.opts)
|
||||
return newPromqlQuery(q.logger, q.promEngine, queryCopy, timeRange, qt.requestType, qt.vars)
|
||||
|
||||
case *chSQLQuery:
|
||||
queryCopy := qt.query.Copy()
|
||||
|
||||
@@ -48,7 +48,6 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
|
||||
nil, // telemetryStore
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // promV2
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
@@ -121,7 +120,6 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // promV2
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryaudit"
|
||||
@@ -23,7 +22,6 @@ import (
|
||||
func NewFactory(
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
prometheus prometheus.Prometheus,
|
||||
promV2 *clickhouseprometheusv2.Provider,
|
||||
cache cache.Cache,
|
||||
flagger flagger.Flagger,
|
||||
) factory.ProviderFactory[querier.Querier, querier.Config] {
|
||||
@@ -34,7 +32,7 @@ func NewFactory(
|
||||
settings factory.ProviderSettings,
|
||||
cfg querier.Config,
|
||||
) (querier.Querier, error) {
|
||||
return newProvider(ctx, settings, cfg, telemetryStore, prometheus, promV2, cache, flagger)
|
||||
return newProvider(ctx, settings, cfg, telemetryStore, prometheus, cache, flagger)
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -45,7 +43,6 @@ func newProvider(
|
||||
cfg querier.Config,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
prometheus prometheus.Prometheus,
|
||||
promV2 *clickhouseprometheusv2.Provider,
|
||||
cache cache.Cache,
|
||||
flagger flagger.Flagger,
|
||||
) (querier.Querier, error) {
|
||||
@@ -183,7 +180,6 @@ func newProvider(
|
||||
telemetryStore,
|
||||
telemetryMetadataStore,
|
||||
prometheus,
|
||||
promV2,
|
||||
traceStmtBuilder,
|
||||
logStmtBuilder,
|
||||
auditStmtBuilder,
|
||||
|
||||
@@ -232,6 +232,25 @@ func NewReader(
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) GetInstantQueryMetricsResult(ctx context.Context, queryParams *model.InstantQueryMetricsParams) (*promql.Result, *stats.QueryStats, *model.ApiError) {
|
||||
qry, err := r.prometheus.Engine().NewInstantQuery(ctx, r.prometheus.Storage(), nil, queryParams.Query, queryParams.Time)
|
||||
if err != nil {
|
||||
return nil, nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
|
||||
res := qry.Exec(ctx)
|
||||
|
||||
// Optional stats field in response if parameter "stats" is not empty.
|
||||
var qs stats.QueryStats
|
||||
if queryParams.Stats != "" {
|
||||
qs = stats.NewQueryStats(qry.Stats())
|
||||
}
|
||||
|
||||
qry.Close()
|
||||
return res, &qs, nil
|
||||
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) GetQueryRangeResult(ctx context.Context, query *model.QueryRangeParams) (*promql.Result, *stats.QueryStats, *model.ApiError) {
|
||||
qry, err := r.prometheus.Engine().NewRangeQuery(ctx, r.prometheus.Storage(), nil, query.Query, query.Start, query.End, query.Step)
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/modules/thirdpartyapi"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
|
||||
"log/slog"
|
||||
@@ -26,10 +25,11 @@ import (
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/http/middleware"
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/promapi"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/integrations"
|
||||
"github.com/SigNoz/signoz/pkg/signoz"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
@@ -482,12 +482,8 @@ func (aH *APIHandler) Respond(w http.ResponseWriter, data interface{}) {
|
||||
|
||||
// RegisterRoutes registers routes for this handler on the given router
|
||||
func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *middleware.AuthZ) {
|
||||
// PromQL-only endpoints, in Prometheus' own API shape, live under a
|
||||
// /prometheus prefix so they are distinguishable from the SigNoz query
|
||||
// APIs; Prometheus-compatible clients can be pointed at the prefix.
|
||||
promAPI := promapi.NewHandler(aH.logger, aH.Signoz.Prometheus)
|
||||
router.HandleFunc("/prometheus/api/v1/query_range", am.ViewAccess(promAPI.QueryRange)).Methods(http.MethodGet, http.MethodPost)
|
||||
router.HandleFunc("/prometheus/api/v1/query", am.ViewAccess(promAPI.Query)).Methods(http.MethodGet, http.MethodPost)
|
||||
router.HandleFunc("/api/v1/query_range", am.ViewAccess(aH.queryRangeMetrics)).Methods(http.MethodGet)
|
||||
router.HandleFunc("/api/v1/query", am.ViewAccess(aH.queryMetrics)).Methods(http.MethodGet)
|
||||
router.HandleFunc("/api/v1/rules", am.ViewAccess(aH.listRules)).Methods(http.MethodGet)
|
||||
router.HandleFunc("/api/v1/rules/{id}", am.ViewAccess(aH.getRule)).Methods(http.MethodGet)
|
||||
router.HandleFunc("/api/v1/rules", am.EditAccess(aH.createRule)).Methods(http.MethodPost)
|
||||
@@ -1096,8 +1092,6 @@ func (aH *APIHandler) queryDashboardVarsV2(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
querybuilder.LogIfStatementIsNotValid(r.Context(), aH.logger, query)
|
||||
|
||||
dashboardVars, err := aH.reader.QueryDashboardVars(r.Context(), query)
|
||||
if err != nil {
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil)
|
||||
@@ -1106,6 +1100,115 @@ func (aH *APIHandler) queryDashboardVarsV2(w http.ResponseWriter, r *http.Reques
|
||||
aH.Respond(w, dashboardVars)
|
||||
}
|
||||
|
||||
func (aH *APIHandler) queryRangeMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
query, apiErrorObj := parseQueryRangeRequest(r)
|
||||
|
||||
if apiErrorObj != nil {
|
||||
RespondError(w, apiErrorObj, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: add structured logging for query and apiError if needed
|
||||
|
||||
ctx := r.Context()
|
||||
if to := r.FormValue("timeout"); to != "" {
|
||||
var cancel context.CancelFunc
|
||||
timeout, err := parseMetricsDuration(to)
|
||||
if aH.HandleError(w, err, http.StatusBadRequest) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
res, qs, apiError := aH.reader.GetQueryRangeResult(ctx, query)
|
||||
|
||||
if apiError != nil {
|
||||
RespondError(w, apiError, nil)
|
||||
return
|
||||
}
|
||||
|
||||
if res.Err != nil {
|
||||
aH.logger.ErrorContext(r.Context(), "error in query range metrics", errors.Attr(res.Err))
|
||||
}
|
||||
|
||||
if res.Err != nil {
|
||||
switch res.Err.(type) {
|
||||
case promql.ErrQueryCanceled:
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorCanceled, Err: res.Err}, nil)
|
||||
case promql.ErrQueryTimeout:
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorTimeout, Err: res.Err}, nil)
|
||||
}
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorExec, Err: res.Err}, nil)
|
||||
return
|
||||
}
|
||||
|
||||
response_data := &model.QueryData{
|
||||
ResultType: res.Value.Type(),
|
||||
Result: res.Value,
|
||||
Stats: qs,
|
||||
}
|
||||
|
||||
aH.Respond(w, response_data)
|
||||
|
||||
}
|
||||
|
||||
func (aH *APIHandler) queryMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
queryParams, apiErrorObj := parseInstantQueryMetricsRequest(r)
|
||||
|
||||
if apiErrorObj != nil {
|
||||
RespondError(w, apiErrorObj, nil)
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: add structured logging for query and apiError if needed
|
||||
|
||||
ctx := r.Context()
|
||||
if to := r.FormValue("timeout"); to != "" {
|
||||
var cancel context.CancelFunc
|
||||
timeout, err := parseMetricsDuration(to)
|
||||
if aH.HandleError(w, err, http.StatusBadRequest) {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel = context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
res, qs, apiError := aH.reader.GetInstantQueryMetricsResult(ctx, queryParams)
|
||||
|
||||
if apiError != nil {
|
||||
RespondError(w, apiError, nil)
|
||||
return
|
||||
}
|
||||
|
||||
if res.Err != nil {
|
||||
aH.logger.ErrorContext(r.Context(), "error in query range metrics", errors.Attr(res.Err))
|
||||
}
|
||||
|
||||
if res.Err != nil {
|
||||
switch res.Err.(type) {
|
||||
case promql.ErrQueryCanceled:
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorCanceled, Err: res.Err}, nil)
|
||||
case promql.ErrQueryTimeout:
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorTimeout, Err: res.Err}, nil)
|
||||
}
|
||||
RespondError(w, &model.ApiError{Typ: model.ErrorExec, Err: res.Err}, nil)
|
||||
}
|
||||
|
||||
responseData := &model.QueryData{
|
||||
ResultType: res.Value.Type(),
|
||||
Result: res.Value,
|
||||
Stats: qs,
|
||||
}
|
||||
|
||||
aH.Respond(w, responseData)
|
||||
|
||||
}
|
||||
|
||||
func (aH *APIHandler) registerEvent(w http.ResponseWriter, r *http.Request) {
|
||||
request, err := parseRegisterEventRequest(r)
|
||||
if aH.HandleError(w, err, http.StatusBadRequest) {
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
queues2 "github.com/SigNoz/signoz/pkg/query-service/app/integrations/messagingQueues/queues"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
promModel "github.com/prometheus/common/model"
|
||||
"go.uber.org/multierr"
|
||||
|
||||
errorsV2 "github.com/SigNoz/signoz/pkg/errors"
|
||||
@@ -87,6 +88,95 @@ func parseRegisterEventRequest(r *http.Request) (*model.RegisterEventParams, err
|
||||
return postData, nil
|
||||
}
|
||||
|
||||
func parseMetricsTime(s string) (time.Time, error) {
|
||||
if t, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
s, ns := math.Modf(t)
|
||||
return time.Unix(int64(s), int64(ns*float64(time.Second))), nil
|
||||
// return time.Unix(0, t), nil
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
|
||||
}
|
||||
|
||||
func parseMetricsDuration(s string) (time.Duration, error) {
|
||||
if d, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
ts := d * float64(time.Second)
|
||||
if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) {
|
||||
return 0, fmt.Errorf("cannot parse %q to a valid duration. It overflows int64", s)
|
||||
}
|
||||
return time.Duration(ts), nil
|
||||
}
|
||||
if d, err := promModel.ParseDuration(s); err == nil {
|
||||
return time.Duration(d), nil
|
||||
}
|
||||
return 0, fmt.Errorf("cannot parse %q to a valid duration", s)
|
||||
}
|
||||
|
||||
func parseInstantQueryMetricsRequest(r *http.Request) (*model.InstantQueryMetricsParams, *model.ApiError) {
|
||||
var ts time.Time
|
||||
if t := r.FormValue("time"); t != "" {
|
||||
var err error
|
||||
ts, err = parseMetricsTime(t)
|
||||
if err != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
} else {
|
||||
ts = time.Now()
|
||||
}
|
||||
|
||||
return &model.InstantQueryMetricsParams{
|
||||
Time: ts,
|
||||
Query: r.FormValue("query"),
|
||||
Stats: r.FormValue("stats"),
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
func parseQueryRangeRequest(r *http.Request) (*model.QueryRangeParams, *model.ApiError) {
|
||||
|
||||
start, err := parseMetricsTime(r.FormValue("start"))
|
||||
if err != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
end, err := parseMetricsTime(r.FormValue("end"))
|
||||
if err != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
if end.Before(start) {
|
||||
err := errors.New("end timestamp must not be before start time")
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
|
||||
step, err := parseMetricsDuration(r.FormValue("step"))
|
||||
if err != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
|
||||
if step <= 0 {
|
||||
err := errors.New("zero or negative query resolution step widths are not accepted. Try a positive integer")
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
|
||||
// For safety, limit the number of returned points per timeseries.
|
||||
// This is sufficient for 60s resolution for a week or 1h resolution for a year.
|
||||
if end.Sub(start)/step > 11000 {
|
||||
err := errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")
|
||||
return nil, &model.ApiError{Typ: model.ErrorBadData, Err: err}
|
||||
}
|
||||
|
||||
queryRangeParams := model.QueryRangeParams{
|
||||
Start: start,
|
||||
End: end,
|
||||
Step: step,
|
||||
Query: r.FormValue("query"),
|
||||
Stats: r.FormValue("stats"),
|
||||
}
|
||||
|
||||
return &queryRangeParams, nil
|
||||
}
|
||||
|
||||
func parseGetUsageRequest(r *http.Request) (*model.GetUsageParams, error) {
|
||||
startTime, err := parseTime("start", r)
|
||||
if err != nil {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
)
|
||||
|
||||
type Reader interface {
|
||||
GetInstantQueryMetricsResult(ctx context.Context, query *model.InstantQueryMetricsParams) (*promql.Result, *stats.QueryStats, *model.ApiError)
|
||||
GetQueryRangeResult(ctx context.Context, query *model.QueryRangeParams) (*promql.Result, *stats.QueryStats, *model.ApiError)
|
||||
GetTopLevelOperations(ctx context.Context, start, end time.Time, services []string) (*map[string][]string, *model.ApiError)
|
||||
GetEntryPointOperations(ctx context.Context, query *model.GetTopOperationsParams) (*[]model.TopOperationsItem, error)
|
||||
|
||||
@@ -4,6 +4,12 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type InstantQueryMetricsParams struct {
|
||||
Time time.Time
|
||||
Query string
|
||||
Stats string
|
||||
}
|
||||
|
||||
type QueryRangeParams struct {
|
||||
Start time.Time
|
||||
End time.Time
|
||||
|
||||
@@ -105,7 +105,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
|
||||
}
|
||||
|
||||
// Create querier with test values
|
||||
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, nil, cache, flagger)
|
||||
providerFactory := signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger)
|
||||
mockQuerier, err := providerFactory.New(context.Background(), providerSettings, querier.Config{})
|
||||
require.NoError(t, err)
|
||||
|
||||
|
||||
@@ -47,7 +47,6 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // promV2
|
||||
nil, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
@@ -101,7 +100,6 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // promV2
|
||||
nil, // traceStmtBuilder
|
||||
logStmtBuilder, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
@@ -151,7 +149,6 @@ func prepareQuerierForTraces(t *testing.T, telemetryStore telemetrystore.Telemet
|
||||
telemetryStore,
|
||||
metadataStore,
|
||||
nil, // prometheus
|
||||
nil, // promV2
|
||||
traceStmtBuilder, // traceStmtBuilder
|
||||
nil, // logStmtBuilder
|
||||
nil, // auditStmtBuilder
|
||||
|
||||
@@ -96,9 +96,9 @@ func (r *aggExprRewriter) Rewrite(
|
||||
}
|
||||
|
||||
if visitor.isRate {
|
||||
return fmt.Sprintf("%s/%d", chparser.Format(sel.SelectItems[0]), rateInterval), visitor.chArgs, nil
|
||||
return fmt.Sprintf("%s/%d", sel.SelectItems[0].String(), rateInterval), visitor.chArgs, nil
|
||||
}
|
||||
return chparser.Format(sel.SelectItems[0]), visitor.chArgs, nil
|
||||
return sel.SelectItems[0].String(), visitor.chArgs, nil
|
||||
}
|
||||
|
||||
// RewriteMulti rewrites a slice of expressions.
|
||||
@@ -207,7 +207,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
// Handle *If functions with predicate + values
|
||||
if aggFunc.FuncCombinator {
|
||||
// Map the predicate (last argument)
|
||||
origPred := chparser.Format(args[len(args)-1])
|
||||
origPred := args[len(args)-1].String()
|
||||
whereClause, err := PrepareWhereClause(
|
||||
origPred,
|
||||
FilterExprVisitorOpts{
|
||||
@@ -242,7 +242,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
|
||||
// Map each value column argument
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
origVal := chparser.Format(args[i])
|
||||
origVal := args[i].String()
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal)
|
||||
expr, err := v.fieldMapper.ColumnExpressionFor(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, dataType, v.fieldKeys)
|
||||
if err != nil {
|
||||
@@ -259,7 +259,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
} else {
|
||||
// Non-If functions: map every argument as a column/value
|
||||
for i, arg := range args {
|
||||
orig := chparser.Format(arg)
|
||||
orig := arg.String()
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig)
|
||||
expr, err := v.fieldMapper.ColumnExpressionFor(v.ctx, v.orgID, v.startNs, v.endNs, &fieldKey, dataType, v.fieldKeys)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
)
|
||||
|
||||
// internalDatabases hold server metadata, credentials and grants rather than telemetry.
|
||||
var internalDatabases = map[string]struct{}{
|
||||
"system": {},
|
||||
"information_schema": {},
|
||||
}
|
||||
|
||||
// The parser's grammar has gaps against SQL that ClickHouse itself accepts. See TestErrIfStatementIsNotValid_ShouldPassButFails.
|
||||
func ErrIfStatementIsNotValid(query string) (err error) {
|
||||
defer func() {
|
||||
// The parser has a history of panicking on malformed input rather than returning an error.
|
||||
if recovered := recover(); recovered != nil {
|
||||
err = errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid ClickHouse SQL (recovered): %v", recovered)
|
||||
}
|
||||
}()
|
||||
|
||||
stmts, parseErr := chparser.NewParser(query).ParseStmts()
|
||||
if parseErr != nil {
|
||||
// Wrapped rather than formatted in, so that callers can recover the parser's *ParseError and read the position off it.
|
||||
return errors.WrapInvalidInputf(parseErr, errors.CodeInvalidInput, "invalid ClickHouse SQL: %s", parseErr.Error())
|
||||
}
|
||||
|
||||
if len(stmts) != 1 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "ClickHouse SQL must contain exactly one statement, found %d statements", len(stmts))
|
||||
}
|
||||
|
||||
selectQuery, ok := stmts[0].(*chparser.SelectQuery)
|
||||
if !ok {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "only SELECT statements are allowed in ClickHouse SQL queries")
|
||||
}
|
||||
|
||||
visitor := &chparser.DefaultASTVisitor{Visit: func(node chparser.Expr) error {
|
||||
switch expr := node.(type) {
|
||||
case *chparser.TableFunctionExpr:
|
||||
// Source table functions remain usable in ClickHouse read-only mode.
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "ClickHouse table functions are not allowed in SQL queries: %s", chparser.Format(expr.Name))
|
||||
|
||||
case *chparser.TableIdentifier:
|
||||
// Reading these is unaffected by ClickHouse read-only mode.
|
||||
if expr.Database == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, ok := internalDatabases[strings.ToLower(expr.Database.Name)]; ok {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "the ClickHouse %s database is not allowed in SQL queries", expr.Database.Name)
|
||||
}
|
||||
|
||||
case *chparser.SettingExpr:
|
||||
// A query-level setting takes precedence over the context setting.
|
||||
if strings.EqualFold(expr.Name.Name, "readonly") {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "the ClickHouse readonly setting cannot be overridden")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}}
|
||||
|
||||
return selectQuery.Accept(visitor)
|
||||
}
|
||||
|
||||
// TODO(@therealpandey): remove this and move to ErrIfStatementIsNotValid.
|
||||
func LogIfStatementIsNotValid(ctx context.Context, logger *slog.Logger, query string) {
|
||||
if err := ErrIfStatementIsNotValid(query); err != nil {
|
||||
logger.WarnContext(ctx, "clickhouse sql is not valid", errors.Attr(err), slog.String("query", query))
|
||||
}
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestErrIfStatementIsNotValid_Pass(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
query string
|
||||
}{
|
||||
// Shapes a telemetry read is allowed to take.
|
||||
{"Select", "SELECT region AS r, zone FROM metrics WHERE metric_name = 'cpu' GROUP BY region, zone"},
|
||||
{"TrailingSemicolon", "SELECT count() FROM signoz_logs.distributed_logs_v2;"},
|
||||
{"CommonTableExpression", "WITH t AS (SELECT fingerprint FROM signoz_metrics.time_series_v4) SELECT * FROM t"},
|
||||
{"Join", "SELECT * FROM t1 LEFT JOIN t2 ON t1.a = t2.b"},
|
||||
{"GlobalIn", "SELECT a FROM t WHERE a GLOBAL IN (SELECT b FROM t2)"},
|
||||
{"Union", "SELECT * FROM t UNION ALL SELECT * FROM t2"},
|
||||
{"Intersect", "SELECT * FROM t INTERSECT SELECT * FROM t2"},
|
||||
{"WindowFunction", "SELECT sum(v) OVER (PARTITION BY a ORDER BY t) FROM t"},
|
||||
{"UnrelatedSetting", "SELECT * FROM t SETTINGS max_threads = 4"},
|
||||
{"TerminatedBlockComment", "SELECT /* keep me */ count() FROM t"},
|
||||
{"BlockCommentMarkerInsideStringLiteral", "SELECT count() FROM t WHERE body = '/* not a comment'"},
|
||||
// The parser used to loop forever on this; it now reads the comment to the end of
|
||||
// the input, so this doubles as a canary for that regression.
|
||||
{"TrailingUnterminatedBlockComment", "SELECT count() FROM t /* unterminated"},
|
||||
// The rule keys on the database, not on the table name.
|
||||
{"TableNamedSystemInTelemetryDatabase", "SELECT * FROM signoz_logs.system"},
|
||||
{"SignedLiteralAfterClosingParen", "SELECT (toUnixTimestamp(now()) - 3600)*1000000000"},
|
||||
// order by interval
|
||||
{"OrderByInterval", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval ORDER BY interval"},
|
||||
{"OrderByIntervalAndDirection", "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS `interval` ORDER BY `interval` ASC"},
|
||||
{"TrimFunction", "SELECT trimBoth('/api/endpoint/', '/');"},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := ErrIfStatementIsNotValid(testCase.query)
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrIfStatementIsNotValid_Fail(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
query string
|
||||
}{
|
||||
// Not a single statement, or not a statement at all.
|
||||
{"Empty", ""},
|
||||
{"UnterminatedBlockCommentOnly", "/* x"},
|
||||
{"Unparseable", "SELECT FROM WHERE"},
|
||||
{"MultipleStatements", "SELECT 1; DROP TABLE signoz_logs.logs_v2"},
|
||||
// Parses, but is not a SELECT.
|
||||
{"Drop", "DROP TABLE signoz_logs.logs_v2"},
|
||||
{"Insert", "INSERT INTO signoz_logs.logs_v2 SELECT * FROM signoz_logs.logs_v2"},
|
||||
{"AlterDelete", "ALTER TABLE signoz_logs.logs_v2 DELETE WHERE 1 = 1"},
|
||||
{"CreateTable", "CREATE TABLE evil (a Int) ENGINE = Memory"},
|
||||
{"Grant", "GRANT ALL ON *.* TO admin"},
|
||||
{"Set", "SET readonly = 0"},
|
||||
// These the parser rejects outright rather than classifying.
|
||||
{"ShowGrants", "SHOW GRANTS"},
|
||||
{"IntoOutfile", "SELECT * FROM t INTO OUTFILE '/tmp/x.csv'"},
|
||||
// Table functions, which read through something other than a telemetry table.
|
||||
{"UrlTableFunction", "SELECT * FROM url('http://attacker.example/x', CSV, 'a String')"},
|
||||
{"FileTableFunction", "SELECT * FROM file('/etc/passwd', CSV, 'a String')"},
|
||||
{"ExecutableTableFunction", "SELECT * FROM executable('script.sh', CSV, 'a String')"},
|
||||
{"TableFunctionInJoin", "SELECT * FROM t1 JOIN url('http://x', CSV, 'a String') u ON 1 = 1"},
|
||||
{"TableFunctionInCommonTableExpression", "WITH c AS (SELECT * FROM url('http://x', CSV, 'a String')) SELECT * FROM c"},
|
||||
{"TableFunctionInWhereSubquery", "SELECT * FROM t WHERE a IN (SELECT * FROM file('/etc/passwd', CSV, 'a String'))"},
|
||||
{"TableFunctionInUnion", "SELECT * FROM t UNION ALL SELECT * FROM url('http://x', CSV, 'a String')"},
|
||||
// Internal databases, which hold grants and server metadata rather than telemetry.
|
||||
{"SystemUsers", "SELECT * FROM system.users"},
|
||||
{"SystemUppercase", "SELECT * FROM SYSTEM.USERS"},
|
||||
{"SystemQuoted", "SELECT count() FROM `system`.`tables`"},
|
||||
{"SystemInSubquery", "SELECT * FROM (SELECT name FROM system.parts)"},
|
||||
{"SystemInJoin", "SELECT * FROM signoz_logs.distributed_logs_v2 AS l JOIN system.users AS u ON 1 = 1"},
|
||||
{"SystemInIntersect", "SELECT * FROM t INTERSECT SELECT * FROM system.users"},
|
||||
{"InformationSchema", "SELECT * FROM information_schema.tables"},
|
||||
// A query-level setting takes precedence over the one the caller applies.
|
||||
{"ReadonlySettingOverride", "SELECT * FROM t SETTINGS readonly = 0"},
|
||||
{"ReadonlySettingOverrideAmongOthers", "SELECT * FROM t SETTINGS max_threads = 4, readonly = 0"},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := ErrIfStatementIsNotValid(testCase.query)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Queries the parser cannot read. ClickHouse runs all of them.
|
||||
func TestErrIfStatementIsNotValid_ShouldPassButFails(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
query string
|
||||
// The construct the parser stops after, which is the one it cannot read.
|
||||
expectedStopsAfter string
|
||||
// The same construct written so the parser accepts it.
|
||||
fix string
|
||||
}{
|
||||
{
|
||||
name: "IntervalAliasInOrderBy",
|
||||
query: "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, resource_string_service$$name AS `service.name`, attributes_string['http.route'] AS `http.route`, quantile(0.95)(duration_nano) / 1000000000 AS value FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_string_service$$name = 'svc-a' AND resources_string['deployment.environment'] = 'dev' AND attributes_string['http.route'] = '/v1' AND http_method = 'POST' AND timestamp BETWEEN toDateTime(1784601720) AND toDateTime(1784602620) AND ts_bucket_start BETWEEN 1784601720 - 1800 AND 1784602620 GROUP BY `service.name`, `http.route`, interval ORDER BY interval ASC",
|
||||
expectedStopsAfter: "ORDER BY interval ASC",
|
||||
fix: "SELECT count() AS interval FROM t ORDER BY `interval` ASC",
|
||||
},
|
||||
{
|
||||
name: "IntervalAliasInOrderByDesc",
|
||||
query: "SELECT count() AS value, toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS interval, serviceName, resourceTagsMap['deployment.environment'] AS environment, exceptionStacktrace FROM signoz_traces.distributed_signoz_error_index_v2 WHERE exceptionType != 'OSError' AND resourceTagsMap['deployment.environment'] = 'staging' AND timestamp BETWEEN toDateTime(1785186300) AND toDateTime(1785186600) GROUP BY serviceName, interval, environment, exceptionStacktrace ORDER BY interval DESC",
|
||||
expectedStopsAfter: "ORDER BY interval DESC",
|
||||
fix: "SELECT count() AS interval FROM t ORDER BY `interval` DESC",
|
||||
},
|
||||
{
|
||||
name: "StandardTrimSyntax",
|
||||
query: "SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 5 MINUTE) AS interval, resources_string['host.name'] as host_name, toFloat64(countIf( lower(trim(BOTH ' ' FROM replaceOne( JSONExtractString(body, 'Action'), 'health_status: ', '' ))) IN ('unhealthy','starting','failing') )) as value FROM signoz_logs.distributed_logs_v2 WHERE timestamp BETWEEN 1784602320000000000 AND 1784602620000000000 AND ts_bucket_start BETWEEN 1784602320 - 300 AND 1784602620 AND JSONExtractString(body, 'Type') = 'container' AND JSONExtractString(body, 'Actor', 'Attributes', 'name') IS NOT NULL AND resources_string['host.name'] IS NOT NULL AND resources_string['host.name'] = 'aihub-nightly' GROUP BY interval, host_name ORDER BY interval, host_name",
|
||||
expectedStopsAfter: "trim(BOTH '",
|
||||
fix: "SELECT trimBoth(replaceOne( JSONExtractString(body, 'Action'), 'health_status: ', '' ), ' ')",
|
||||
},
|
||||
{
|
||||
name: "SignedLiteralAfterClosingParen",
|
||||
query: "SELECT now() AS ts, toFloat64(count()) AS value FROM ( SELECT attributes_string['TableName'] AS T, attributes_string['MissingId'] AS M, max(fromUnixTimestamp64Nano(timestamp)) AS last_seen, dateDiff('minute', min(fromUnixTimestamp64Nano(timestamp)), max(fromUnixTimestamp64Nano(timestamp))) AS age_min FROM signoz_logs.distributed_logs_v2 WHERE body='missing_map_record' AND timestamp >= (toUnixTimestamp(now())-3600)*1000000000 GROUP BY T, M ) WHERE age_min >= 20 AND last_seen >= now() - toIntervalMinute(8)",
|
||||
expectedStopsAfter: "(toUnixTimestamp(now())",
|
||||
fix: "SELECT (toUnixTimestamp(now()) - 3600) * 1000000000",
|
||||
},
|
||||
{
|
||||
name: "SignedLiteralAfterClosingParenMinimal",
|
||||
query: "SELECT (1)-1",
|
||||
expectedStopsAfter: "(1)",
|
||||
fix: "SELECT (1) - 1",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := ErrIfStatementIsNotValid(testCase.query)
|
||||
|
||||
var parseErr *chparser.ParseError
|
||||
require.ErrorAs(t, err, &parseErr, "expected a parser failure rather than a rule violation")
|
||||
|
||||
// The parser reports the offset it stopped at, which sits just past the construct
|
||||
// it choked on, so the text leading up to it is what needs looking at.
|
||||
consumed := testCase.query[:parseErr.Pos]
|
||||
assert.Equal(t, testCase.expectedStopsAfter, consumed[max(0, len(consumed)-len(testCase.expectedStopsAfter)):])
|
||||
|
||||
assert.NoError(t, ErrIfStatementIsNotValid(testCase.fix))
|
||||
})
|
||||
}
|
||||
}
|
||||
1
pkg/querybuilder/init.go
Normal file
1
pkg/querybuilder/init.go
Normal file
@@ -0,0 +1 @@
|
||||
package querybuilder
|
||||
@@ -316,17 +316,17 @@ func (e *ClickHouseFilterExtractor) extractColumnStrByExpr(expr clickhouse.Expr)
|
||||
// FunctionExpr is a function call like "toDate(timestamp)"
|
||||
case *clickhouse.FunctionExpr:
|
||||
// For function expressions, return the complete function call string
|
||||
return clickhouse.Format(ex)
|
||||
return ex.String()
|
||||
// ColumnExpr is a column expression like "m.region", "toDate(timestamp)"
|
||||
case *clickhouse.ColumnExpr:
|
||||
// ColumnExpr wraps another expression - extract the underlying expression
|
||||
if ex.Expr != nil {
|
||||
return e.extractColumnStrByExpr(ex.Expr)
|
||||
}
|
||||
return clickhouse.Format(ex)
|
||||
return ex.String()
|
||||
default:
|
||||
// For other expression types, return the string representation
|
||||
return clickhouse.Format(expr)
|
||||
return expr.String()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -524,7 +524,7 @@ func (e *ClickHouseFilterExtractor) extractFullExpression(expr clickhouse.Expr)
|
||||
if expr == nil {
|
||||
return ""
|
||||
}
|
||||
return clickhouse.Format(expr)
|
||||
return expr.String()
|
||||
}
|
||||
|
||||
// isSimpleColumnReference checks if an expression is just a simple column reference
|
||||
@@ -657,7 +657,7 @@ func (e *ClickHouseFilterExtractor) extractCTEName(cte *clickhouse.CTEStmt) stri
|
||||
case *clickhouse.Ident:
|
||||
return name.Name
|
||||
default:
|
||||
return clickhouse.Format(cte.Expr)
|
||||
return cte.Expr.String()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/pprof/nooppprof"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/querier/signozquerier"
|
||||
"github.com/SigNoz/signoz/pkg/sharder"
|
||||
@@ -245,7 +244,6 @@ func NewTelemetryStoreProviderFactories() factory.NamedMap[factory.ProviderFacto
|
||||
func NewPrometheusProviderFactories(telemetryStore telemetrystore.TelemetryStore) factory.NamedMap[factory.ProviderFactory[prometheus.Prometheus, prometheus.Config]] {
|
||||
return factory.MustNewNamedMap(
|
||||
clickhouseprometheus.NewFactory(telemetryStore),
|
||||
clickhouseprometheusv2.NewFactory(telemetryStore),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -287,9 +285,9 @@ func NewStatsReporterProviderFactories(aggregator statsreporter.Aggregator, orgG
|
||||
)
|
||||
}
|
||||
|
||||
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, promV2 *clickhouseprometheusv2.Provider, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
|
||||
func NewQuerierProviderFactories(telemetryStore telemetrystore.TelemetryStore, prometheus prometheus.Prometheus, cache cache.Cache, flagger flagger.Flagger) factory.NamedMap[factory.ProviderFactory[querier.Querier, querier.Config]] {
|
||||
return factory.MustNewNamedMap(
|
||||
signozquerier.NewFactory(telemetryStore, prometheus, promV2, cache, flagger),
|
||||
signozquerier.NewFactory(telemetryStore, prometheus, cache, flagger),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -40,7 +40,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/tag/impltag"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus/clickhouseprometheusv2"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
"github.com/SigNoz/signoz/pkg/ruler"
|
||||
@@ -243,11 +242,6 @@ func New(
|
||||
|
||||
retentionGetter := implretention.NewGetter(implretention.NewStore(sqlstore))
|
||||
|
||||
// promV2 is the clickhousev2 provider handed to the querier for shadow
|
||||
// comparison and pinned serving (declared before the serving provider,
|
||||
// whose variable shadows the package name below).
|
||||
var promV2 *clickhouseprometheusv2.Provider
|
||||
|
||||
// Initialize prometheus from the available prometheus provider factories
|
||||
prometheus, err := factory.NewProviderFromNamedMap(
|
||||
ctx,
|
||||
@@ -260,29 +254,12 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// With the default provider, also stand up the clickhousev2 provider for
|
||||
// the querier: PromQL queries shadow-compare against it behind the
|
||||
// use_prometheus_clickhouse_v2 flag (see pkg/querier/promql_shadow.go).
|
||||
// It never serves by default. An explicit
|
||||
// prometheus::provider: clickhousev2 makes v2 the serving provider
|
||||
// outright, so there is nothing to compare against.
|
||||
if config.Prometheus.Provider() == "clickhouse" {
|
||||
v2Config := config.Prometheus
|
||||
// The v2 engine only evaluates shadow and pinned queries; disable its
|
||||
// active query tracker so two trackers never share a file.
|
||||
v2Config.ActiveQueryTrackerConfig.Enabled = false
|
||||
promV2, err = clickhouseprometheusv2.New(ctx, providerSettings, v2Config, telemetrystore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize querier from the available querier provider factories
|
||||
querier, err := factory.NewProviderFromNamedMap(
|
||||
ctx,
|
||||
providerSettings,
|
||||
config.Querier,
|
||||
NewQuerierProviderFactories(telemetrystore, prometheus, promV2, cache, flagger),
|
||||
NewQuerierProviderFactories(telemetrystore, prometheus, cache, flagger),
|
||||
config.Querier.Provider(),
|
||||
)
|
||||
if err != nil {
|
||||
|
||||
@@ -2359,7 +2359,10 @@ func (t *telemetryMetaStore) fetchTemporalityTypeForTable(ctx context.Context, t
|
||||
if temporality != metrictypes.Unknown {
|
||||
temporalities[metricName] = append(temporalities[metricName], temporality)
|
||||
}
|
||||
if metricType == metrictypes.SumType && !isMonotonic {
|
||||
// Monotonicity carries meaning only for cumulative sums: a non-monotonic
|
||||
// cumulative sum is a gauge for all practical purposes. Delta sums are
|
||||
// counters regardless of monotonicity.
|
||||
if metricType == metrictypes.SumType && !isMonotonic && temporality == metrictypes.Cumulative {
|
||||
metricType = metrictypes.GaugeType
|
||||
}
|
||||
types[metricName] = metricType
|
||||
@@ -2412,7 +2415,9 @@ func (t *telemetryMetaStore) fetchMeterSourceMetricsTemporalityAndType(ctx conte
|
||||
if err := rows.Scan(&metricName, &temporality, &metricType, &isMonotonic); err != nil {
|
||||
return nil, nil, errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to scan temporality result")
|
||||
}
|
||||
if metricType == metrictypes.SumType && !isMonotonic {
|
||||
// Non-monotonic cumulative sums are treated as gauges; delta sums are
|
||||
// counters regardless of monotonicity.
|
||||
if metricType == metrictypes.SumType && !isMonotonic && temporality == metrictypes.Cumulative {
|
||||
metricType = metrictypes.GaugeType
|
||||
}
|
||||
temporalities[metricName] = temporality
|
||||
|
||||
@@ -16,7 +16,9 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -109,3 +111,85 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
t.Errorf("there were unfulfilled expectations: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Sum metrics are classified as counters unless they are non-monotonic AND
|
||||
// cumulative; monotonicity carries no meaning for delta sums, so a
|
||||
// non-monotonic delta sum must remain a Sum (counter).
|
||||
func TestFetchTemporalityAndTypeMultiSumClassification(t *testing.T) {
|
||||
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, ®exMatcher{})
|
||||
mock := mockTelemetryStore.Mock()
|
||||
|
||||
metadata := NewTelemetryMetaStore(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockTelemetryStore,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.TagAttributesV2TableName,
|
||||
telemetrytraces.SpanAttributesKeysTblName,
|
||||
telemetrytraces.SpanIndexV3TableName,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.AttributesMetadataTableName,
|
||||
telemetrymeter.DBName,
|
||||
telemetrymeter.SamplesAgg1dTableName,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2TableName,
|
||||
telemetrylogs.TagAttributesV2TableName,
|
||||
telemetrylogs.LogAttributeKeysTblName,
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetryaudit.DBName,
|
||||
telemetryaudit.AuditLogsTableName,
|
||||
telemetryaudit.TagAttributesTableName,
|
||||
telemetryaudit.LogAttributeKeysTblName,
|
||||
telemetryaudit.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
flaggertest.New(t),
|
||||
)
|
||||
|
||||
metricNames := []string{
|
||||
"delta.nonmono.sum",
|
||||
"delta.mono.sum",
|
||||
"cumulative.nonmono.sum",
|
||||
"cumulative.mono.sum",
|
||||
}
|
||||
|
||||
metadataCols := []cmock.ColumnType{
|
||||
{Name: "metric_name", Type: "String"},
|
||||
{Name: "temporality", Type: "String"},
|
||||
{Name: "type", Type: "String"},
|
||||
{Name: "is_monotonic", Type: "Bool"},
|
||||
}
|
||||
|
||||
mock.ExpectQuery(`SELECT metric_name, temporality, any\(type\) AS type, argMax\(is_monotonic, unix_milli\) as is_monotonic FROM signoz_metrics\.`).
|
||||
WithArgs(nil, nil, nil).
|
||||
WillReturnRows(cmock.NewRows(metadataCols, [][]any{
|
||||
{"delta.nonmono.sum", metrictypes.Delta, metrictypes.SumType, false},
|
||||
{"delta.mono.sum", metrictypes.Delta, metrictypes.SumType, true},
|
||||
{"cumulative.nonmono.sum", metrictypes.Cumulative, metrictypes.SumType, false},
|
||||
{"cumulative.mono.sum", metrictypes.Cumulative, metrictypes.SumType, true},
|
||||
}))
|
||||
mock.ExpectQuery(`SELECT metric_name, argMax\(temporality, unix_milli\) as temporality, any\(type\) AS type, argMax\(is_monotonic, unix_milli\) as is_monotonic FROM signoz_meter\.`).
|
||||
WithArgs(nil).
|
||||
WillReturnRows(cmock.NewRows(metadataCols, [][]any{}))
|
||||
|
||||
temporalities, types, _, err := metadata.FetchTemporalityAndTypeMulti(
|
||||
context.Background(),
|
||||
valuer.GenerateUUID(),
|
||||
1700000000000,
|
||||
1700003600000,
|
||||
metricNames...,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, metrictypes.SumType, types["delta.nonmono.sum"])
|
||||
assert.Equal(t, metrictypes.SumType, types["delta.mono.sum"])
|
||||
assert.Equal(t, metrictypes.GaugeType, types["cumulative.nonmono.sum"])
|
||||
assert.Equal(t, metrictypes.SumType, types["cumulative.mono.sum"])
|
||||
|
||||
assert.Equal(t, metrictypes.Delta, temporalities["delta.nonmono.sum"])
|
||||
assert.Equal(t, metrictypes.Cumulative, temporalities["cumulative.nonmono.sum"])
|
||||
|
||||
if err := mock.ExpectationsWereMet(); err != nil {
|
||||
t.Errorf("there were unfulfilled expectations: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
|
||||
}
|
||||
|
||||
// Parse column name to extract context and data type
|
||||
columnName := parser.Format(expr.Name)
|
||||
columnName := expr.Name.String()
|
||||
|
||||
// Remove backticks if present
|
||||
columnName = strings.TrimPrefix(columnName, "`")
|
||||
@@ -75,7 +75,7 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
|
||||
// Extract field name from the DEFAULT expression
|
||||
// The DEFAULT expression should be something like: resources_string['k8s.cluster.name']
|
||||
// We need to extract the key inside the square brackets
|
||||
defaultExprStr := parser.Format(expr.DefaultExpr)
|
||||
defaultExprStr := expr.DefaultExpr.String()
|
||||
|
||||
// Look for the pattern: map['key']
|
||||
startIdx := strings.Index(defaultExprStr, "['")
|
||||
|
||||
@@ -370,14 +370,6 @@ type QueryRangeRequest struct {
|
||||
// NoCache is a flag to disable caching for the request.
|
||||
NoCache bool `json:"noCache,omitempty"`
|
||||
|
||||
// PromQLProvider serves this request's PromQL queries via the named
|
||||
// prometheus provider ("clickhousev2") instead of the default — the same
|
||||
// data read through a different implementation. It is set from the
|
||||
// X-SigNoz-PromQL-Provider header by the API handler, never from the
|
||||
// body: a rollout-scoped comparison hook for integration tests and
|
||||
// support should not become part of the public request schema.
|
||||
PromQLProvider string `json:"-"`
|
||||
|
||||
FormatOptions *FormatOptions `json:"formatOptions,omitempty"`
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ func (qp *QueryProcessor) ProcessQuery(query string, transformer FilterTransform
|
||||
// Reconstruct the query
|
||||
var resultBuilder strings.Builder
|
||||
for _, stmt := range stmts {
|
||||
resultBuilder.WriteString(parser.Format(stmt))
|
||||
resultBuilder.WriteString(stmt.String())
|
||||
resultBuilder.WriteString(";")
|
||||
}
|
||||
|
||||
|
||||
2
scripts/promqltestcorpus/.gitignore
vendored
2
scripts/promqltestcorpus/.gitignore
vendored
@@ -1,2 +0,0 @@
|
||||
# go build artifact (go run . is the supported entry)
|
||||
/promqltestcorpus
|
||||
@@ -1,660 +0,0 @@
|
||||
// SigNoz corpus policy: which upstream cases are representable through the
|
||||
// API, the grid variants that steer coarse-step code paths, the API's value
|
||||
// rounding, and the frozen JSON model. Nothing in this file mirrors
|
||||
// upstream code; it encodes what our conformance harness needs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/model/value"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"github.com/prometheus/prometheus/tsdb/chunkenc"
|
||||
"github.com/prometheus/prometheus/util/almost"
|
||||
"github.com/prometheus/prometheus/util/teststorage"
|
||||
)
|
||||
|
||||
// seriesDescParser is the slice of parser.Parser the loader needs.
|
||||
type seriesDescParser interface {
|
||||
ParseSeriesDesc(input string) (labels.Labels, []parser.SequenceValue, error)
|
||||
}
|
||||
|
||||
// Calendar and wall-clock functions are not invariant under time
|
||||
// translation, and the Python suite shifts every case to recent timestamps
|
||||
// (epoch-0 samples would sit 55 years past ClickHouse TTLs). Everything else
|
||||
// PromQL computes depends only on time differences.
|
||||
var timeDependentFuncs = map[string]bool{
|
||||
"time": true, "timestamp": true, "month": true, "year": true,
|
||||
"minute": true, "hour": true, "day_of_month": true, "day_of_week": true,
|
||||
"day_of_year": true, "days_in_month": true,
|
||||
}
|
||||
|
||||
const (
|
||||
lookbackMs = 300_000
|
||||
instantStepMs = 1_000
|
||||
maxSamples = 50_000_000
|
||||
)
|
||||
|
||||
type corpusSeries struct {
|
||||
Labels map[string]string `json:"labels"`
|
||||
Samples [][2]any `json:"samples"` // [offset_ms, value]
|
||||
}
|
||||
|
||||
type corpusDataset struct {
|
||||
ID int `json:"id"`
|
||||
Source string `json:"source"`
|
||||
Series []corpusSeries `json:"series"`
|
||||
}
|
||||
|
||||
type corpusPoint = [2]any // [offset_ms, value]
|
||||
|
||||
type corpusResult struct {
|
||||
Labels map[string]string `json:"labels"`
|
||||
Points []corpusPoint `json:"points"`
|
||||
}
|
||||
|
||||
type corpusCase struct {
|
||||
Dataset int `json:"dataset"`
|
||||
Source string `json:"source"`
|
||||
Variant string `json:"variant"`
|
||||
Expr string `json:"expr"`
|
||||
StartMs int64 `json:"start_ms"`
|
||||
EndMs int64 `json:"end_ms"`
|
||||
StepMs int64 `json:"step_ms"`
|
||||
Instant bool `json:"instant"`
|
||||
Expected []corpusResult `json:"expected"`
|
||||
}
|
||||
|
||||
type corpus struct {
|
||||
Meta struct {
|
||||
PrometheusVersion string `json:"prometheus_version"`
|
||||
LookbackMs int64 `json:"lookback_ms"`
|
||||
InstantStepMs int64 `json:"instant_step_ms"`
|
||||
Note string `json:"note"`
|
||||
} `json:"meta"`
|
||||
Datasets []corpusDataset `json:"datasets"`
|
||||
Cases []corpusCase `json:"cases"`
|
||||
}
|
||||
|
||||
func generate(files []string, engine *promql.Engine, seriesParser parser.Parser, exprParser parser.Parser, promVersion string) (*corpus, map[string]int, error) {
|
||||
var c corpus
|
||||
c.Meta.PrometheusVersion = promVersion
|
||||
c.Meta.LookbackMs = lookbackMs
|
||||
c.Meta.InstantStepMs = instantStepMs
|
||||
c.Meta.Note = "expected values carry the API's 3-significant-decimal rounding (querybuildertypesv5 sanitizeValue); instant evals are encoded as start==end range queries"
|
||||
|
||||
skips := map[string]int{}
|
||||
datasetIDs := map[string]int{}
|
||||
|
||||
for _, file := range files {
|
||||
base := filepath.Base(file)
|
||||
if base == "native_histograms.test" || base == "type_and_unit.test" {
|
||||
// native histograms: the samples pipeline under test stores
|
||||
// floats; type_and_unit: experimental __type__/__unit__ metadata
|
||||
// labels our store does not materialize.
|
||||
skips["file:"+strings.TrimSuffix(base, ".test")]++
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cmds := parseScript(string(raw))
|
||||
|
||||
segment := 0
|
||||
var loads []command
|
||||
segmentBad := "" // non-empty: reason the segment cannot be represented
|
||||
for _, cmd := range cmds {
|
||||
switch cmd.kind {
|
||||
case "clear":
|
||||
segment++
|
||||
loads = nil
|
||||
segmentBad = ""
|
||||
case "skip":
|
||||
skips["command:"+cmd.head]++
|
||||
case "load":
|
||||
if reason := checkLoad(seriesParser, cmd); reason != "" {
|
||||
segmentBad = reason
|
||||
skips["load:"+reason]++
|
||||
continue
|
||||
}
|
||||
loads = append(loads, cmd)
|
||||
case "eval":
|
||||
if segmentBad != "" {
|
||||
skips["segment:"+segmentBad]++
|
||||
continue
|
||||
}
|
||||
if len(loads) == 0 {
|
||||
skips["eval:no-data"]++
|
||||
continue
|
||||
}
|
||||
ccs, reason, err := buildCases(engine, seriesParser, exprParser, cmd, loads, base, skips)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if reason != "" {
|
||||
skips["eval:"+reason]++
|
||||
continue
|
||||
}
|
||||
key := fmt.Sprintf("%s#%d#%d", base, segment, len(loads))
|
||||
id, ok := datasetIDs[key]
|
||||
if !ok {
|
||||
ds, reason, err := dumpDataset(seriesParser, loads)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if reason != "" {
|
||||
skips["dataset:"+reason]++
|
||||
continue
|
||||
}
|
||||
id = len(c.Datasets)
|
||||
datasetIDs[key] = id
|
||||
ds.ID = id
|
||||
ds.Source = key
|
||||
c.Datasets = append(c.Datasets, *ds)
|
||||
}
|
||||
for _, cc := range ccs {
|
||||
cc.Dataset = id
|
||||
cc.Source = fmt.Sprintf("%s:%d", base, cmd.line)
|
||||
c.Cases = append(c.Cases, cc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(c.Cases) == 0 {
|
||||
return nil, nil, fmt.Errorf("no corpus cases produced")
|
||||
}
|
||||
return &c, skips, nil
|
||||
}
|
||||
|
||||
func writeCorpus(out string, c *corpus) error {
|
||||
buf, err := json.MarshalIndent(c, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(out, buf, 0o644)
|
||||
}
|
||||
|
||||
// checkLoad validates a load block is representable: parsable series
|
||||
// notation, float samples only ("load_with_nhcb" and histogram literals are
|
||||
// out of scope — the samples pipeline under test stores float samples).
|
||||
func checkLoad(p parser.Parser, cmd command) string {
|
||||
fields := strings.Fields(cmd.head)
|
||||
if len(fields) != 2 || fields[0] != "load" {
|
||||
return "unsupported-load-variant"
|
||||
}
|
||||
if _, err := model.ParseDuration(fields[1]); err != nil {
|
||||
return "bad-interval"
|
||||
}
|
||||
for _, line := range cmd.body {
|
||||
metric, vals, err := p.ParseSeriesDesc(line)
|
||||
if err != nil {
|
||||
return "unparsable-series"
|
||||
}
|
||||
if metric.Get(model.MetricNameLabel) == "" {
|
||||
return "unnamed-series"
|
||||
}
|
||||
for _, v := range vals {
|
||||
if v.Histogram != nil {
|
||||
return "histogram-samples"
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// durationExprUsesRange reports whether a duration-expression tree contains
|
||||
// range(). Instant evals are encoded as one-step range queries (the API
|
||||
// rejects start == end), which changes what range() evaluates to, so they
|
||||
// cannot carry it; range evals keep it — each variant's oracle is computed
|
||||
// on the exact window it requests.
|
||||
func durationExprUsesRange(e parser.Expr) bool {
|
||||
d, ok := e.(*parser.DurationExpr)
|
||||
if !ok || d == nil {
|
||||
return false
|
||||
}
|
||||
if d.Op == parser.RANGE {
|
||||
return true
|
||||
}
|
||||
return durationExprUsesRange(d.LHS) || durationExprUsesRange(d.RHS)
|
||||
}
|
||||
|
||||
// buildCases parses one eval header, filters unservable expressions, and
|
||||
// emits the base case plus grid variants — each with expectations computed by
|
||||
// the reference engine over the loads. The variants exist because upstream's
|
||||
// own grids are fine-stepped: without them the coarse-step code paths (the
|
||||
// window-sliver filter, the disjoint over_time form, the lifted instant/last
|
||||
// gates) would pass through this corpus untouched. A variant is just another
|
||||
// grid over the same data and expression; the engine is the oracle either way.
|
||||
func buildCases(engine *promql.Engine, seriesParser parser.Parser, exprParser parser.Parser, cmd command, loads []command, sourceFile string, skips map[string]int) ([]corpusCase, string, error) {
|
||||
for _, line := range cmd.body {
|
||||
if patExpect.MatchString(line) && strings.HasPrefix(line, "expect fail") {
|
||||
return nil, "expect-fail", nil
|
||||
}
|
||||
}
|
||||
|
||||
base := corpusCase{Variant: "base"}
|
||||
if m := patEvalInstant.FindStringSubmatch(cmd.head); m != nil {
|
||||
at, err := parseTestDuration(m[2])
|
||||
if err != nil {
|
||||
return nil, "bad-duration", nil
|
||||
}
|
||||
base.Instant = true
|
||||
base.StartMs, base.EndMs, base.StepMs = at, at, instantStepMs
|
||||
base.Expr = m[3]
|
||||
} else if m := patEvalRange.FindStringSubmatch(cmd.head); m != nil {
|
||||
from, err1 := parseTestDuration(m[2])
|
||||
to, err2 := parseTestDuration(m[3])
|
||||
step, err3 := parseTestDuration(m[4])
|
||||
if err1 != nil || err2 != nil || err3 != nil {
|
||||
return nil, "bad-duration", nil
|
||||
}
|
||||
if step <= 0 || to < from {
|
||||
return nil, "bad-grid", nil
|
||||
}
|
||||
base.StartMs, base.EndMs, base.StepMs = from, to, step
|
||||
base.Expr = m[5]
|
||||
} else {
|
||||
return nil, "unrecognized", nil
|
||||
}
|
||||
|
||||
expr, err := exprParser.ParseExpr(base.Expr)
|
||||
if err != nil {
|
||||
return nil, "needs-experimental-parser", nil
|
||||
}
|
||||
if vt := expr.Type(); vt != parser.ValueTypeVector && vt != parser.ValueTypeScalar {
|
||||
return nil, "non-instant-type", nil
|
||||
}
|
||||
unservable := ""
|
||||
hasSelector := false
|
||||
hasSubquery := false
|
||||
var maxRangeMs int64
|
||||
parser.Inspect(expr, func(node parser.Node, _ []parser.Node) error {
|
||||
switch n := node.(type) {
|
||||
case *parser.Call:
|
||||
if timeDependentFuncs[n.Func.Name] {
|
||||
unservable = "time-dependent"
|
||||
}
|
||||
case *parser.VectorSelector:
|
||||
hasSelector = true
|
||||
if n.Timestamp != nil || n.StartOrEnd != 0 {
|
||||
unservable = "at-modifier"
|
||||
}
|
||||
if n.OriginalOffset < 0 {
|
||||
// The server ships with negative offsets disabled.
|
||||
unservable = "negative-offset"
|
||||
}
|
||||
if base.Instant && durationExprUsesRange(n.OriginalOffsetExpr) {
|
||||
unservable = "range-duration-in-instant"
|
||||
}
|
||||
for _, m := range n.LabelMatchers {
|
||||
if m.Name == "__type__" || m.Name == "__unit__" {
|
||||
unservable = "type-unit-metadata"
|
||||
}
|
||||
}
|
||||
case *parser.MatrixSelector:
|
||||
if r := n.Range.Milliseconds(); r > maxRangeMs {
|
||||
maxRangeMs = r
|
||||
}
|
||||
if base.Instant && durationExprUsesRange(n.RangeExpr) {
|
||||
unservable = "range-duration-in-instant"
|
||||
}
|
||||
case *parser.SubqueryExpr:
|
||||
hasSubquery = true
|
||||
if n.Timestamp != nil || n.StartOrEnd != 0 {
|
||||
unservable = "at-modifier"
|
||||
}
|
||||
if n.OriginalOffset < 0 {
|
||||
unservable = "negative-offset"
|
||||
}
|
||||
if base.Instant && (durationExprUsesRange(n.RangeExpr) || durationExprUsesRange(n.StepExpr) || durationExprUsesRange(n.OriginalOffsetExpr)) {
|
||||
unservable = "range-duration-in-instant"
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if unservable != "" {
|
||||
return nil, unservable, nil
|
||||
}
|
||||
|
||||
stor, err := loadSeriesStorage(seriesParser, loads)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer func() { _ = stor.Close() }()
|
||||
|
||||
expected, reason := computeExpected(engine, stor, base.Expr, base.StartMs, base.EndMs, base.StepMs)
|
||||
if reason != "" {
|
||||
return nil, reason, nil
|
||||
}
|
||||
if err := crossCheckUpstream(engine, seriesParser, stor, cmd, base, sourceFile, skips); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
base.Expected = expected
|
||||
out := []corpusCase{base}
|
||||
|
||||
// Grid variants. Subquery expressions keep their own inner grids; varying
|
||||
// the outer grid there multiplies cases without steering the code paths
|
||||
// the variants exist for, so they emit only the base.
|
||||
if hasSubquery || !hasSelector {
|
||||
return out, "", nil
|
||||
}
|
||||
type variant struct {
|
||||
name string
|
||||
startMs, endMs, stepMs int64
|
||||
}
|
||||
var variants []variant
|
||||
if base.Instant {
|
||||
// A coarse multi-point grid ending at the instant: step above the
|
||||
// 5m lookback drives the lifted instant gate and the sliver filter.
|
||||
const coarse = 600_000
|
||||
variants = append(variants, variant{"instant-coarse", base.EndMs - 2*coarse, base.EndMs, coarse})
|
||||
} else {
|
||||
span := base.EndMs - base.StartMs
|
||||
if maxRangeMs > 0 {
|
||||
// Step wider than every window in the expression: the sliver
|
||||
// filter and the disjoint over_time form become active.
|
||||
if coarse := 2 * maxRangeMs; span >= coarse {
|
||||
variants = append(variants, variant{"coarse-step", base.StartMs, base.EndMs, coarse})
|
||||
}
|
||||
// Whole-bucket tiling (range == 2 steps) drives the windowed
|
||||
// over_time slide with W = 2.
|
||||
if tiled := maxRangeMs / 2; tiled >= 1000 && maxRangeMs%2000 == 0 && tiled != base.StepMs && span >= tiled {
|
||||
variants = append(variants, variant{"tiled", base.StartMs, base.EndMs, tiled})
|
||||
}
|
||||
}
|
||||
// A start off every natural alignment shifts which samples each
|
||||
// window sees; an end short of the lattice exercises the
|
||||
// last-grid-point handling.
|
||||
if span > 17_000 {
|
||||
variants = append(variants, variant{"unaligned-start", base.StartMs + 17_000, base.EndMs, base.StepMs})
|
||||
}
|
||||
if lastIdx := span / base.StepMs; lastIdx >= 2 {
|
||||
offEnd := base.StartMs + lastIdx*base.StepMs - base.StepMs/3
|
||||
variants = append(variants, variant{"off-lattice-end", base.StartMs, offEnd, base.StepMs})
|
||||
}
|
||||
}
|
||||
for _, v := range variants {
|
||||
if v.stepMs <= 0 || v.endMs <= v.startMs || v.stepMs%1000 != 0 {
|
||||
continue
|
||||
}
|
||||
expected, reason := computeExpected(engine, stor, base.Expr, v.startMs, v.endMs, v.stepMs)
|
||||
if reason != "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, corpusCase{
|
||||
Variant: v.name, Expr: base.Expr,
|
||||
StartMs: v.startMs, EndMs: v.endMs, StepMs: v.stepMs,
|
||||
Expected: expected,
|
||||
})
|
||||
}
|
||||
return out, "", nil
|
||||
}
|
||||
|
||||
// computeExpected evaluates the expression on one grid with the reference
|
||||
// engine and serializes the result with the API's value rounding.
|
||||
func computeExpected(engine *promql.Engine, stor *teststorage.TestStorage, expr string, startMs, endMs, stepMs int64) ([]corpusResult, string) {
|
||||
qry, err := engine.NewRangeQuery(context.Background(), stor, nil, expr,
|
||||
time.UnixMilli(startMs), time.UnixMilli(endMs), time.Duration(stepMs)*time.Millisecond)
|
||||
if err != nil {
|
||||
return nil, "engine-parse"
|
||||
}
|
||||
defer qry.Close()
|
||||
res := qry.Exec(context.Background())
|
||||
if res.Err != nil {
|
||||
// Covers upstream's expected-error cases and engine features the
|
||||
// range form cannot evaluate; a case we cannot compute is a case we
|
||||
// cannot assert.
|
||||
return nil, "engine-error"
|
||||
}
|
||||
matrix, ok := res.Value.(promql.Matrix)
|
||||
if !ok {
|
||||
return nil, "non-matrix-result"
|
||||
}
|
||||
expected := []corpusResult{}
|
||||
for _, s := range matrix {
|
||||
if len(s.Histograms) > 0 {
|
||||
return nil, "histogram-result"
|
||||
}
|
||||
r := corpusResult{Labels: s.Metric.Map(), Points: []corpusPoint{}}
|
||||
for _, p := range s.Floats {
|
||||
r.Points = append(r.Points, corpusPoint{p.T, encodeFloat(roundToNonZeroDecimals(p.F, 3))})
|
||||
}
|
||||
expected = append(expected, r)
|
||||
}
|
||||
return expected, ""
|
||||
}
|
||||
|
||||
// dumpDataset walks the loaded storage and serializes every float sample.
|
||||
func dumpDataset(seriesParser parser.Parser, loads []command) (*corpusDataset, string, error) {
|
||||
stor, err := loadSeriesStorage(seriesParser, loads)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer func() { _ = stor.Close() }()
|
||||
|
||||
q, err := stor.Querier(math.MinInt64/2, math.MaxInt64/2)
|
||||
if err != nil {
|
||||
return nil, "querier", nil
|
||||
}
|
||||
defer q.Close()
|
||||
|
||||
ds := &corpusDataset{}
|
||||
ss := q.Select(context.Background(), true, nil, labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".*"))
|
||||
var it chunkenc.Iterator
|
||||
for ss.Next() {
|
||||
s := ss.At()
|
||||
cs := corpusSeries{Labels: s.Labels().Map(), Samples: [][2]any{}}
|
||||
it = s.Iterator(it)
|
||||
for vt := it.Next(); vt != chunkenc.ValNone; vt = it.Next() {
|
||||
if vt != chunkenc.ValFloat {
|
||||
return nil, "histogram-samples", nil
|
||||
}
|
||||
ts, v := it.At()
|
||||
if value.IsStaleNaN(v) {
|
||||
cs.Samples = append(cs.Samples, [2]any{ts, "stale"})
|
||||
continue
|
||||
}
|
||||
cs.Samples = append(cs.Samples, [2]any{ts, encodeFloat(v)})
|
||||
}
|
||||
ds.Series = append(ds.Series, cs)
|
||||
}
|
||||
if err := ss.Err(); err != nil {
|
||||
return nil, "series-set", nil
|
||||
}
|
||||
return ds, "", nil
|
||||
}
|
||||
|
||||
// parseTestDuration accepts promqltest's time notation: a Prometheus
|
||||
// duration ("5m", "1m30s"), a bare "0", or bare seconds.
|
||||
func parseTestDuration(s string) (int64, error) {
|
||||
if d, err := model.ParseDuration(s); err == nil {
|
||||
return int64(time.Duration(d) / time.Millisecond), nil
|
||||
}
|
||||
if n, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
return int64(n * 1000), nil
|
||||
}
|
||||
return 0, fmt.Errorf("unparsable duration %q", s)
|
||||
}
|
||||
|
||||
func encodeFloat(f float64) any {
|
||||
switch {
|
||||
case math.IsNaN(f):
|
||||
return "NaN"
|
||||
case math.IsInf(f, 1):
|
||||
return "Inf"
|
||||
case math.IsInf(f, -1):
|
||||
return "-Inf"
|
||||
default:
|
||||
return f
|
||||
}
|
||||
}
|
||||
|
||||
// roundToNonZeroDecimals mirrors querybuildertypesv5's sanitizeValue rounding
|
||||
// (pkg/types/querybuildertypes/querybuildertypesv5/resp.go) so the frozen
|
||||
// expectations equal what the API emits for the same float.
|
||||
func roundToNonZeroDecimals(val float64, n int) float64 {
|
||||
if val == 0 || math.IsNaN(val) || math.IsInf(val, 0) {
|
||||
return val
|
||||
}
|
||||
absVal := math.Abs(val)
|
||||
if absVal >= 1 {
|
||||
multiplier := math.Pow(10, float64(n))
|
||||
rounded := math.Round(val*multiplier) / multiplier
|
||||
if math.IsInf(rounded, 0) {
|
||||
// Mirrors the overflow guard in querybuildertypesv5.
|
||||
return val
|
||||
}
|
||||
if rounded == math.Trunc(rounded) {
|
||||
return rounded
|
||||
}
|
||||
str := strconv.FormatFloat(rounded, 'f', -1, 64)
|
||||
result, _ := strconv.ParseFloat(str, 64)
|
||||
return result
|
||||
}
|
||||
order := math.Floor(math.Log10(absVal))
|
||||
scale := math.Pow(10, -order+float64(n)-1)
|
||||
rounded := math.Round(val*scale) / scale
|
||||
str := strconv.FormatFloat(rounded, 'f', -1, 64)
|
||||
result, _ := strconv.ParseFloat(str, 64)
|
||||
return result
|
||||
}
|
||||
|
||||
// crossCheckFileAllowlist names files whose written expectations assume
|
||||
// engine options we deliberately run differently, with the reason. Every
|
||||
// other mismatch between our engine-computed expectations and upstream's
|
||||
// hand-written ones aborts generation: the corpus must never contradict
|
||||
// the testdata it claims to represent.
|
||||
var crossCheckFileAllowlist = map[string]string{
|
||||
"name_label_dropping.test": "expectations written for EnableDelayedNameRemoval; our engine matches the server default (off)",
|
||||
}
|
||||
|
||||
// crossCheckUpstream validates the transcription chain — load parsing,
|
||||
// eval parsing, storage loading — by comparing the reference engine's raw
|
||||
// output on the base grid against the expectations upstream wrote under the
|
||||
// same eval, with upstream's own tolerance (almost.Equal, 1e-6 relative).
|
||||
// The corpus's authority is "what the reference engine computes over
|
||||
// upstream's data"; this pins that computation to upstream's own record of
|
||||
// it.
|
||||
func crossCheckUpstream(engine *promql.Engine, seriesParser parser.Parser, stor *teststorage.TestStorage, cmd command, base corpusCase, sourceFile string, skips map[string]int) error {
|
||||
type expSeries struct {
|
||||
labels labels.Labels
|
||||
points map[int64]float64
|
||||
}
|
||||
var expected []expSeries
|
||||
scalarOnly := false
|
||||
var scalarValue float64
|
||||
for _, line := range cmd.body {
|
||||
if strings.HasPrefix(line, "expect") {
|
||||
// expect fail/warn/info/ordered directives and "expect range
|
||||
// vector"/"expect string" annotations, not series expectations.
|
||||
continue
|
||||
}
|
||||
if f, err := strconv.ParseFloat(line, 64); err == nil && len(cmd.body) == 1 {
|
||||
scalarOnly, scalarValue = true, f
|
||||
break
|
||||
}
|
||||
metric, vals, err := seriesParser.ParseSeriesDesc(line)
|
||||
if err != nil {
|
||||
skips["crosscheck-skip:unparsable-expectation"]++
|
||||
return nil
|
||||
}
|
||||
points := map[int64]float64{}
|
||||
for k, v := range vals {
|
||||
if v.Histogram != nil {
|
||||
skips["crosscheck-skip:histogram-expectation"]++
|
||||
return nil
|
||||
}
|
||||
if v.Omitted {
|
||||
continue
|
||||
}
|
||||
points[base.StartMs+int64(k)*base.StepMs] = v.Value
|
||||
}
|
||||
expected = append(expected, expSeries{labels: metric, points: points})
|
||||
}
|
||||
|
||||
qry, err := engine.NewRangeQuery(context.Background(), stor, nil, base.Expr,
|
||||
time.UnixMilli(base.StartMs), time.UnixMilli(base.EndMs), time.Duration(base.StepMs)*time.Millisecond)
|
||||
if err != nil {
|
||||
return fmt.Errorf("crosscheck parse %q: %w", base.Expr, err)
|
||||
}
|
||||
defer qry.Close()
|
||||
res := qry.Exec(context.Background())
|
||||
if res.Err != nil {
|
||||
return fmt.Errorf("crosscheck eval %q: %w", base.Expr, res.Err)
|
||||
}
|
||||
matrix, ok := res.Value.(promql.Matrix)
|
||||
if !ok {
|
||||
skips["crosscheck-skip:non-matrix"]++
|
||||
return nil
|
||||
}
|
||||
|
||||
mismatch := func(format string, args ...any) error {
|
||||
if reason, ok := crossCheckFileAllowlist[sourceFile]; ok {
|
||||
skips["crosscheck-allowlisted:"+sourceFile]++
|
||||
_ = reason
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("%s:%d: corpus contradicts upstream expectation for %q: %s",
|
||||
sourceFile, cmd.line, base.Expr, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
if scalarOnly {
|
||||
if len(matrix) != 1 || matrix[0].Metric.Len() != 0 {
|
||||
return mismatch("scalar expectation but %d series", len(matrix))
|
||||
}
|
||||
if len(matrix[0].Floats) == 0 || !almost.Equal(matrix[0].Floats[len(matrix[0].Floats)-1].F, scalarValue, defaultEpsilon) {
|
||||
return mismatch("scalar %v != expected %v", matrix[0].Floats, scalarValue)
|
||||
}
|
||||
skips["crosscheck-ok"]++
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(matrix) != len(expected) {
|
||||
return mismatch("engine returned %d series, upstream wrote %d", len(matrix), len(expected))
|
||||
}
|
||||
for _, exp := range expected {
|
||||
var got *promql.Series
|
||||
for i := range matrix {
|
||||
if labels.Equal(matrix[i].Metric, exp.labels) {
|
||||
got = &matrix[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
if got == nil {
|
||||
return mismatch("series %s missing from engine result", exp.labels)
|
||||
}
|
||||
gotPoints := map[int64]float64{}
|
||||
for _, p := range got.Floats {
|
||||
gotPoints[p.T] = p.F
|
||||
}
|
||||
if len(gotPoints) != len(exp.points) {
|
||||
return mismatch("series %s: %d points, upstream wrote %d", exp.labels, len(gotPoints), len(exp.points))
|
||||
}
|
||||
for ts, want := range exp.points {
|
||||
gotV, ok := gotPoints[ts]
|
||||
if !ok {
|
||||
return mismatch("series %s: no point at %d", exp.labels, ts)
|
||||
}
|
||||
if math.IsNaN(want) && math.IsNaN(gotV) {
|
||||
continue
|
||||
}
|
||||
if !almost.Equal(gotV, want, defaultEpsilon) {
|
||||
return mismatch("series %s at %d: engine %v, upstream wrote %v", exp.labels, ts, gotV, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
skips["crosscheck-ok"]++
|
||||
return nil
|
||||
}
|
||||
@@ -1,103 +0,0 @@
|
||||
module github.com/SigNoz/signoz/scripts/promqltestcorpus
|
||||
|
||||
go 1.25.7
|
||||
|
||||
require (
|
||||
github.com/prometheus/common v0.67.5
|
||||
github.com/prometheus/prometheus v0.311.3
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect
|
||||
github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dennwc/varint v1.0.0 // indirect
|
||||
github.com/edsrzf/mmap-go v1.2.0 // indirect
|
||||
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
github.com/golang/snappy v1.0.0 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.18.0 // indirect
|
||||
github.com/grafana/regexp v0.0.0-20250905093917-f7b3be9d1853 // indirect
|
||||
github.com/jpillora/backoff v1.0.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
|
||||
github.com/oklog/ulid/v2 v2.1.1 // indirect
|
||||
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/prometheus/client_golang v1.23.2 // indirect
|
||||
github.com/prometheus/client_golang/exp v0.0.0-20260325093428-d8591d0db856 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/otlptranslator v1.0.0 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/prometheus/sigv4 v0.4.1 // indirect
|
||||
github.com/stretchr/testify v1.11.1 // indirect
|
||||
github.com/x448/float16 v0.8.4 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect
|
||||
go.opentelemetry.io/otel v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.42.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.42.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/goleak v1.3.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.4 // indirect
|
||||
golang.org/x/crypto v0.49.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect
|
||||
golang.org/x/net v0.52.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/term v0.41.0 // indirect
|
||||
golang.org/x/text v0.35.0 // indirect
|
||||
golang.org/x/time v0.15.0 // indirect
|
||||
google.golang.org/api v0.272.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260311181403-84a4fc48630c // indirect
|
||||
google.golang.org/grpc v1.79.3 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/apimachinery v0.35.3 // indirect
|
||||
k8s.io/client-go v0.35.3 // indirect
|
||||
k8s.io/klog/v2 v2.140.0 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect
|
||||
k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 // indirect
|
||||
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
|
||||
sigs.k8s.io/randfill v1.0.0 // indirect
|
||||
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
|
||||
sigs.k8s.io/yaml v1.6.0 // indirect
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user