Compare commits

..

4 Commits

Author SHA1 Message Date
Srikanth Chekuri
b180df8e3e Merge branch 'main' into issue-5535 2026-07-06 20:55:56 +05:30
Srikanth Chekuri
c6bb7569af Merge branch 'main' into issue-5535 2026-07-06 11:51:09 +05:30
srikanthccv
5d431f9f6f chore: add to ci 2026-07-06 11:50:46 +05:30
srikanthccv
1f0113645e chore: add integration tests for metrics under reduction - query part 2026-07-06 11:19:45 +05:30
28 changed files with 1605 additions and 772 deletions

View File

@@ -56,6 +56,8 @@ jobs:
- querier_json_body
- querier_skip_resource_fingerprint
- ttl
- clickhousecluster
- metricreduction
sqlstore-provider:
- postgres
- sqlite

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="#1287b1" viewBox="0 0 24 24"><title>Apache Cassandra</title><path d="M10.374 10.53a3 3 0 0 1-.428-.222l.555.143c0 .02-.01.036-.01.055l-.117.025zm-.283 1.506-.315.253.852-1.079-1.078.391c.002.017.009.033.009.05a.57.57 0 0 1-.184.42q.154.327.375.616a3.2 3.2 0 0 1 .34-.651zm.717-2.347-.652-.82a.43.43 0 0 1-.506.162c-.054.073-.083.162-.13.24l1.258.463zm-1.666.444c-.07.314-.087.637-.05.956a.57.57 0 0 1 .451.475l.946-.606c-.067-.022-.126-.06-.191-.088l-1.119-.08.64-.14a3.2 3.2 0 0 1-.668-.554zM20.1 11.648c-.164.202.833 1.022.833 1.022s-1.654-1.022-2.234-.72c-.278.144.574.811 1.175 1.242-.428-.274-.982-.571-1.175-.408-.328.277 1.565 2.549 1.565 2.549s-2.145-2.322-2.36-2.209c-.214.114.593 1.224.593 1.224s-1.06-1.16-1.35-.959c-.29.202 1.514 3.218 1.514 3.218s-1.956-3.091-2.763-2.574c1.268 2.782.795 3.18.795 3.18s-.162-2.839-1.742-2.764c-.795.038.379 2.12.379 2.12s-1.08-1.902-1.8-1.864c1.326 2.51.854 3.53.854 3.53s.219-2.143-1.58-3.336c.682.606-.427 3.336-.427 3.336s.976-4.023-.719-3.256c-.268.121-.019 2.007-.019 2.007s-.34-2.158-.851-2.045c-.298.066-1.893 2.99-1.893 2.99s1.306-3.16.908-3.027c-.29.096-.833 1.4-.833 1.4s.265-1.287 0-1.363c-.264-.075-1.74 1.363-1.74 1.363s1.097-1.287.908-1.552c-.287-.402-.623-.42-1.022-.265-.581.226-1.363 1.287-1.363 1.287s.78-1.074.643-1.476c-.219-.647-2.46 1.249-2.46 1.249s1.325-1.25 1.022-1.514c-.303-.265-1.947-.183-2.46-.185-1.515-.004-2.039-.36-2.498-.724 1.987.997 3.803-.151 6.094.494l.21.06c-1.3-.558-2.144-1.378-2.226-2.354-.036-.416.074-.827.297-1.222.619-.4 1.29-.773 2.06-1.095a4 4 0 0 0-.064.698c0 2.44 2.203 4.417 4.92 4.417s4.92-1.977 4.92-4.417c0-.45-.083-.881-.223-1.29 1.431.404 2.45.968 3.132 1.335.022.092.045.184.053.279.024.274-.018.547-.11.814.095-.147.198-.288.28-.445.367-.997 1.855.227 1.855.227s-1.085-.454-1.06-.24c.026.215 1.628.96 1.628.96s-1.45-.455-1.362-.114c.088.34 1.817 1.703 1.817 1.703s-1.956-1.489-2.12-1.287zm-7.268 2.65.042-.008-.06.01zM9.256 9.753c.12.13.26.234.396.343l.927-.029-1.064-.788c-.093.154-.195.303-.26.474Zm10.62 3.44c.3.215.54.373.54.373s-.24-.181-.54-.374zM7.507 8.617c-.14.229-.214.492-.215.76a3.99 3.99 0 0 0 2.358 3.64q0-.008.003-.014a3.2 3.2 0 0 1-.58-.788c-.648.099-.926-.794-.336-1.08a3.2 3.2 0 0 1 .138-1.388 3.16 3.16 0 0 1-.52-1.36q-.444.105-.848.23m1.488.82q.163-.36.402-.661a.435.435 0 0 1 .568-.557c.077-.059.166-.099.248-.15a16 16 0 0 0-1.727.284c.114.388.272.76.509 1.084m2.285 3.928c1.4 0 2.633-.723 3.344-1.816a3.4 3.4 0 0 0-1.265-.539l-.297-.023.916.9-1.197-.467.704 1.078-1.074-.832-.012.006.347 1.278-.596-1.134-.098 1.33-.401-1.326-.472 1.261.114-1.359-.015-.008-.814 1.154.286-1.067c-.34.322-.605.713-.781 1.146q.143.153.303.29.484.126 1.008.128m10.145-4.434c.971-.567 1.716-1.955 1.716-1.955s-1.893 1.955-3.205 1.665c1.186-.934 1.766-2.549 1.766-2.549s-1.506 2.325-2.448 2.423c1.086-.959 1.54-2.322 1.54-2.322s-1.237 1.817-2.196 1.944c1.287-1.161 1.338-1.893 1.338-1.893s-1.781 2.302-2.499 1.943c.858-.934 1.439-2.12 1.439-2.12s-1.489 2.019-1.893 1.69c-.277-.05.454-.958.454-.958s-.908.807-1.16.606c.454-.278 1.236-1.64 1.236-1.64S16 7.505 15.621 7.304l.731-1.483s-.73 1.483-1.715 1.23c.454-.58.63-1.112.63-1.112s-.756 1.213-1.69.885c-.22-.077.273-.635.273-.635s-.626.61-1.055.534c-.43-.076.025-.858.025-.858s-.757 1.186-.908 1.136.075-.833.075-.833-.555.908-.858.858 0-.934 0-.934-.328.984-.58.909c-.252-.076-.303-.656-.303-.656s-.068.788-.429.858c-2.725.53-5.728 1.69-9.489 5.45C3.887 10.738 5.3 7.91 11.962 7.659c5.044-.191 7.399 2.137 8.177 2.17C22.51 9.93 24 7.633 24 7.633s-1.489 1.716-2.574 1.3zm-7.74.872-.608.464v.001l.054.003a3.4 3.4 0 0 0 .554-.468m1.583-.426c0-.536-.237-.929-.594-1.217a3.2 3.2 0 0 1-.165.825.393.393 0 0 1-.328.681c-.154.233-.34.445-.549.63l.661.034-.995.237c-.025.018-.045.041-.07.058a3.2 3.2 0 0 1 1.536.691c.32-.574.504-1.235.504-1.94zM10.99 7.996a3.5 3.5 0 0 0-.785.46.43.43 0 0 1-.013.357l.885.643.023-.016-.36-1.262.627 1.12c.018-.006.04-.006.058-.011l-.02-1.251.398 1.163.477-1.15.016 1.268.012.007.713-1.005-.363 1.218.009.01 1.04-.69-.759 1.05.002.005.95-.34.041-.045a.395.395 0 0 1 .394-.632 3.4 3.4 0 0 0 .27-.784 14 14 0 0 0-2.798-.168c-.286.011-.55.033-.817.053"/></svg>

Before

Width:  |  Height:  |  Size: 4.1 KiB

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="48 -2.25 262.5 364"><defs><style>.cls-1{fill:#326ce5}.cls-3{fill:none}</style></defs><path d="M59.724 97.778a10.183 10.183 0 0 1 0-17.073l114.07-74.16a10.18 10.18 0 0 1 11.1 0l114.07 74.16a10.183 10.183 0 0 1 0 17.073l-114.07 74.16a10.18 10.18 0 0 1-11.1 0z" class="cls-1"/><path fill="#c1d2f7" d="M197.356 110.866h7.912a6.003 6.003 0 0 0 5.2-9.005l-25.924-44.902a6.004 6.004 0 0 0-10.399 0l-25.924 44.902a6.003 6.003 0 0 0 5.2 9.005h7.912a6.003 6.003 0 0 1 6.004 6.003v51.257l5.31 3.452a12.29 12.29 0 0 0 13.395 0l5.31-3.452v-51.257a6.003 6.003 0 0 1 6.004-6.003"/><path d="M173.793 353.271a10.1 10.1 0 0 0 3.457 1.402 119 119 0 0 0-6.623-3.46zm-6.456-161.195-11.316 7.356a111 111 0 0 0 11.316 6.6zm24.015 23.716c9.739 3.115 19.813 5.648 30.11 8.186 10.92 2.692 21.973 5.427 32.818 9.009l-35.416-23.025a216 216 0 0 1-27.512-9.41zm0 62.191v.586c0 2.57-2.688 4.652-6.003 4.652H173.34c-3.315 0-6.003-2.083-6.003-4.652v-6.605c-24.906-6.402-49.873-14.485-70.86-33.82l-14.529 9.445c22.468 22.05 50.28 28.933 79.715 36.189 27.178 6.698 55.18 13.651 78.685 33.442l14.68-9.544c-18.532-16.504-40.41-23.687-63.676-29.693M93.315 300.95l38.908 25.295c20.064 5.113 40.2 11.248 58.31 23.36l14.901-9.687c-19.752-13.672-42.55-19.313-66.419-25.197-15.287-3.767-30.835-7.616-45.7-13.771m74.022-82.254a124.3 124.3 0 0 1-21.534-12.62l-14.805 9.625a124.5 124.5 0 0 0 36.339 18.708zm51.41 16.294c-9.106-2.245-18.304-4.524-27.395-7.299v13.682q5.051 1.286 10.177 2.538c29.934 7.379 60.888 15.01 85.708 39.829.61.61 1.162 1.24 1.752 1.857l9.974-6.484a10.1 10.1 0 0 0 3.475-3.8 101 101 0 0 0-3.289-3.486c-22.585-22.585-50.669-29.508-80.402-36.837m-19.933 19.933q-3.722-.918-7.462-1.855v13.21c25.817 6.561 51.803 14.688 73.455 35.04l14.521-9.44-.11-.118c-22.586-22.585-50.67-29.509-80.404-36.837m-31.477-8.587c-16.28-5.278-32.118-12.397-46.344-24.131l-14.682 9.545c17.87 15.557 38.784 22.673 61.026 28.5z" class="cls-3"/><path d="M73.24 254.96c-.348-.348-.658-.71-1-1.06l-12.514 8.136a10 10 0 0 0-1.644 1.397c1.053 1.157 2.115 2.31 3.245 3.44 22.585 22.585 50.67 29.509 80.403 36.837 25.343 6.247 51.401 12.717 73.865 29.603l14.806-9.626c-20.865-16.42-45.53-22.509-71.453-28.898-29.934-7.379-60.888-15.01-85.708-39.83" class="cls-3"/><path d="m218.864 209.962-27.512-17.886v8.476a216 216 0 0 0 27.512 9.41m-27.512 5.83v11.9c9.091 2.774 18.289 5.053 27.395 7.298 29.733 7.329 57.817 14.252 80.402 36.837a101 101 0 0 1 3.29 3.486 10.193 10.193 0 0 0-3.476-13.277l-44.683-29.05c-10.845-3.58-21.898-6.316-32.818-9.008-10.297-2.538-20.371-5.071-30.11-8.186m-24.015-9.76a111 111 0 0 1-11.316-6.6l-10.218 6.644a124.3 124.3 0 0 0 21.534 12.62zm34.192 37.88q-5.125-1.264-10.177-2.539v11.695q3.737.938 7.462 1.855c29.733 7.328 57.818 14.252 80.403 36.837.039.039.073.079.111.118l9.66-6.28c-.589-.618-1.142-1.248-1.751-1.858-24.82-24.82-55.774-32.45-85.708-39.829m-34.192-9.503a124.5 124.5 0 0 1-36.34-18.708l-10.004 6.504c14.226 11.734 30.064 18.853 46.344 24.13zm0 25.842c-22.242-5.828-43.156-12.944-61.026-28.5l-9.834 6.392c20.987 19.336 45.954 27.42 70.86 33.821zm24.015 17.732c23.266 6.006 45.144 13.19 63.675 29.693l9.78-6.358c-21.652-20.352-47.638-28.479-73.455-35.04zM81.948 247.59l-9.707 6.311c.341.35.652.712 1 1.06 24.82 24.82 55.773 32.45 85.707 39.829 25.922 6.39 50.588 12.478 71.452 28.898l9.948-6.467c-23.505-19.791-51.507-26.744-78.685-33.442-29.436-7.256-57.247-14.138-79.715-36.189m-20.621 19.284c-1.13-1.13-2.192-2.283-3.245-3.44a10.154 10.154 0 0 0 1.644 15.68l33.589 21.837c14.865 6.155 30.413 10.004 45.7 13.771 23.869 5.884 46.667 11.525 66.419 25.197l10.16-6.605c-22.463-16.886-48.521-23.356-73.864-29.603-29.733-7.328-57.818-14.252-80.403-36.837m109.3 84.34a119 119 0 0 1 6.623 3.46 10.16 10.16 0 0 0 7.645-1.402l5.639-3.666c-18.11-12.112-38.247-18.247-58.311-23.36z" class="cls-1"/></svg>

Before

Width:  |  Height:  |  Size: 3.7 KiB

View File

@@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" viewBox="0 0 24 24"><title>PlanetScale</title><path d="M0 12C0 5.373 5.373 0 12 0c4.873 0 9.067 2.904 10.947 7.077l-15.87 15.87a12 12 0 0 1-1.935-1.099L14.99 12H12l-8.485 8.485A11.96 11.96 0 0 1 0 12m12.004 12L24 12.004C23.998 18.628 18.628 23.998 12.004 24"/></svg>

Before

Width:  |  Height:  |  Size: 326 B

View File

@@ -177,13 +177,7 @@ export default function ConversationView({
conversationId={conversationId}
messages={messages}
isStreaming={isStreamingHere}
onSendSuggestedPrompt={(text): void => {
handleSend(
text,
undefined,
autoContexts.length > 0 ? autoContexts : undefined,
);
}}
onSendSuggestedPrompt={(text): void => handleSend(text)}
/>
{showDisclaimer && (
<div className={disclaimerClass} role="note" aria-live="polite">

View File

@@ -1,142 +0,0 @@
import { MemoryRouter } from 'react-router-dom';
// eslint-disable-next-line no-restricted-imports
import { fireEvent, render } from '@testing-library/react';
import { MessageContext } from 'api/ai-assistant/chat';
import { useAIAssistantStore } from 'container/AIAssistant/store/useAIAssistantStore';
import { VariantContext } from 'container/AIAssistant/VariantContext';
const CHIP_ID = 'recent-errors';
const CHIP_TEXT = 'Show me recent errors';
// Auto-derived page context that a normal (typed) send would attach. The chip
// send must forward the exact same array.
const mockAutoContexts: MessageContext[] = [
{
source: 'auto',
type: 'dashboard',
resourceId: 'dashboard-123',
resourceName: 'Checkout dashboard',
},
];
jest.mock('api/common/logEvent', () => ({
__esModule: true,
default: jest.fn(),
}));
jest.mock('container/AIAssistant/getAutoContexts', () => ({
getAutoContexts: jest.fn(() => mockAutoContexts),
}));
jest.mock('container/AIAssistant/hooks/useAIAssistantAnalyticsContext', () => ({
normalizePage: (page: string): string => page,
useAIAssistantAnalyticsContext: (): unknown => ({ threadId: 'thread-1' }),
}));
// ChatInput is heavy and irrelevant here — the chip path lives entirely in the
// empty state. Provide a lightweight stub plus the `autoContextKey` named export
// ConversationView imports for its dismissed-context filter.
jest.mock('container/AIAssistant/components/ChatInput', () => ({
__esModule: true,
default: (): JSX.Element => <div data-testid="chat-input" />,
autoContextKey: (): string => '',
}));
jest.mock('container/AIAssistant/components/ConversationSkeleton', () => ({
__esModule: true,
default: (): JSX.Element => <div data-testid="skeleton" />,
}));
// VirtualizedMessages renders the real empty-state chips; stub only its
// never-rendered-in-empty-state children and the virtual list.
jest.mock('components/Noz/Noz', () => ({
__esModule: true,
default: (): JSX.Element => <div data-testid="noz" />,
}));
jest.mock('container/AIAssistant/components/MessageBubble', () => ({
__esModule: true,
default: (): null => null,
}));
jest.mock('container/AIAssistant/components/StreamingMessage', () => ({
__esModule: true,
default: (): null => null,
}));
jest.mock('react-virtuoso', () => ({
__esModule: true,
Virtuoso: (): null => null,
}));
jest.mock(
'container/AIAssistant/components/VirtualizedMessages/useEmptyStateChips',
() => ({
useEmptyStateChips: (): { chips: { id: string; text: string }[] } => ({
chips: [{ id: CHIP_ID, text: CHIP_TEXT }],
}),
}),
);
// eslint-disable-next-line import/first
import ConversationView from '../ConversationView';
const CONVERSATION_ID = 'conv-1';
function renderView(variant: 'panel' | 'page' | 'modal'): {
getByTestId: (id: string) => HTMLElement;
} {
return render(
<MemoryRouter initialEntries={['/dashboard/dashboard-123']}>
<VariantContext.Provider value={variant}>
<ConversationView conversationId={CONVERSATION_ID} />
</VariantContext.Provider>
</MemoryRouter>,
);
}
describe('ConversationView — empty-state chip context', () => {
let sendMessage: jest.Mock;
beforeEach(() => {
jest.clearAllMocks();
sendMessage = jest.fn();
useAIAssistantStore.setState({
conversations: {
[CONVERSATION_ID]: {
id: CONVERSATION_ID,
messages: [],
createdAt: 1,
updatedAt: 1,
},
},
streams: {},
activeConversationId: CONVERSATION_ID,
isLoadingThread: false,
sendMessage,
} as unknown as Partial<ReturnType<typeof useAIAssistantStore.getState>>);
});
it('forwards the page auto-contexts when a chip is clicked (embedded variant)', () => {
const { getByTestId } = renderView('panel');
fireEvent.click(getByTestId(`empty-state-chip-${CHIP_ID}`));
// The chip send must carry the same auto-contexts a typed message would.
expect(sendMessage).toHaveBeenCalledTimes(1);
expect(sendMessage).toHaveBeenCalledWith(
CHIP_TEXT,
undefined,
mockAutoContexts,
);
});
it('sends undefined contexts on the standalone page (no page context to attach)', () => {
const { getByTestId } = renderView('page');
fireEvent.click(getByTestId(`empty-state-chip-${CHIP_ID}`));
expect(sendMessage).toHaveBeenCalledTimes(1);
expect(sendMessage).toHaveBeenCalledWith(CHIP_TEXT, undefined, undefined);
});
});

View File

@@ -1625,9 +1625,6 @@ export const getHostQueryPayload = (
const diskPendingKey = dotMetricsEnabled
? 'system.disk.pending_operations'
: 'system_disk_pending_operations';
const fsUsageKey = dotMetricsEnabled
? 'system.filesystem.usage'
: 'system_filesystem_usage';
return [
{
@@ -2660,155 +2657,6 @@ export const getHostQueryPayload = (
start,
end,
},
{
selectedTime: 'GLOBAL_TIME',
graphType: PANEL_TYPES.TIME_SERIES,
query: {
builder: {
queryData: [
{
aggregateAttribute: {
dataType: DataTypes.Float64,
id: 'system_filesystem_usage--float64--Gauge--true',
key: fsUsageKey,
type: 'Gauge',
},
aggregateOperator: 'avg',
dataSource: DataSource.METRICS,
disabled: true,
expression: 'A',
filters: {
items: [
{
id: 'fs_f1',
key: {
dataType: DataTypes.String,
id: 'host_name--string--tag--false',
key: hostNameKey,
type: 'tag',
},
op: '=',
value: hostName,
},
{
id: 'fs_f2',
key: {
dataType: DataTypes.String,
id: 'state--string--tag--false',
key: 'state',
type: 'tag',
},
op: '=',
value: 'used',
},
],
op: 'AND',
},
functions: [],
groupBy: [
{
dataType: DataTypes.String,
id: 'mountpoint--string--tag--false',
key: 'mountpoint',
type: 'tag',
},
],
having: [
{
columnName: `SUM(${fsUsageKey})`,
op: '>',
value: 0,
},
],
legend: '{{mountpoint}}',
limit: null,
orderBy: [],
queryName: 'A',
reduceTo: ReduceOperators.AVG,
spaceAggregation: 'sum',
stepInterval: 60,
timeAggregation: 'avg',
},
{
aggregateAttribute: {
dataType: DataTypes.Float64,
id: 'system_filesystem_usage--float64--Gauge--true',
key: fsUsageKey,
type: 'Gauge',
},
aggregateOperator: 'avg',
dataSource: DataSource.METRICS,
disabled: true,
expression: 'B',
filters: {
items: [
{
id: 'fs_f3',
key: {
dataType: DataTypes.String,
id: 'host_name--string--tag--false',
key: hostNameKey,
type: 'tag',
},
op: '=',
value: hostName,
},
],
op: 'AND',
},
functions: [],
groupBy: [
{
dataType: DataTypes.String,
id: 'mountpoint--string--tag--false',
key: 'mountpoint',
type: 'tag',
},
],
having: [
{
columnName: `SUM(${fsUsageKey})`,
op: '>',
value: 0,
},
],
legend: '{{mountpoint}}',
limit: null,
orderBy: [],
queryName: 'B',
reduceTo: ReduceOperators.AVG,
spaceAggregation: 'sum',
stepInterval: 60,
timeAggregation: 'avg',
},
],
queryFormulas: [
{
disabled: false,
expression: 'A/B',
legend: '{{mountpoint}}',
queryName: 'F1',
},
],
queryTraceOperator: [],
},
clickhouse_sql: [{ disabled: false, legend: '', name: 'A', query: '' }],
id: 'a1b2c3d4-e5f6-7890-abcd-ef1234567890',
promql: [{ disabled: false, legend: '', name: 'A', query: '' }],
queryType: EQueryType.QUERY_BUILDER,
},
variables: {},
formatForWeb: false,
start,
end,
},
];
};
@@ -2884,5 +2732,4 @@ export const hostWidgetInfo = [
{ title: 'System disk operations/s', yAxisUnit: 'short' },
{ title: 'Queue size', yAxisUnit: 'short' },
{ title: 'System disk operation time/s', yAxisUnit: 's' },
{ title: 'Disk Usage (%) by mountpoint', yAxisUnit: 'percentunit' },
];

View File

@@ -20,7 +20,6 @@ import azureOpenaiUrl from '@/assets/Logos/azure-openai.svg';
import azureSqlDatabaseMetricsUrl from '@/assets/Logos/azure-sql-database-metrics.svg';
import azureVmUrl from '@/assets/Logos/azure-vm.svg';
import basetenUrl from '@/assets/Logos/baseten.svg';
import cassandraUrl from '@/assets/Logos/cassandra.svg';
import celeryUrl from '@/assets/Logos/celery.svg';
import certManagerUrl from '@/assets/Logos/cert-manager.svg';
import claudeCodeUrl from '@/assets/Logos/claude-code.svg';
@@ -52,7 +51,6 @@ import externalApiMonitoringUrl from '@/assets/Logos/external-api-monitoring.svg
import fluentbitUrl from '@/assets/Logos/fluentbit.svg';
import fluentdUrl from '@/assets/Logos/fluentd.svg';
import flutterMonitoringUrl from '@/assets/Logos/flutter-monitoring.svg';
import fluxcdUrl from '@/assets/Logos/fluxcd.svg';
import flyIoUrl from '@/assets/Logos/fly-io.svg';
import fromLogFileUrl from '@/assets/Logos/from-log-file.svg';
import gcpAppEngineUrl from '@/assets/Logos/gcp-app-engine.svg';
@@ -96,7 +94,6 @@ import langtraceUrl from '@/assets/Logos/langtrace.svg';
import litellmUrl from '@/assets/Logos/litellm.svg';
import livekitUrl from '@/assets/Logos/livekit.svg';
import llamaindexUrl from '@/assets/Logos/llamaindex.svg';
import llmMonitoringUrl from '@/assets/Logos/llm-monitoring.svg';
import logrusUrl from '@/assets/Logos/logrus.svg';
import logsUrl from '@/assets/Logos/logs.svg';
import logstashUrl from '@/assets/Logos/logstash.svg';
@@ -123,7 +120,6 @@ import opentelemetryUrl from '@/assets/Logos/opentelemetry.svg';
import phpUrl from '@/assets/Logos/php.svg';
import pinoUrl from '@/assets/Logos/pino.svg';
import pipecatUrl from '@/assets/Logos/pipecat.svg';
import planetscaleUrl from '@/assets/Logos/planetscale.svg';
import postgresqlUrl from '@/assets/Logos/postgresql.svg';
import prometheusUrl from '@/assets/Logos/prometheus.svg';
import pydanticAiUrl from '@/assets/Logos/pydantic-ai.svg';
@@ -1530,28 +1526,6 @@ const onboardingConfigWithLinks = [
id: 'nginx-tracing',
link: '/docs/instrumentation/opentelemetry-nginx/',
},
{
dataSource: 'nginx-ingress-controller',
label: 'NGINX Ingress Controller',
imgUrl: nginxUrl,
tags: ['infrastructure monitoring'],
module: 'metrics',
relatedSearchKeywords: [
'ingress',
'ingress controller',
'kubernetes ingress',
'monitoring',
'nginx ingress',
'nginx ingress controller',
'nginx ingress metrics',
'nginx ingress monitoring',
'nginx ingress observability',
'observability',
'opentelemetry nginx ingress',
],
id: 'nginx-ingress-controller',
link: '/docs/metrics-management/nginx-ingress-controller/',
},
{
dataSource: 'opentelemetry-cloudflare',
label: 'Cloudflare Tracing',
@@ -1612,119 +1586,6 @@ const onboardingConfigWithLinks = [
id: 'opentelemetry-cloudflare-logs',
link: '/docs/logs-management/send-logs/cloudflare-logs/',
},
{
dataSource: 'cloudflare-workers',
label: 'Cloudflare Workers',
imgUrl: cloudflareUrl,
tags: ['apm/traces'],
module: 'apm',
relatedSearchKeywords: [
'cloudflare',
'cloudflare workers',
'cloudflare workers monitoring',
'cloudflare workers observability',
'cloudflare workers otlp',
'edge computing monitoring',
'monitor cloudflare workers',
'monitoring',
'observability',
'opentelemetry cloudflare workers',
'otlp',
'serverless monitoring',
],
id: 'cloudflare-workers',
link: '/docs/integrations/outposts/cloudflare-workers/',
},
{
dataSource: 'opentelemetry-cassandra',
label: 'Cassandra',
imgUrl: cassandraUrl,
tags: ['database'],
module: 'apm',
relatedSearchKeywords: [
'apache cassandra',
'cassandra',
'cassandra database',
'cassandra logs',
'cassandra metrics',
'cassandra monitoring',
'cassandra observability',
'database',
'monitoring',
'nosql',
'observability',
'opentelemetry cassandra',
],
id: 'opentelemetry-cassandra',
link: '/docs/integrations/opentelemetry-cassandra/',
},
{
dataSource: 'fluxcd',
label: 'FluxCD',
imgUrl: fluxcdUrl,
tags: ['infrastructure monitoring'],
module: 'metrics',
relatedSearchKeywords: [
'continuous delivery',
'flux',
'fluxcd',
'fluxcd dashboard',
'fluxcd metrics',
'fluxcd monitoring',
'fluxcd observability',
'gitops',
'kubernetes',
'monitoring',
'observability',
'opentelemetry fluxcd',
],
id: 'fluxcd',
link: '/docs/metrics-management/fluxcd-metrics/',
},
{
dataSource: 'planetscale',
label: 'PlanetScale',
imgUrl: planetscaleUrl,
tags: ['database'],
module: 'apm',
relatedSearchKeywords: [
'database',
'monitoring',
'mysql',
'observability',
'opentelemetry planetscale',
'planetscale',
'planetscale database',
'planetscale metrics',
'planetscale monitoring',
'planetscale observability',
'serverless database',
],
id: 'planetscale',
link: '/docs/metrics-management/opentelemetry-planetscale/',
},
{
dataSource: 'hermes-agent',
label: 'Hermes Agent',
imgUrl: llmMonitoringUrl,
tags: ['LLM Monitoring'],
module: 'apm',
relatedSearchKeywords: [
'ai agent monitoring',
'hermes',
'hermes agent',
'hermes agent monitoring',
'hermes agent observability',
'hermes monitoring',
'llm monitoring',
'monitoring',
'nous research',
'observability',
'opentelemetry hermes',
],
id: 'hermes-agent',
link: '/docs/hermes-monitoring/',
},
{
dataSource: 'convex-logs',
label: 'Convex Logs',

View File

@@ -26,8 +26,8 @@ export default function AIAssistantPage(): JSX.Element {
// Skip the mount-time Opened fire when the user expanded an already-open
// drawer/modal — that surface already emitted Opened with the right source.
// Router state (vs a module flag) survives page remounts and aborted
// navigations.
// Router state (vs a module flag) survives StrictMode double-mount and
// aborted navigations.
const fromInApp = location.state?.fromInApp === true;
useEffect(() => {
if (fromInApp) {
@@ -52,34 +52,18 @@ export default function AIAssistantPage(): JSX.Element {
(s) => s.startNewConversation,
);
// Keep refs so the effect can read the latest store state without re-firing
// when it mutates the store mid-effect (it only depends on the URL param).
// Keep a ref so the effect can read latest conversations without re-firing
// when startNewConversation mutates the store mid-effect.
const conversationsRef = useRef(conversations);
conversationsRef.current = conversations;
const activeConversationIdRef = useRef(activeConversationId);
activeConversationIdRef.current = activeConversationId;
useEffect(() => {
// URL points at a known conversation → just activate it.
if (conversationId && conversationsRef.current[conversationId]) {
if (conversationsRef.current[conversationId]) {
setActiveConversation(conversationId);
return;
} else {
const newId = startNewConversation();
history.replace(ROUTES.AI_ASSISTANT.replace(':conversationId', newId));
}
// The URL has no usable conversation id (bare `/ai-assistant`, or a stale
// param). Prefer resuming the active conversation — including the
// rehydrating placeholder for the persisted thread — over minting a new
// one. This is what stops a throwaway blank chat from flashing as a
// second thread during load, and stops a duplicate when the page
// remounts during startup route churn (the active id is already set, so
// we resume instead of create). Starting fresh is the last resort, only
// when there is genuinely nothing to resume.
const activeId = activeConversationIdRef.current;
const resumeId =
activeId && conversationsRef.current[activeId]
? activeId
: startNewConversation();
history.replace(ROUTES.AI_ASSISTANT.replace(':conversationId', resumeId));
// Only re-run when the URL param changes, not when conversations mutates.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [conversationId]);

View File

@@ -1,181 +0,0 @@
import { MemoryRouter, Route } from 'react-router-dom';
// eslint-disable-next-line no-restricted-imports
import { render } from '@testing-library/react';
import ROUTES from 'constants/routes';
import { useAIAssistantStore } from 'container/AIAssistant/store/useAIAssistantStore';
jest.mock('api/common/logEvent', () => ({
__esModule: true,
default: jest.fn(),
}));
jest.mock('container/AIAssistant/ConversationView', () => ({
__esModule: true,
default: (): JSX.Element => <div data-testid="conversation-view" />,
}));
jest.mock('container/AIAssistant/components/ConversationsList', () => ({
__esModule: true,
default: (): JSX.Element => <div data-testid="conversations-list" />,
}));
jest.mock('components/Noz/Noz', () => ({
__esModule: true,
default: (): JSX.Element => <div data-testid="noz" />,
}));
jest.mock('container/AIAssistant/hooks/useAIAssistantAnalyticsContext', () => ({
normalizePage: (page: string): string => page,
useAIAssistantAnalyticsContext: (): unknown => ({ mode: 'page' }),
}));
// eslint-disable-next-line import/first
import AIAssistantPage from '../AIAssistantPage';
function renderAt(entry: string): { unmount: () => void } {
return render(
<MemoryRouter initialEntries={[entry]}>
<Route
exact
path={[ROUTES.AI_ASSISTANT_BASE, ROUTES.AI_ASSISTANT]}
component={AIAssistantPage}
/>
</MemoryRouter>,
);
}
function renderAtBase(): { unmount: () => void } {
return renderAt(ROUTES.AI_ASSISTANT_BASE);
}
function conversationCount(): number {
return Object.keys(useAIAssistantStore.getState().conversations).length;
}
function conversationIds(): string[] {
return Object.keys(useAIAssistantStore.getState().conversations);
}
function activeId(): string | null {
return useAIAssistantStore.getState().activeConversationId;
}
describe('AIAssistantPage', () => {
beforeEach(() => {
useAIAssistantStore.setState({
conversations: {},
streams: {},
activeConversationId: null,
});
});
it('opens exactly one conversation when navigating to /ai-assistant', () => {
const { unmount } = renderAtBase();
expect(conversationCount()).toBe(1);
unmount();
});
it('does not stack a second conversation when the page remounts at the bare URL (route churn)', () => {
// First mount at `/ai-assistant` creates one blank conversation and
// redirects to `/ai-assistant/:id`.
const { unmount } = renderAtBase();
expect(conversationCount()).toBe(1);
const firstId = conversationIds()[0];
// Startup route-list churn unmounts and remounts the page while the URL
// is momentarily back at the bare `/ai-assistant`. This previously
// created a second blank conversation — now it reuses the first.
unmount();
const { unmount: unmount2 } = renderAtBase();
expect(conversationCount()).toBe(1);
// The surviving conversation is the original one, resumed — not a fresh mint.
expect(conversationIds()).toStrictEqual([firstId]);
expect(activeId()).toBe(firstId);
unmount2();
});
it('activates the conversation named in the URL without creating a new one', () => {
useAIAssistantStore.setState({
conversations: {
existing: {
id: 'existing',
messages: [],
createdAt: 1,
updatedAt: 1,
},
},
streams: {},
activeConversationId: null,
});
const { unmount } = renderAt(
ROUTES.AI_ASSISTANT.replace(':conversationId', 'existing'),
);
expect(conversationCount()).toBe(1);
expect(activeId()).toBe('existing');
unmount();
});
it('resumes the active conversation on /ai-assistant/new instead of minting a new one', () => {
// The sidenav only routes to `/ai-assistant/new` as a fallback, but if an
// active conversation exists the page must resume it rather than spawn a
// throwaway blank thread for the unknown "new" param.
useAIAssistantStore.setState({
conversations: {
active: {
id: 'active',
messages: [],
createdAt: 1,
updatedAt: 1,
},
},
streams: {},
activeConversationId: 'active',
});
const { unmount } = renderAt(
ROUTES.AI_ASSISTANT.replace(':conversationId', 'new'),
);
expect(conversationCount()).toBe(1);
expect(conversationIds()).toStrictEqual(['active']);
expect(activeId()).toBe('active');
unmount();
});
it('resumes the persisted (hydrating) conversation during load instead of creating a second', () => {
// Simulates `onRehydrateStorage` priming the persisted active
// conversation as a hydrating placeholder before `fetchThreads` resolves.
useAIAssistantStore.setState({
conversations: {
persisted: {
id: 'persisted',
messages: [],
createdAt: 1,
updatedAt: 1,
isHydrating: true,
},
},
streams: {},
activeConversationId: 'persisted',
});
const { unmount } = renderAtBase();
// Opening the bare URL must resume the persisted conversation, not mint a
// throwaway blank alongside it (which flashed as a 2nd thread during load).
expect(conversationCount()).toBe(1);
expect(
Object.keys(useAIAssistantStore.getState().conversations),
).toStrictEqual(['persisted']);
unmount();
});
});

View File

@@ -29,7 +29,7 @@ type module struct {
}
func NewModule(store llmpricingruletypes.Store, flagger flagger.Flagger, querier querier.Querier) llmpricingrule.Module {
return &module{store: store, flagger: flagger, querier: querier}
return &module{store: store, flagger: flagger}
}
func (module *module) List(ctx context.Context, orgID valuer.UUID, offset, limit int, search string, isOverride *bool) ([]*llmpricingruletypes.LLMPricingRule, int, error) {

View File

@@ -39,48 +39,48 @@ func TestReducedStatementBuilder(t *testing.T) {
name: "gauge_sum_latest",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationLatest, metrictypes.SpaceAggregationSum),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(value, unix_milli) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, anyLast(last) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, argMax(value, unix_milli) AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum_last`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
},
},
{
name: "gauge_avg_avg",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum_last`, points.computed_at) AS value, argMax(`count_series`, points.computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
},
},
{
name: "gauge_min_min",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMin, metrictypes.SpaceAggregationMin),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`min`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(min) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, min(value) AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`min`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, min(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
},
},
{
name: "gauge_max_max",
query: reducedQuery("test.metric", metrictypes.GaugeType, metrictypes.Unspecified, metrictypes.TimeAggregationMax, metrictypes.SpaceAggregationMax),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(value) AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`max`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(value) AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`max`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, max(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
},
},
{
name: "counter_sum_rate",
query: reducedQuery("test.metric.sum", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationRate, metrictypes.SpaceAggregationSum),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric.sum", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), 0, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), "test.metric.sum", uint64(1746999600000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) / 300 AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric.sum", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), 0, "test.metric.sum", uint64(1746999600000), uint64(1747172760000), false, "test.metric.sum", uint64(1746999600000), uint64(1747172760000)},
},
},
{
name: "counter_avg_increase",
query: reducedQuery("test.metric", metrictypes.SumType, metrictypes.Cumulative, metrictypes.TimeAggregationIncrease, metrictypes.SpaceAggregationAvg),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), "test.metric", uint64(1746999600000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value, per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum`, points.computed_at) AS value, argMax(`count_series`, points.computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric", uint64(1746999600000), uint64(1747172760000), 0, "test.metric", uint64(1746999600000), uint64(1747172760000), false, "test.metric", uint64(1746999600000), uint64(1747172760000)},
},
},
{
@@ -103,16 +103,16 @@ func TestReducedStatementBuilder(t *testing.T) {
name: "histogram_p99",
query: reducedQuery("test.metric.bucket", metrictypes.HistogramType, metrictypes.Cumulative, metrictypes.TimeAggregationUnspecified, metrictypes.SpaceAggregationPercentile99),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(value) / 300 AS per_series_value FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum`, computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts, `le`), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT ts, `le`, multiIf(row_number() OVER rate_window = 1, nan, (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) < 0, per_series_value / (ts - lagInFrame(ts, 1) OVER rate_window), (per_series_value - lagInFrame(per_series_value, 1) OVER rate_window) / (ts - lagInFrame(ts, 1) OVER rate_window)) AS per_series_value FROM (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, max(max) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts, `le` ORDER BY fingerprint, ts) WINDOW rate_window AS (PARTITION BY fingerprint ORDER BY fingerprint, ts)), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, `le`, sum(value) / 300 AS per_series_value FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, `le`, argMax(`sum`, points.computed_at) AS value FROM signoz_metrics.distributed_samples_v4_reduced_sum_60s AS points INNER JOIN (SELECT fingerprint, JSONExtractString(labels, 'le') AS `le` FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint, `le`) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli, `le`) GROUP BY fingerprint, ts, `le`), __spatial_aggregation_cte AS (SELECT ts, `le`, sum(per_series_value) AS value FROM __temporal_aggregation_cte GROUP BY ts, `le`) SELECT ts, histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.990) AS value FROM __spatial_aggregation_cte GROUP BY ts ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric.bucket", uint64(1746921600000), uint64(1747172760000), "cumulative", false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), 0, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000), false, "test.metric.bucket", uint64(1746999900000), uint64(1747172760000)},
},
},
{
name: "summary_avg",
query: reducedQuery("test.metric", metrictypes.SummaryType, metrictypes.Unspecified, metrictypes.TimeAggregationAvg, metrictypes.SpaceAggregationAvg),
expected: qbtypes.Statement{
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT reduced_fingerprint AS fingerprint, unix_milli, argMax(`sum_last`, computed_at) AS value, argMax(`count_series`, computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY reduced_fingerprint, unix_milli) AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), "test.metric", uint64(1746999900000), uint64(1747172760000), false},
Query: "SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, sum(sum) / sum(count) AS per_series_value FROM signoz_metrics.distributed_samples_v4_agg_5m AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_1day WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND LOWER(temporality) LIKE LOWER(?) AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, ts ORDER BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, avg(per_series_value) AS value FROM __temporal_aggregation_cte WHERE isNaN(per_series_value) = ? GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) UNION ALL SELECT * FROM (WITH __temporal_aggregation_cte AS (SELECT fingerprint, toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(300)) AS ts, avg(value) AS per_series_value, avg(weight) AS per_series_weight FROM (SELECT points.reduced_fingerprint AS fingerprint, points.unix_milli AS unix_milli, argMax(`sum_last`, points.computed_at) AS value, argMax(`count_series`, points.computed_at) AS weight FROM signoz_metrics.distributed_samples_v4_reduced_last_60s AS points INNER JOIN (SELECT fingerprint FROM signoz_metrics.time_series_v4_reduced WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli <= ? AND __normalized = ? GROUP BY fingerprint) AS filtered_time_series ON points.reduced_fingerprint = filtered_time_series.fingerprint WHERE metric_name IN (?) AND unix_milli >= ? AND unix_milli < ? GROUP BY fingerprint, unix_milli) GROUP BY fingerprint, ts), __spatial_aggregation_cte AS (SELECT ts, sum(per_series_value) / sum(per_series_weight) AS value FROM __temporal_aggregation_cte GROUP BY ts) SELECT * FROM __spatial_aggregation_cte ORDER BY ts) ORDER BY ts",
Args: []any{"test.metric", uint64(1746921600000), uint64(1747172760000), "unspecified", false, "test.metric", uint64(1746999900000), uint64(1747172760000), 0, "test.metric", uint64(1746999900000), uint64(1747172760000), false, "test.metric", uint64(1746999900000), uint64(1747172760000)},
},
},
}

View File

@@ -338,19 +338,24 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
// dedup recomputed buckets: latest computed_at wins per (series, 60s bucket)
dedup := sqlbuilder.NewSelectBuilder()
dedup.Select("reduced_fingerprint AS fingerprint", "unix_milli")
dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS value", value))
if weight != "" {
dedup.SelectMore(fmt.Sprintf("argMax(%s, computed_at) AS weight", weight))
dedup.Select("points.reduced_fingerprint AS fingerprint", "points.unix_milli AS unix_milli")
for _, g := range query.GroupBy {
dedup.SelectMore(fmt.Sprintf("`%s`", g.Name))
}
dedup.From(fmt.Sprintf("%s.%s", DBName, WhichReducedSamplesTableToUse(agg.Type)))
dedup.SelectMore(fmt.Sprintf("argMax(%s, points.computed_at) AS value", value))
if weight != "" {
dedup.SelectMore(fmt.Sprintf("argMax(%s, points.computed_at) AS weight", weight))
}
dedup.From(fmt.Sprintf("%s.%s AS points", DBName, WhichReducedSamplesTableToUse(agg.Type)))
dedup.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
dedup.Where(
dedup.In("metric_name", agg.MetricName),
dedup.GTE("unix_milli", start),
dedup.LT("unix_milli", end),
)
dedup.GroupBy("reduced_fingerprint", "unix_milli")
dedupQuery, dedupArgs := dedup.BuildWithFlavor(sqlbuilder.ClickHouse)
dedup.GroupBy("fingerprint", "unix_milli")
dedup.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
dedupQuery, dedupArgs := dedup.BuildWithFlavor(sqlbuilder.ClickHouse, timeSeriesCTEArgs...)
sb := sqlbuilder.NewSelectBuilder()
sb.Select("fingerprint")
@@ -364,13 +369,11 @@ func (b *MetricQueryStatementBuilder) buildReducedTemporalAggregationCTE(
// denominator is reduced with avg
sb.SelectMore("avg(weight) AS per_series_weight")
}
sb.From(fmt.Sprintf("(%s) AS points", dedupQuery))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.From(fmt.Sprintf("(%s)", dedupQuery))
sb.GroupBy("fingerprint", "ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
initArgs := append(append([]any{}, dedupArgs...), timeSeriesCTEArgs...)
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, initArgs...)
q, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse, dedupArgs...)
return fmt.Sprintf("__temporal_aggregation_cte AS (%s)", q), args, true
}

View File

@@ -2,6 +2,7 @@ import os
from collections.abc import Callable, Generator
from datetime import datetime
from typing import Any
from uuid import uuid4
import clickhouse_connect
import clickhouse_connect.driver
@@ -10,37 +11,93 @@ import docker
import docker.errors
import pytest
from testcontainers.clickhouse import ClickHouseContainer
from testcontainers.core.container import Network
from testcontainers.core.container import DockerContainer, Network
from fixtures import reuse, types
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
CLICKHOUSE_USERNAME = "signoz"
CLICKHOUSE_PASSWORD = "password"
@pytest.fixture(name="clickhouse", scope="package")
def clickhouse(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
zookeeper: types.TestContainerDocker,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerClickhouse:
"""
Package-scoped fixture for Clickhouse TestContainer.
CUSTOM_FUNCTION_CONFIG = """
<functions>
<function>
<type>executable</type>
<name>histogramQuantile</name>
<return_type>Float64</return_type>
<argument>
<type>Array(Float64)</type>
<name>buckets</name>
</argument>
<argument>
<type>Array(Float64)</type>
<name>counts</name>
</argument>
<argument>
<type>Float64</type>
<name>quantile</name>
</argument>
<format>CSV</format>
<command>./histogramQuantile</command>
</function>
</functions>
"""
# Distributed inserts to a remote shard are async by default. We force
# sycn at the profile level for deterministic tests.
CLUSTER_USERS_CONFIG = """
<clickhouse>
<profiles>
<default>
<insert_distributed_sync>1</insert_distributed_sync>
</default>
</profiles>
</clickhouse>
"""
def render_remote_servers(shard_hosts: list[tuple[str, int]], secret: str | None = None) -> str:
"""Render the <remote_servers> block for a cluster named `cluster` with one
single-replica shard per (host, port).
"""
shards = "".join(
f"""
<shard>
<replica>
<host>{host}</host>
<port>{port}</port>
</replica>
</shard>"""
for host, port in shard_hosts
)
def create() -> types.TestContainerClickhouse:
version = request.config.getoption("--clickhouse-version")
# Multi-node clusters need `secret` because distributed queries otherwise
# authenticate as the `default` user, which the docker entrypoint restricts
# to localhost when a custom user is configured.
secret_block = (
f"""
<secret>{secret}</secret>"""
if secret
else ""
)
container = ClickHouseContainer(
image=f"clickhouse/clickhouse-server:{version}",
port=9000,
username="signoz",
password="password",
)
return f"""
<remote_servers>
<cluster>{secret_block}{shards}
</cluster>
</remote_servers>"""
cluster_config = f"""
def render_node_config(
zookeeper_address: str,
zookeeper_port: int,
shard: str,
remote_servers: str,
distributed_ddl_path: str = "/clickhouse/task_queue/ddl",
) -> str:
return f"""
<clickhouse>
<logger>
<level>information</level>
@@ -55,33 +112,23 @@ def clickhouse(
</logger>
<macros>
<shard>01</shard>
<shard>{shard}</shard>
<replica>01</replica>
</macros>
<zookeeper>
<node>
<host>{zookeeper.container_configs["2181"].address}</host>
<port>{zookeeper.container_configs["2181"].port}</port>
<host>{zookeeper_address}</host>
<port>{zookeeper_port}</port>
</node>
</zookeeper>
<remote_servers>
<cluster>
<shard>
<replica>
<host>127.0.0.1</host>
<port>9000</port>
</replica>
</shard>
</cluster>
</remote_servers>
{remote_servers}
<user_defined_executable_functions_config>*function.xml</user_defined_executable_functions_config>
<user_scripts_path>/var/lib/clickhouse/user_scripts/</user_scripts_path>
<distributed_ddl>
<path>/clickhouse/task_queue/ddl</path>
<path>{distributed_ddl_path}</path>
<profile>default</profile>
</distributed_ddl>
@@ -122,38 +169,66 @@ def clickhouse(
</clickhouse>
"""
custom_function_config = """
<functions>
<function>
<type>executable</type>
<name>histogramQuantile</name>
<return_type>Float64</return_type>
<argument>
<type>Array(Float64)</type>
<name>buckets</name>
</argument>
<argument>
<type>Array(Float64)</type>
<name>counts</name>
</argument>
<argument>
<type>Float64</type>
<name>quantile</name>
</argument>
<format>CSV</format>
<command>./histogramQuantile</command>
</function>
</functions>
"""
tmp_dir = tmpfs("clickhouse")
def install_histogram_quantile(container: ClickHouseContainer) -> None:
wrapped = container.get_wrapped_container()
exit_code, output = wrapped.exec_run(
[
"bash",
"-c",
(
'version="v0.0.1" && '
'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
"node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
"cd /tmp && "
'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
"tar -xzf histogram-quantile.tar.gz && "
"mkdir -p /var/lib/clickhouse/user_scripts && "
"mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && "
"chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile"
),
],
)
if exit_code != 0:
raise RuntimeError(f"Failed to install histogramQuantile binary: {output.decode()}")
def create_clickhouse( # pylint: disable=too-many-arguments,too-many-positional-arguments
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
keeper: types.TestContainerDocker,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
cache_key: str = "clickhouse",
version: str | None = None,
) -> types.TestContainerClickhouse:
coordinator = next(iter(keeper.container_configs.values()))
def create() -> types.TestContainerClickhouse:
clickhouse_version = version or request.config.getoption("--clickhouse-version")
container = ClickHouseContainer(
image=f"clickhouse/clickhouse-server:{clickhouse_version}",
port=9000,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
)
cluster_config = render_node_config(
zookeeper_address=coordinator.address,
zookeeper_port=coordinator.port,
shard="01",
remote_servers=render_remote_servers([("127.0.0.1", 9000)]),
)
tmp_dir = tmpfs(cache_key)
cluster_config_file_path = os.path.join(tmp_dir, "cluster.xml")
with open(cluster_config_file_path, "w", encoding="utf-8") as f:
f.write(cluster_config)
custom_function_file_path = os.path.join(tmp_dir, "custom-function.xml")
with open(custom_function_file_path, "w", encoding="utf-8") as f:
f.write(custom_function_config)
f.write(CUSTOM_FUNCTION_CONFIG)
container.with_volume_mapping(cluster_config_file_path, "/etc/clickhouse-server/config.d/cluster.xml")
container.with_volume_mapping(
@@ -163,27 +238,7 @@ def clickhouse(
container.with_network(network)
container.start()
# Download and install the histogramQuantile binary
wrapped = container.get_wrapped_container()
exit_code, output = wrapped.exec_run(
[
"bash",
"-c",
(
'version="v0.0.1" && '
'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
"node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
"cd /tmp && "
'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
"tar -xzf histogram-quantile.tar.gz && "
"mkdir -p /var/lib/clickhouse/user_scripts && "
"mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && "
"chmod +x /var/lib/clickhouse/user_scripts/histogramQuantile"
),
],
)
if exit_code != 0:
raise RuntimeError(f"Failed to install histogramQuantile binary: {output.decode()}")
install_histogram_quantile(container)
connection = clickhouse_connect.get_client(
user=container.username,
@@ -253,7 +308,7 @@ def clickhouse(
return reuse.wrap(
request,
pytestconfig,
"clickhouse",
cache_key,
empty=lambda: types.TestContainerSQL(
container=types.TestContainerDocker(id="", host_configs={}, container_configs={}),
conn=None,
@@ -265,6 +320,334 @@ def clickhouse(
)
@pytest.fixture(name="clickhouse", scope="package")
def clickhouse(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
zookeeper: types.TestContainerDocker,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerClickhouse:
"""
Package-scoped fixture for Clickhouse TestContainer.
"""
return create_clickhouse(
tmpfs=tmpfs,
network=network,
keeper=zookeeper,
request=request,
pytestconfig=pytestconfig,
)
def local_series_counts(
node_conns: list[clickhouse_connect.driver.client.Client],
table: str,
metric_name: str,
) -> list[int]:
"""Distinct series per node via the LOCAL (non-distributed) table."""
return [
int(
conn.query(
f"SELECT count(DISTINCT fingerprint) FROM signoz_metrics.{table} WHERE metric_name = %(metric_name)s",
parameters={"metric_name": metric_name},
).result_rows[0][0]
)
for conn in node_conns
]
def assert_spans_shards(
node_conns: list[clickhouse_connect.driver.client.Client],
table: str,
metric_name: str,
total: int,
) -> None:
"""Guard for distributed tests: a green run on a cluster proves nothing
unless the seeded series actually landed on more than one shard."""
counts = local_series_counts(node_conns, table, metric_name)
assert sum(counts) == total, f"expected {total} series in {table} across shards, got {counts}"
assert min(counts) > 0, f"seeded series in {table} all landed on one shard: {counts}"
@pytest.fixture(name="clickhouse_node_conns", scope="function")
def clickhouse_node_conns(
clickhouse: types.TestContainerClickhouse,
) -> Generator[list[clickhouse_connect.driver.client.Client], Any]:
"""Per-node clients (index 0 = the initiator) for asserting shard-local
state via the local, non-distributed tables. Empty for single-node
fixtures, which don't populate `nodes`."""
conns = [
clickhouse_connect.get_client(
user=clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME"],
password=clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD"],
host=node.host_configs["8123"].address,
port=node.host_configs["8123"].port,
)
for node in clickhouse.nodes
]
yield conns
for conn in conns:
conn.close()
KEEPER_CONFIG = """
<clickhouse>
<listen_host>0.0.0.0</listen_host>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>1</server_id>
<log_storage_path>/var/lib/clickhouse-keeper/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse-keeper/coordination/snapshots</snapshot_storage_path>
<coordination_settings>
<operation_timeout_ms>10000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<raft_logs_level>warning</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>localhost</hostname>
<port>9234</port>
</server>
</raft_configuration>
</keeper_server>
</clickhouse>
"""
def create_clickhouse_keeper(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
cache_key: str = "clickhousekeeper",
version: str | None = None,
) -> types.TestContainerDocker:
def create() -> types.TestContainerDocker:
keeper_version = version or request.config.getoption("--clickhouse-version")
tmp_dir = tmpfs(cache_key)
keeper_config_file_path = os.path.join(tmp_dir, "keeper_config.xml")
with open(keeper_config_file_path, "w", encoding="utf-8") as f:
f.write(KEEPER_CONFIG)
container = DockerContainer(image=f"clickhouse/clickhouse-keeper:{keeper_version}")
container.with_volume_mapping(keeper_config_file_path, "/etc/clickhouse-keeper/keeper_config.xml")
container.with_exposed_ports(9181)
container.with_network(network=network)
container.start()
return types.TestContainerDocker(
id=container.get_wrapped_container().id,
host_configs={
"9181": types.TestContainerUrlConfig(
scheme="tcp",
address=container.get_container_host_ip(),
port=container.get_exposed_port(9181),
)
},
container_configs={
"9181": types.TestContainerUrlConfig(
scheme="tcp",
address=container.get_wrapped_container().name,
port=9181,
)
},
)
def delete(container: types.TestContainerDocker):
client = docker.from_env()
try:
client.containers.get(container_id=container.id).stop()
client.containers.get(container_id=container.id).remove(v=True)
except docker.errors.NotFound:
logger.info(
"Skipping removal of ClickHouse Keeper, Keeper(%s) not found. Maybe it was manually removed?",
{"id": container.id},
)
def restore(cache: dict) -> types.TestContainerDocker:
return types.TestContainerDocker.from_cache(cache)
return reuse.wrap(
request,
pytestconfig,
cache_key,
lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
create,
delete,
restore,
)
def create_clickhouse_cluster( # pylint: disable=too-many-arguments,too-many-positional-arguments
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
keeper: types.TestContainerDocker,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
cache_key: str = "clickhouse_cluster",
shards: int = 2,
version: str | None = None,
) -> types.TestContainerClickhouse:
"""
To some extent, taken inspiration from how ClickHouse's own integration
harness composes real clusters: deterministic hostnames
(network aliases), per-node shard macros, and a shared cluster definition
named `cluster`.
`conn`/`env` point at node 1 i.e the initiator every query-service query and
migration goes through. Per-node containers are exposed via `nodes` so
tests can assert shard-local state. `keeper` is any coordination service
(ZooKeeper or ClickHouse Keeper).
"""
coordinator = next(iter(keeper.container_configs.values()))
def create() -> types.TestContainerClickhouse:
clickhouse_version = version or request.config.getoption("--clickhouse-version")
# Unique aliases per creation: docker allows duplicate network aliases
# (DNS round-robin), so a stale cluster must never share names with a
# fresh one.
suffix = uuid4().hex[:6]
aliases = [f"signoz-ch-{suffix}-{i:02d}" for i in range(1, shards + 1)]
remote_servers = render_remote_servers([(alias, 9000) for alias in aliases], secret=cache_key)
# Own DDL queue path: the keeper instance may be shared with other
# environments under --reuse; its DDL queue stays separate.
distributed_ddl_path = f"/clickhouse/{cache_key}-{suffix}/task_queue/ddl"
nodes: list[types.TestContainerDocker] = []
started: list[ClickHouseContainer] = []
try:
for i, alias in enumerate(aliases, start=1):
node_config = render_node_config(
zookeeper_address=coordinator.address,
zookeeper_port=coordinator.port,
shard=f"{i:02d}",
remote_servers=remote_servers,
distributed_ddl_path=distributed_ddl_path,
)
tmp_dir = tmpfs(f"clickhouse-{suffix}-{i:02d}")
cluster_config_file_path = os.path.join(tmp_dir, "cluster.xml")
with open(cluster_config_file_path, "w", encoding="utf-8") as f:
f.write(node_config)
custom_function_file_path = os.path.join(tmp_dir, "custom-function.xml")
with open(custom_function_file_path, "w", encoding="utf-8") as f:
f.write(CUSTOM_FUNCTION_CONFIG)
users_config_file_path = os.path.join(tmp_dir, "users.xml")
with open(users_config_file_path, "w", encoding="utf-8") as f:
f.write(CLUSTER_USERS_CONFIG)
container = ClickHouseContainer(
image=f"clickhouse/clickhouse-server:{clickhouse_version}",
port=9000,
username=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
)
container.with_volume_mapping(cluster_config_file_path, "/etc/clickhouse-server/config.d/cluster.xml")
container.with_volume_mapping(custom_function_file_path, "/etc/clickhouse-server/custom-function.xml")
container.with_volume_mapping(users_config_file_path, "/etc/clickhouse-server/users.d/integration-cluster.xml")
container.with_network(network)
container.with_network_aliases(alias)
container.start()
started.append(container)
install_histogram_quantile(container)
nodes.append(
types.TestContainerDocker(
id=container.get_wrapped_container().id,
host_configs={
"9000": types.TestContainerUrlConfig(
"tcp",
container.get_container_host_ip(),
container.get_exposed_port(9000),
),
"8123": types.TestContainerUrlConfig(
"tcp",
container.get_container_host_ip(),
container.get_exposed_port(8123),
),
},
container_configs={
"9000": types.TestContainerUrlConfig("tcp", alias, 9000),
"8123": types.TestContainerUrlConfig("tcp", alias, 8123),
},
)
)
except Exception:
for container in started:
container.stop()
raise
connection = clickhouse_connect.get_client(
user=CLICKHOUSE_USERNAME,
password=CLICKHOUSE_PASSWORD,
host=nodes[0].host_configs["8123"].address,
port=nodes[0].host_configs["8123"].port,
)
return types.TestContainerClickhouse(
container=nodes[0],
conn=connection,
env={
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN": f"tcp://{CLICKHOUSE_USERNAME}:{CLICKHOUSE_PASSWORD}@{aliases[0]}:{9000}",
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME": CLICKHOUSE_USERNAME,
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD": CLICKHOUSE_PASSWORD,
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER": "cluster",
},
nodes=nodes,
)
def delete(resource: types.TestContainerClickhouse) -> None:
client = docker.from_env()
for node in resource.nodes or [resource.container]:
try:
client.containers.get(container_id=node.id).stop()
client.containers.get(container_id=node.id).remove(v=True)
except docker.errors.NotFound:
logger.info(
"Skipping removal of Clickhouse cluster node, node(%s) not found. Maybe it was manually removed?",
{"id": node.id},
)
def restore(cache: dict) -> types.TestContainerClickhouse:
nodes = [types.TestContainerDocker.from_cache(node) for node in cache["nodes"]]
env = cache["env"]
host_config = nodes[0].host_configs["8123"]
conn = clickhouse_connect.get_client(
user=env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_USERNAME"],
password=env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PASSWORD"],
host=host_config.address,
port=host_config.port,
)
return types.TestContainerClickhouse(
container=nodes[0],
conn=conn,
env=env,
nodes=nodes,
)
return reuse.wrap(
request,
pytestconfig,
cache_key,
empty=lambda: types.TestContainerClickhouse(
container=types.TestContainerDocker(id="", host_configs={}, container_configs={}),
conn=None,
env={},
),
create=create,
delete=delete,
restore=restore,
)
@pytest.fixture(name="check_query_log")
def check_query_log(
signoz: types.SigNoz,

View File

@@ -18,19 +18,22 @@ from fixtures.logger import setup_logger
logger = setup_logger(__name__)
@pytest.fixture(name="zeus", scope="package")
def zeus(
ZEUS_NETWORK_ALIAS = "signoz-zeus-it"
def create_zeus(
network: Network,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
cache_key: str = "zeus",
alias: str | None = None,
) -> types.TestContainerDocker:
"""
Package-scoped fixture for running zeus
"""
def create() -> types.TestContainerDocker:
container = WireMockContainer(image="wiremock/wiremock:2.35.1-1", secure=False)
container.with_network(network)
if alias:
container.with_network_aliases(alias)
container.start()
return types.TestContainerDocker(
@@ -42,7 +45,7 @@ def zeus(
container.get_exposed_port(8080),
)
},
container_configs={"8080": types.TestContainerUrlConfig("http", container.get_wrapped_container().name, 8080)},
container_configs={"8080": types.TestContainerUrlConfig("http", alias or container.get_wrapped_container().name, 8080)},
)
def delete(container: types.TestContainerDocker):
@@ -62,7 +65,7 @@ def zeus(
return reuse.wrap(
request,
pytestconfig,
"zeus",
cache_key,
lambda: types.TestContainerDocker(id="", host_configs={}, container_configs={}),
create,
delete,
@@ -70,6 +73,18 @@ def zeus(
)
@pytest.fixture(name="zeus", scope="package")
def zeus(
network: Network,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerDocker:
"""
Package-scoped fixture for running zeus
"""
return create_zeus(network=network, request=request, pytestconfig=pytestconfig)
@pytest.fixture(name="gateway", scope="package")
def gateway(
network: Network,

51
tests/fixtures/metricreduction.py vendored Normal file
View File

@@ -0,0 +1,51 @@
import datetime
from collections.abc import Sequence
from fixtures.metrics import MetricsBufferSample, MetricsBufferTimeSeries
def build_ruled_gauge_buffer(
metric_name: str,
base_epoch: int,
services: Sequence[str],
pods_per_service: int,
minutes: int,
value: float = 1.0,
) -> tuple[list[MetricsBufferTimeSeries], list[MetricsBufferSample]]:
"""Collector-shaped buffer rows for a gauge under a reduction rule that
keeps `service`: per raw series a raw series row (is_reduced=false, full
labels, reduced_fingerprint -> group) plus the group's reduced series row
(is_reduced=true, kept labels), and one raw sample per series per minute
carrying both fingerprints. Returns (time_series, samples) for
insert_buffer_metrics."""
reduced_series = {
service: MetricsBufferTimeSeries(
metric_name=metric_name,
labels={"service": service},
timestamp=datetime.datetime.fromtimestamp(base_epoch, tz=datetime.UTC),
is_reduced=True,
)
for service in services
}
raw_series = [
MetricsBufferTimeSeries(
metric_name=metric_name,
labels={"service": service, "pod": f"pod-{service}-{i}"},
timestamp=datetime.datetime.fromtimestamp(base_epoch, tz=datetime.UTC),
reduced_fingerprint=reduced_series[service].fingerprint,
)
for service in services
for i in range(pods_per_service)
]
samples = [
MetricsBufferSample(
metric_name=metric_name,
fingerprint=ts.fingerprint,
timestamp=datetime.datetime.fromtimestamp(base_epoch + minute * 60, tz=datetime.UTC),
value=value,
reduced_fingerprint=ts.reduced_fingerprint,
)
for ts in raw_series
for minute in range(minutes)
]
return raw_series + list(reduced_series.values()), samples

View File

@@ -11,6 +11,14 @@ import pytest
from fixtures import types
from fixtures.time import parse_timestamp
_REDUCED_METRICS_TABLES_TO_TRUNCATE = [
"time_series_v4_reduced",
"samples_v4_reduced_last_60s",
"samples_v4_reduced_sum_60s",
"time_series_v4_buffer",
"samples_v4_buffer",
]
class MetricsTimeSeries(ABC):
"""Represents a row in the time_series_v4 table."""
@@ -414,6 +422,267 @@ class Metrics(ABC):
return metrics
class MetricsReducedTimeSeries(ABC):
"""Represents a row in the time_series_v4_reduced table i.e what
the time_series_v4_reduced_mv materializes for a metric under a
reduction rule. One row per kept-label group. `fingerprint` holds the
reduced fingerprint and `labels` contains only the kept labels.
The fingerprint recipe (md5, like MetricsTimeSeries) does not match the
collector's real hash; it only needs to be consistent with the
reduced_fingerprint used in the reduced samples rows.
"""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
kept_labels: dict[str, str],
timestamp: datetime.datetime,
temporality: str = "Unspecified",
description: str = "",
unit: str = "",
type_: str = "Gauge",
is_monotonic: bool = False,
env: str = "default",
) -> None:
kept_labels = dict(kept_labels)
kept_labels["__name__"] = metric_name
self.env = env
# mirror time_series_v4_reduced_mv: monotonic cumulative counters are
# reduced as deltas
if temporality == "Cumulative" and is_monotonic:
temporality = "Delta"
self.temporality = temporality
self.metric_name = metric_name
self.description = description
self.unit = unit
self.type = type_
self.is_monotonic = is_monotonic
self.labels = json.dumps(kept_labels, separators=(",", ":"))
self.attrs = kept_labels
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.normalized = False
fingerprint_str = metric_name + self.labels
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.description,
self.unit,
self.type,
self.is_monotonic,
self.fingerprint,
self.unix_milli,
self.labels,
self.attrs,
{},
{},
self.normalized,
]
class MetricsReducedSampleLast60s(ABC):
"""Represents a row in the samples_v4_reduced_last_60s table. One 60s
bucket per reduced group, as the samples_v4_reduced_last_60s_mv refresh
would emit it (gauges and non-monotonic cumulative sums)."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
reduced_fingerprint: np.uint64,
timestamp: datetime.datetime,
sum_last: float,
min_value: float,
max_value: float,
sum_values: float,
count_series: int,
count_samples: int,
temporality: str = "Unspecified",
env: str = "default",
computed_at: datetime.datetime | None = None,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.reduced_fingerprint = reduced_fingerprint
# buckets are 60s-aligned: intDiv(unix_milli, 60000) * 60000
self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
self.sum_last = np.float64(sum_last)
self.min = np.float64(min_value)
self.max = np.float64(max_value)
self.sum_values = np.float64(sum_values)
self.count_series = np.uint64(count_series)
self.count_samples = np.uint64(count_samples)
# the refresh stamps now(); default to shortly after the bucket closes
if computed_at is None:
computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
self.computed_at = computed_at
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.reduced_fingerprint,
self.unix_milli,
self.sum_last,
self.min,
self.max,
self.sum_values,
self.count_series,
self.count_samples,
self.computed_at,
]
class MetricsReducedSampleSum60s(ABC):
"""Represents a row in the samples_v4_reduced_sum_60s table. One 60s
bucket per reduced group for delta counters and histograms."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
reduced_fingerprint: np.uint64,
timestamp: datetime.datetime,
sum_value: float,
count_series: int,
count_samples: int,
temporality: str = "Delta",
env: str = "default",
computed_at: datetime.datetime | None = None,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.reduced_fingerprint = reduced_fingerprint
self.unix_milli = np.int64((int(timestamp.timestamp() * 1e3) // 60000) * 60000)
self.sum = np.float64(sum_value)
self.count_series = np.uint64(count_series)
self.count_samples = np.uint64(count_samples)
if computed_at is None:
computed_at = datetime.datetime.fromtimestamp(int(self.unix_milli) / 1e3, tz=datetime.UTC) + datetime.timedelta(seconds=180)
self.computed_at = computed_at
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.reduced_fingerprint,
self.unix_milli,
self.sum,
self.count_series,
self.count_samples,
self.computed_at,
]
class MetricsBufferTimeSeries(ABC):
"""Represents a row in the time_series_v4_buffer table. This is the collector's
universal landing target under cardinality control. For a ruled metric the
collector writes two rows per series: the raw one (is_reduced=false, full
labels, reduced_fingerprint pointing at its group) and the group's reduced
one (is_reduced=true, kept labels, fingerprint = reduced fingerprint)."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
labels: dict[str, str],
timestamp: datetime.datetime,
reduced_fingerprint: np.uint64 | int = 0,
is_reduced: bool = False,
temporality: str = "Unspecified",
description: str = "",
unit: str = "",
type_: str = "Gauge",
is_monotonic: bool = False,
env: str = "default",
) -> None:
labels = dict(labels)
labels["__name__"] = metric_name
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.description = description
self.unit = unit
self.type = type_
self.is_monotonic = is_monotonic
self.reduced_fingerprint = np.uint64(reduced_fingerprint)
self.is_reduced = is_reduced
self.labels = json.dumps(labels, separators=(",", ":"))
self.attrs = labels
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.normalized = False
fingerprint_str = metric_name + self.labels
self.fingerprint = np.uint64(int(hashlib.md5(fingerprint_str.encode()).hexdigest()[:16], 16))
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.description,
self.unit,
self.type,
self.is_monotonic,
self.fingerprint,
self.reduced_fingerprint,
self.is_reduced,
self.unix_milli,
self.labels,
self.attrs,
{},
{},
self.normalized,
]
class MetricsBufferSample(ABC):
"""Represents a row in the samples_v4_buffer table. Ruled samples carry
the raw fingerprint plus the group's reduced_fingerprint; unruled samples
have reduced_fingerprint = 0."""
def __init__( # pylint: disable=too-many-arguments
self,
metric_name: str,
fingerprint: np.uint64,
timestamp: datetime.datetime,
value: float,
reduced_fingerprint: np.uint64 | int = 0,
is_monotonic: bool = False,
temporality: str = "Unspecified",
env: str = "default",
flags: int = 0,
) -> None:
self.env = env
self.temporality = temporality
self.metric_name = metric_name
self.fingerprint = fingerprint
self.reduced_fingerprint = np.uint64(reduced_fingerprint)
self.is_monotonic = is_monotonic
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.value = np.float64(value)
self.flags = np.uint32(flags)
def to_row(self) -> list:
return [
self.env,
self.temporality,
self.metric_name,
self.fingerprint,
self.reduced_fingerprint,
self.is_monotonic,
self.unix_milli,
self.value,
self.flags,
]
def insert_metrics_to_clickhouse(conn, metrics: list[Metrics]) -> None:
"""
Insert metrics into ClickHouse tables.
@@ -576,6 +845,163 @@ def insert_metrics(
)
def insert_reduced_metrics_to_clickhouse(
conn,
time_series: list[MetricsReducedTimeSeries],
last_samples: list[MetricsReducedSampleLast60s] | None = None,
sum_samples: list[MetricsReducedSampleSum60s] | None = None,
) -> None:
"""Insert reduced series into distributed_time_series_v4_reduced and 60s
buckets into the reduced samples tables. These tables exist only when
the schema migrator version includes the metrics cardinality-control
migration."""
if time_series:
conn.insert(
database="signoz_metrics",
table="distributed_time_series_v4_reduced",
column_names=[
"env",
"temporality",
"metric_name",
"description",
"unit",
"type",
"is_monotonic",
"fingerprint",
"unix_milli",
"labels",
"attrs",
"scope_attrs",
"resource_attrs",
"__normalized",
],
data=[ts.to_row() for ts in time_series],
)
if last_samples:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4_reduced_last_60s",
column_names=[
"env",
"temporality",
"metric_name",
"reduced_fingerprint",
"unix_milli",
"sum_last",
"min",
"max",
"sum_values",
"count_series",
"count_samples",
"computed_at",
],
data=[sample.to_row() for sample in last_samples],
)
if sum_samples:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4_reduced_sum_60s",
column_names=[
"env",
"temporality",
"metric_name",
"reduced_fingerprint",
"unix_milli",
"sum",
"count_series",
"count_samples",
"computed_at",
],
data=[sample.to_row() for sample in sum_samples],
)
def insert_buffer_metrics_to_clickhouse(
conn,
time_series: list[MetricsBufferTimeSeries],
samples: list[MetricsBufferSample],
) -> None:
if time_series:
conn.insert(
database="signoz_metrics",
table="distributed_time_series_v4_buffer",
column_names=[
"env",
"temporality",
"metric_name",
"description",
"unit",
"type",
"is_monotonic",
"fingerprint",
"reduced_fingerprint",
"is_reduced",
"unix_milli",
"labels",
"attrs",
"scope_attrs",
"resource_attrs",
"__normalized",
],
data=[ts.to_row() for ts in time_series],
)
if samples:
conn.insert(
database="signoz_metrics",
table="distributed_samples_v4_buffer",
column_names=[
"env",
"temporality",
"metric_name",
"fingerprint",
"reduced_fingerprint",
"is_monotonic",
"unix_milli",
"value",
"flags",
],
data=[sample.to_row() for sample in samples],
)
@pytest.fixture(name="insert_reduced_metrics", scope="function")
def insert_reduced_metrics(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[..., None], Any]:
def _insert_reduced_metrics(
time_series: list[MetricsReducedTimeSeries],
last_samples: list[MetricsReducedSampleLast60s] | None = None,
sum_samples: list[MetricsReducedSampleSum60s] | None = None,
) -> None:
insert_reduced_metrics_to_clickhouse(clickhouse.conn, time_series, last_samples, sum_samples)
yield _insert_reduced_metrics
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
@pytest.fixture(name="insert_buffer_metrics", scope="function")
def insert_buffer_metrics(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[..., None], Any]:
def _insert_buffer_metrics(
time_series: list[MetricsBufferTimeSeries],
samples: list[MetricsBufferSample],
) -> None:
insert_buffer_metrics_to_clickhouse(clickhouse.conn, time_series, samples)
yield _insert_buffer_metrics
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
for table in _REDUCED_METRICS_TABLES_TO_TRUNCATE:
clickhouse.conn.query(f"TRUNCATE TABLE signoz_metrics.{table} ON CLUSTER '{cluster}' SYNC")
@pytest.fixture(name="remove_metrics_ttl_and_storage_settings", scope="function")
def remove_metrics_ttl_and_storage_settings(signoz: types.SigNoz):
"""

View File

@@ -8,27 +8,30 @@ from fixtures.logger import setup_logger
logger = setup_logger(__name__)
def create_migrator(
def create_migrator( # pylint: disable=too-many-arguments,too-many-positional-arguments
network: Network,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
cache_key: str = "migrator",
env_overrides: dict | None = None,
version: str | None = None,
) -> types.Operation:
"""
Factory function for running schema migrations.
Accepts optional env_overrides to customize the migrator environment.
Accepts optional env_overrides to customize the migrator environment, and
an optional version to pin a schema-migrator release different from the
--schema-migrator-version option.
"""
def create() -> None:
version = request.config.getoption("--schema-migrator-version")
migrator_version = version or request.config.getoption("--schema-migrator-version")
client = docker.from_env()
environment = dict(env_overrides) if env_overrides else {}
container = client.containers.run(
image=f"signoz/signoz-schema-migrator:{version}",
image=f"signoz/signoz-schema-migrator:{migrator_version}",
command=f"sync --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
detach=True,
auto_remove=False,
@@ -47,7 +50,7 @@ def create_migrator(
container.remove()
container = client.containers.run(
image=f"signoz/signoz-schema-migrator:{version}",
image=f"signoz/signoz-schema-migrator:{migrator_version}",
command=f"async --replication=true --cluster-name=cluster --up= --dsn={clickhouse.env['SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN']}",
detach=True,
auto_remove=False,

View File

@@ -189,6 +189,35 @@ def make_query_request(
)
def aligned_epoch(ago: timedelta, step_seconds: int = DEFAULT_STEP_INTERVAL) -> int:
"""Epoch seconds for `now - ago`, floored to a step boundary so seeded
points land exactly on the query's toStartOfInterval buckets."""
return (int((datetime.now(tz=UTC) - ago).timestamp()) // step_seconds) * step_seconds
def query_metric_values( # pylint: disable=too-many-arguments,too-many-positional-arguments
signoz: types.SigNoz,
token: str,
metric_name: str,
start_epoch: int,
end_epoch: int,
time_agg: str,
space_agg: str,
step_interval: int = DEFAULT_STEP_INTERVAL,
) -> list[dict]:
"""Run a single metrics builder query over [start_epoch, end_epoch) in
epoch seconds and return its series values sorted by timestamp."""
response = make_query_request(
signoz,
token,
start_ms=start_epoch * 1000,
end_ms=end_epoch * 1000,
queries=[build_builder_query("A", metric_name, time_agg, space_agg, step_interval=step_interval)],
)
assert response.status_code == HTTPStatus.OK, response.text
return sorted(get_series_values(response.json(), "A"), key=lambda v: v["timestamp"])
def build_builder_query(
name: str,
metric_name: str,

View File

@@ -1,4 +1,4 @@
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import Literal
from urllib.parse import urljoin
@@ -84,11 +84,16 @@ class TestContainerClickhouse:
container: TestContainerDocker
conn: clickhouse_connect.driver.client.Client
env: dict[str, str]
# Per-node containers when running a multi-node cluster. Empty for the
# default single-node setup; nodes[0] is the node `conn`/`env` point at
# (the initiator every query goes through).
nodes: list[TestContainerDocker] = field(default_factory=list)
def __cache__(self) -> dict:
return {
"container": self.container.__cache__(),
"env": self.env,
"nodes": [node.__cache__() for node in self.nodes],
}
def __log__(self) -> str:

View File

@@ -0,0 +1,52 @@
import clickhouse_connect.driver.client
from fixtures import types
TOTAL_ROWS = 64
def test_topology(
clickhouse: types.TestContainerClickhouse,
clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
) -> None:
aliases = {node.container_configs["9000"].address for node in clickhouse.nodes}
# Every node sees the same 2-shard cluster definition and identifies
# exactly itself as the local replica
for i, conn in enumerate(clickhouse_node_conns, start=1):
rows = conn.query("SELECT shard_num, host_name, is_local FROM system.clusters WHERE cluster = 'cluster' ORDER BY shard_num").result_rows
assert [row[0] for row in rows] == [1, 2], f"node {i}: expected 2 shards, got {rows}"
assert {row[1] for row in rows} == aliases, f"node {i}: cluster hosts {rows} != node aliases {aliases}"
local = [row[0] for row in rows if row[2]]
assert local == [i], f"node {i}: expected to be local for shard {i} only, got {local}"
def test_replicated_distributed_round_trip(
clickhouse: types.TestContainerClickhouse,
clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
) -> None:
# ON CLUSTER DDL reaches both nodes, Replicated engines register with the
# keeper via per-node macros, and a sharded Distributed insert scatters rows
# across shards while the distributed read returns the union.
conn = clickhouse.conn
try:
conn.query("CREATE DATABASE IF NOT EXISTS it_cluster ON CLUSTER 'cluster'")
conn.query("CREATE TABLE it_cluster.events ON CLUSTER 'cluster' (id UInt64, payload String) ENGINE = ReplicatedMergeTree ORDER BY id")
conn.query("CREATE TABLE it_cluster.distributed_events ON CLUSTER 'cluster' AS it_cluster.events ENGINE = Distributed('cluster', 'it_cluster', 'events', cityHash64(id))")
conn.insert(
database="it_cluster",
table="distributed_events",
column_names=["id", "payload"],
data=[[i, f"payload-{i:03d}"] for i in range(TOTAL_ROWS)],
)
distributed_count = int(conn.query("SELECT count() FROM it_cluster.distributed_events").result_rows[0][0])
assert distributed_count == TOTAL_ROWS
local_counts = [int(node_conn.query("SELECT count() FROM it_cluster.events").result_rows[0][0]) for node_conn in clickhouse_node_conns]
assert sum(local_counts) == TOTAL_ROWS, f"local counts {local_counts} do not add up to {TOTAL_ROWS}"
assert min(local_counts) > 0, f"all rows landed on one shard: {local_counts}"
finally:
conn.query("DROP DATABASE IF EXISTS it_cluster ON CLUSTER 'cluster' SYNC")

View File

@@ -0,0 +1,47 @@
from collections.abc import Generator
from typing import Any
import pytest
from testcontainers.core.container import Network
from fixtures import types
from fixtures.clickhouse import create_clickhouse_cluster, create_clickhouse_keeper
CLICKHOUSE_VERSION = "25.12.5"
@pytest.fixture(name="keeper", scope="package")
def keeper_cluster(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerDocker:
return create_clickhouse_keeper(
tmpfs=tmpfs,
network=network,
request=request,
pytestconfig=pytestconfig,
cache_key="keeper_cluster",
version=CLICKHOUSE_VERSION,
)
@pytest.fixture(name="clickhouse", scope="package")
def clickhouse_cluster(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
keeper: types.TestContainerDocker,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerClickhouse:
return create_clickhouse_cluster(
tmpfs=tmpfs,
network=network,
keeper=keeper,
request=request,
pytestconfig=pytestconfig,
cache_key="clickhouse_cluster",
shards=2,
version=CLICKHOUSE_VERSION,
)

View File

@@ -0,0 +1,203 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
import clickhouse_connect.driver.client
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.clickhouse import assert_spans_shards
from fixtures.metrics import (
Metrics,
MetricsReducedSampleLast60s,
MetricsReducedTimeSeries,
)
from fixtures.querier import aligned_epoch, query_metric_values
def test_stitch_across_epoch(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_metrics: Callable[[list[Metrics]], None],
insert_reduced_metrics: Callable[..., None],
clickhouse_node_conns: list[clickhouse_connect.driver.client.Client],
) -> None:
"""Before the rule activates, samples live in the raw tables; after, only
the reduced 60s tables have data. One query spanning the boundary must
stitch the two branches into a continuous series with no gap and no double
counting: 32 raw series at 2.0 collapse into 16 groups whose sum_last is
4.0, so the summed value stays 320 per step across the epoch. Enough
series to guarantee both shards hold data (guarded below), so the totals
also prove the raw and reduced joins execute shard-local."""
metric_name = "test_reduction_stitch"
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
services = [f"svc-{i:02d}" for i in range(16)]
# first 30 minutes: raw samples (2 pods per service, one sample per minute)
insert_metrics(
[
Metrics(
metric_name=metric_name,
labels={"service": service, "pod": f"{service}-pod-{pod}"},
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
value=2.0,
type_="Gauge",
is_monotonic=False,
)
for service in services
for pod in range(2)
for minute in range(30)
]
)
# next 30 minutes: reduced 60s buckets (one group per service)
time_series = [
MetricsReducedTimeSeries(
metric_name=metric_name,
kept_labels={"service": service},
timestamp=datetime.fromtimestamp(base_epoch + 30 * 60, tz=UTC),
)
for service in services
]
insert_reduced_metrics(
time_series,
[
MetricsReducedSampleLast60s(
metric_name=metric_name,
reduced_fingerprint=ts.fingerprint,
timestamp=datetime.fromtimestamp(base_epoch + (30 + minute) * 60, tz=UTC),
sum_last=4.0,
min_value=2.0,
max_value=2.0,
sum_values=4.0,
count_series=2,
count_samples=2,
)
for ts in time_series
for minute in range(30)
],
)
assert_spans_shards(clickhouse_node_conns, "time_series_v4", metric_name, total=len(services) * 2)
assert_spans_shards(clickhouse_node_conns, "time_series_v4_reduced", metric_name, total=len(services))
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 3600, "sum", "sum", step_interval=300)
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(12)]
assert [v["value"] for v in values] == [320.0] * 12
@pytest.mark.parametrize(
"space_agg, expected",
[
("sum", 12.0), # sum_last: 4 + 8
("avg", 3.0), # sum(sum_last) / sum(count_series): 12 / 4
("min", 1.0), # min(min)
("max", 6.0), # max(max)
],
)
def test_space_aggregations(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_reduced_metrics: Callable[..., None],
space_agg: str,
expected: float,
) -> None:
"""Space aggregations read the reduced pre-aggregated columns: sum/avg
from sum_last with the count_series weight, min/max from the min/max
columns."""
metric_name = f"test_reduction_space_{space_agg}"
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
groups = [
# (service, sum_last, min, max, count_series)
("a", 4.0, 1.0, 3.0, 2),
("b", 8.0, 2.0, 6.0, 2),
]
time_series = {
service: MetricsReducedTimeSeries(
metric_name=metric_name,
kept_labels={"service": service},
timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
)
for service, _, _, _, _ in groups
}
insert_reduced_metrics(
list(time_series.values()),
[
MetricsReducedSampleLast60s(
metric_name=metric_name,
reduced_fingerprint=time_series[service].fingerprint,
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
sum_last=sum_last,
min_value=min_value,
max_value=max_value,
sum_values=sum_last,
count_series=count_series,
count_samples=count_series,
)
for service, sum_last, min_value, max_value, count_series in groups
for minute in range(20)
],
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 20 * 60, "avg", space_agg, step_interval=300)
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
assert [v["value"] for v in values] == [expected] * 4
def test_dedup_latest_computed_at_wins(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_reduced_metrics: Callable[..., None],
) -> None:
"""The refreshable MVs re-emit every bucket on each refresh with a newer
computed_at (APPEND mode); reads must dedup to the latest version per
(series, bucket). Recompute the same buckets with a newer computed_at and
a different value: only the newer value may be counted."""
metric_name = "test_reduction_dedup"
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
time_series = [
MetricsReducedTimeSeries(
metric_name=metric_name,
kept_labels={"service": service},
timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
)
for service in ("a", "b")
]
def buckets(sum_last: float, computed_at_offset_seconds: int) -> list[MetricsReducedSampleLast60s]:
return [
MetricsReducedSampleLast60s(
metric_name=metric_name,
reduced_fingerprint=ts.fingerprint,
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
sum_last=sum_last,
min_value=sum_last,
max_value=sum_last,
sum_values=sum_last,
count_series=1,
count_samples=1,
computed_at=datetime.fromtimestamp(base_epoch + minute * 60 + computed_at_offset_seconds, tz=UTC),
)
for ts in time_series
for minute in range(10)
]
# first refresh emits 1.0; a later refresh recomputes the same buckets to 5.0
insert_reduced_metrics(time_series, buckets(sum_last=1.0, computed_at_offset_seconds=120))
insert_reduced_metrics(time_series, buckets(sum_last=5.0, computed_at_offset_seconds=180))
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 10 * 60, "sum", "sum", step_interval=300)
# 2 groups x 5 buckets x 5.0 per step; 1.0 rows must not contribute
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(2)]
assert [v["value"] for v in values] == [50.0] * 2

View File

@@ -0,0 +1,70 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
import pytest
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.metrics import (
MetricsReducedSampleSum60s,
MetricsReducedTimeSeries,
)
from fixtures.querier import aligned_epoch, query_metric_values
@pytest.mark.parametrize(
"time_agg, expected",
[
# 2 groups x 5 buckets x 30.0 per 300s step
("rate", 1.0), # 300 / 300s
("increase", 300.0),
],
)
def test_counter_rate_and_increase(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_reduced_metrics: Callable[..., None],
time_agg: str,
expected: float,
) -> None:
metric_name = f"test_reduction_counter_{time_agg}"
base_epoch = aligned_epoch(timedelta(hours=30), step_seconds=300)
# monotonic cumulative counter: MetricsReducedTimeSeries mirrors the
# collector's temporality rewrite to Delta
time_series = [
MetricsReducedTimeSeries(
metric_name=metric_name,
kept_labels={"service": service},
timestamp=datetime.fromtimestamp(base_epoch, tz=UTC),
temporality="Cumulative",
type_="Sum",
is_monotonic=True,
)
for service in ("a", "b")
]
assert all(ts.temporality == "Delta" for ts in time_series)
insert_reduced_metrics(
time_series,
sum_samples=[
MetricsReducedSampleSum60s(
metric_name=metric_name,
reduced_fingerprint=ts.fingerprint,
timestamp=datetime.fromtimestamp(base_epoch + minute * 60, tz=UTC),
sum_value=30.0,
count_series=2,
count_samples=2,
temporality="Delta",
)
for ts in time_series
for minute in range(20)
],
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + 20 * 60, time_agg, "sum", step_interval=300)
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
assert [v["value"] for v in values] == [expected] * 4

View File

@@ -0,0 +1,70 @@
from collections.abc import Callable
from datetime import timedelta
from http import HTTPStatus
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.metricreduction import build_ruled_gauge_buffer
from fixtures.querier import (
aligned_epoch,
build_builder_query,
get_all_series,
index_series_by_label,
make_query_request,
query_metric_values,
)
SERVICES = ("a", "b")
PODS_PER_SERVICE = 2
MINUTES = 20
def test_recent_window_reads_buffer_totals(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_buffer_metrics: Callable[..., None],
) -> None:
metric_name = "test_reduction_buffer_totals"
# samples span [now-25m, now-5m); the query window sits inside the last 24h
base_epoch = aligned_epoch(timedelta(minutes=25), step_seconds=300)
insert_buffer_metrics(*build_ruled_gauge_buffer(metric_name, base_epoch, SERVICES, PODS_PER_SERVICE, MINUTES))
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
values = query_metric_values(signoz, token, metric_name, base_epoch, base_epoch + MINUTES * 60, "sum", "sum", step_interval=300)
# 4 raw series x 5 samples x 1.0 per step: full raw resolution, and the
# is_reduced=true series rows must not join in (their fingerprints match
# no samples, and the ts CTE filters them out)
assert [v["timestamp"] for v in values] == [(base_epoch + step * 300) * 1000 for step in range(4)]
assert [v["value"] for v in values] == [float(len(SERVICES) * PODS_PER_SERVICE * 5)] * 4
def test_recent_window_group_by_raw_label(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_buffer_metrics: Callable[..., None],
) -> None:
"""Group-by resolves against the raw buffer series rows (full labels), so
grouping by the kept label still sees every raw series underneath."""
metric_name = "test_reduction_buffer_groupby"
base_epoch = aligned_epoch(timedelta(minutes=25), step_seconds=300)
insert_buffer_metrics(*build_ruled_gauge_buffer(metric_name, base_epoch, SERVICES, PODS_PER_SERVICE, MINUTES))
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = make_query_request(
signoz,
token,
start_ms=base_epoch * 1000,
end_ms=(base_epoch + MINUTES * 60) * 1000,
queries=[build_builder_query("A", metric_name, "sum", "sum", step_interval=300, group_by=["service"])],
)
assert response.status_code == HTTPStatus.OK, response.text
series_by_service = index_series_by_label(get_all_series(response.json(), "A"), "service")
assert set(series_by_service.keys()) == set(SERVICES)
for service in SERVICES:
values = sorted(series_by_service[service]["values"], key=lambda v: v["timestamp"])
# 2 pods x 5 samples x 1.0 per step
assert [v["value"] for v in values] == [float(PODS_PER_SERVICE * 5)] * 4

View File

@@ -0,0 +1,114 @@
from collections.abc import Generator
from typing import Any
import pytest
from testcontainers.core.container import Network
from fixtures import types
from fixtures.auth import register_admin
from fixtures.clickhouse import create_clickhouse_cluster, create_clickhouse_keeper
from fixtures.http import ZEUS_NETWORK_ALIAS, create_zeus
from fixtures.migrator import create_migrator
from fixtures.signoz import create_signoz
SCHEMA_MIGRATOR_VERSION = "v0.144.6-rc.2"
CLICKHOUSE_VERSION = "25.12.5"
@pytest.fixture(name="keeper", scope="package")
def keeper_metricreduction(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerDocker:
return create_clickhouse_keeper(
tmpfs=tmpfs,
network=network,
request=request,
pytestconfig=pytestconfig,
cache_key="keeper_metricreduction",
version=CLICKHOUSE_VERSION,
)
@pytest.fixture(name="zeus", scope="package")
def zeus_metricreduction(
network: Network,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerDocker:
return create_zeus(
network=network,
request=request,
pytestconfig=pytestconfig,
cache_key="zeus_metricreduction",
alias=ZEUS_NETWORK_ALIAS,
)
@pytest.fixture(name="clickhouse", scope="package")
def clickhouse_metricreduction(
tmpfs: Generator[types.LegacyPath, Any],
network: Network,
keeper: types.TestContainerDocker,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.TestContainerClickhouse:
return create_clickhouse_cluster(
tmpfs=tmpfs,
network=network,
keeper=keeper,
request=request,
pytestconfig=pytestconfig,
cache_key="clickhouse_metricreduction",
shards=2,
version=CLICKHOUSE_VERSION,
)
@pytest.fixture(name="migrator", scope="package")
def migrator_metricreduction(
network: Network,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.Operation:
return create_migrator(
network=network,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
cache_key="migrator_metricreduction",
version=SCHEMA_MIGRATOR_VERSION,
)
@pytest.fixture(name="signoz", scope="package")
def signoz_metricreduction( # pylint: disable=too-many-arguments,too-many-positional-arguments
network: Network,
zeus: types.TestContainerDocker,
gateway: types.TestContainerDocker,
sqlstore: types.TestContainerSQL,
clickhouse: types.TestContainerClickhouse,
request: pytest.FixtureRequest,
pytestconfig: pytest.Config,
) -> types.SigNoz:
return create_signoz(
network=network,
zeus=zeus,
gateway=gateway,
sqlstore=sqlstore,
clickhouse=clickhouse,
request=request,
pytestconfig=pytestconfig,
cache_key="signoz_metricreduction",
env_overrides={
"SIGNOZ_FLAGGER_CONFIG_BOOLEAN_ENABLE__METRICS__REDUCTION": True,
},
)
@pytest.fixture(name="create_user_admin", scope="package")
def create_user_admin_metricreduction(signoz: types.SigNoz, request: pytest.FixtureRequest, pytestconfig: pytest.Config) -> types.Operation:
return register_admin(signoz, request, pytestconfig, cache_key="create_user_admin_metricreduction")