mirror of
https://github.com/SigNoz/signoz.git
synced 2026-05-14 06:00:30 +01:00
Compare commits
5 Commits
platform-t
...
tvats-limi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
67421ad2d1 | ||
|
|
56de61b463 | ||
|
|
6156b7fe6a | ||
|
|
25b5914f60 | ||
|
|
a90068786a |
@@ -152,6 +152,7 @@ telemetrystore:
|
||||
max_result_rows: 0
|
||||
ignore_data_skipping_indices: ""
|
||||
secondary_indices_enable_bulk_filtering: false
|
||||
max_query_size: 350000
|
||||
|
||||
##################### Prometheus #####################
|
||||
prometheus:
|
||||
|
||||
@@ -47,6 +47,7 @@ type QuerySettings struct {
|
||||
MaxResultRows int `mapstructure:"max_result_rows"`
|
||||
IgnoreDataSkippingIndices string `mapstructure:"ignore_data_skipping_indices"`
|
||||
SecondaryIndicesEnableBulkFiltering bool `mapstructure:"secondary_indices_enable_bulk_filtering"`
|
||||
MaxClickHouseQuerySize int `mapstructure:"max_query_size"`
|
||||
}
|
||||
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
|
||||
@@ -54,6 +54,10 @@ func (h *provider) BeforeQuery(ctx context.Context, _ *telemetrystore.QueryEvent
|
||||
settings["ignore_data_skipping_indices"] = h.settings.IgnoreDataSkippingIndices
|
||||
}
|
||||
|
||||
if h.settings.MaxClickHouseQuerySize != 0 {
|
||||
settings["max_query_size"] = h.settings.MaxClickHouseQuerySize
|
||||
}
|
||||
|
||||
if ctx.Value("clickhouse_max_threads") != nil {
|
||||
if maxThreads, ok := ctx.Value("clickhouse_max_threads").(int); ok {
|
||||
settings["max_threads"] = maxThreads
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package querybuildertypesv5
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/errors"
|
||||
|
||||
type ClickHouseQuery struct {
|
||||
// name of the query
|
||||
Name string `json:"name"`
|
||||
@@ -15,3 +17,21 @@ type ClickHouseQuery struct {
|
||||
func (q ClickHouseQuery) Copy() ClickHouseQuery {
|
||||
return q
|
||||
}
|
||||
|
||||
// Validate performs preliminary validation on ClickHouseQuery.
|
||||
func (q ClickHouseQuery) Validate() error {
|
||||
if q.Query == "" {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"ClickHouse SQL query is required",
|
||||
)
|
||||
}
|
||||
if len(q.Query) > MaxClickHouseQueryLength {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"ClickHouse SQL query exceeds maximum allowed length of %d characters",
|
||||
MaxClickHouseQueryLength,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package querybuildertypesv5
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/errors"
|
||||
|
||||
type PromQuery struct {
|
||||
// name of the query
|
||||
Name string `json:"name"`
|
||||
@@ -19,3 +21,21 @@ type PromQuery struct {
|
||||
func (q PromQuery) Copy() PromQuery {
|
||||
return q // shallow copy is sufficient
|
||||
}
|
||||
|
||||
// Validate performs preliminary validation on PromQuery.
|
||||
func (q PromQuery) Validate() error {
|
||||
if q.Query == "" {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"PromQL query is required",
|
||||
)
|
||||
}
|
||||
if len(q.Query) > MaxPromQLQueryLength {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"PromQL query exceeds maximum allowed length of %d characters",
|
||||
MaxPromQLQueryLength,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -10,6 +10,12 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxFilterExpressionLength = 300_000
|
||||
MaxClickHouseQueryLength = 300_000
|
||||
MaxPromQLQueryLength = 300_000
|
||||
)
|
||||
|
||||
// getQueryIdentifier returns a friendly identifier for a query based on its type and name/content.
|
||||
func getQueryIdentifier(envelope QueryEnvelope, index int) string {
|
||||
name := envelope.GetQueryName()
|
||||
@@ -155,6 +161,10 @@ func (q *QueryBuilderQuery[T]) Validate(opts ...ValidationOption) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.validateFilterExpression(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -365,6 +375,20 @@ func (q *QueryBuilderQuery[T]) validateFunctions() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateFilterExpression() error {
|
||||
if q.Filter == nil || q.Filter.Expression == "" {
|
||||
return nil
|
||||
}
|
||||
if len(q.Filter.Expression) > MaxFilterExpressionLength {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"filter expression exceeds maximum allowed length of %d characters",
|
||||
MaxFilterExpressionLength,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateSecondaryAggregations() error {
|
||||
for i, secAgg := range q.SecondaryAggregations {
|
||||
// Secondary aggregation expression can be empty - we allow it per requirements
|
||||
@@ -643,13 +667,7 @@ func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) err
|
||||
"invalid PromQL spec",
|
||||
)
|
||||
}
|
||||
if spec.Query == "" {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"PromQL query is required",
|
||||
)
|
||||
}
|
||||
return nil
|
||||
return spec.Validate()
|
||||
case QueryTypeClickHouseSQL:
|
||||
spec, ok := envelope.Spec.(ClickHouseQuery)
|
||||
if !ok {
|
||||
@@ -658,13 +676,7 @@ func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) err
|
||||
"invalid ClickHouse SQL spec",
|
||||
)
|
||||
}
|
||||
if spec.Query == "" {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"ClickHouse SQL query is required",
|
||||
)
|
||||
}
|
||||
return nil
|
||||
return spec.Validate()
|
||||
default:
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
|
||||
1
tests/fixtures/signoz.py
vendored
1
tests/fixtures/signoz.py
vendored
@@ -78,6 +78,7 @@ def create_signoz(
|
||||
"SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__WAIT": "1s",
|
||||
"SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__INTERVAL": "5s",
|
||||
"SIGNOZ_CLOUDINTEGRATION_AGENT_VERSION": "v0.0.8",
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_SETTINGS_MAX__QUERY__SIZE": "350000",
|
||||
}
|
||||
| sqlstore.env
|
||||
| clickhouse.env
|
||||
|
||||
124
tests/integration/tests/querier/15_clickhouse_query.py
Normal file
124
tests/integration/tests/querier/15_clickhouse_query.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
Integration tests for the v5 ClickHouse SQL query length cap and the
|
||||
raised ClickHouse `max_query_size` setting (350000) configured in
|
||||
pkg/telemetrystore/config.go.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import make_query_request
|
||||
|
||||
# Mirrors MaxClickHouseQueryLength in
|
||||
# pkg/types/querybuildertypes/querybuildertypesv5/validation.go
|
||||
MAX_CLICKHOUSE_QUERY_LENGTH = 300_000
|
||||
|
||||
# ClickHouse's built-in default `max_query_size`. SigNoz raises this to
|
||||
# 350000 via telemetrystore settings.
|
||||
CLICKHOUSE_BUILTIN_MAX_QUERY_SIZE = 262_144
|
||||
|
||||
|
||||
def test_clickhouse_query_at_validation_cap_is_accepted(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
prefix = "SELECT 1 /* "
|
||||
suffix = " */"
|
||||
query = prefix + ("x" * (MAX_CLICKHOUSE_QUERY_LENGTH - len(prefix) - len(suffix))) + suffix
|
||||
assert len(query) == MAX_CLICKHOUSE_QUERY_LENGTH
|
||||
|
||||
now = datetime.now(tz=UTC)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
queries=[
|
||||
{
|
||||
"type": "clickhouse_sql",
|
||||
"spec": {"name": "A", "query": query, "disabled": False},
|
||||
}
|
||||
],
|
||||
request_type="raw",
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
|
||||
|
||||
def test_clickhouse_query_above_validation_cap_is_rejected(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
prefix = "SELECT 1 /* "
|
||||
suffix = " */"
|
||||
target_len = MAX_CLICKHOUSE_QUERY_LENGTH + 1
|
||||
query = prefix + ("x" * (target_len - len(prefix) - len(suffix))) + suffix
|
||||
|
||||
assert len(query) > MAX_CLICKHOUSE_QUERY_LENGTH
|
||||
|
||||
now = datetime.now(tz=UTC)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
queries=[
|
||||
{
|
||||
"type": "clickhouse_sql",
|
||||
"spec": {"name": "A", "query": query, "disabled": False},
|
||||
}
|
||||
],
|
||||
request_type="raw",
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
assert "exceeds maximum allowed length" in response.text
|
||||
|
||||
|
||||
def test_clickhouse_query_above_builtin_max_query_size_parses(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
"""
|
||||
Verifies that the raised `max_query_size` setting is applied on every
|
||||
ClickHouse query: a payload above ClickHouse's built-in parser limit
|
||||
(262144 bytes) but below the v5 validation cap (300000) must succeed
|
||||
end-to-end. Without the setting override the ClickHouse parser would
|
||||
reject this and the server would return 500.
|
||||
"""
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
target_len = CLICKHOUSE_BUILTIN_MAX_QUERY_SIZE + 10_000
|
||||
prefix = "SELECT 1 /* "
|
||||
suffix = " */"
|
||||
query = prefix + ("x" * (target_len - len(prefix) - len(suffix))) + suffix
|
||||
|
||||
assert len(query) > CLICKHOUSE_BUILTIN_MAX_QUERY_SIZE
|
||||
assert len(query) < MAX_CLICKHOUSE_QUERY_LENGTH
|
||||
|
||||
now = datetime.now(tz=UTC)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
queries=[
|
||||
{
|
||||
"type": "clickhouse_sql",
|
||||
"spec": {"name": "A", "query": query, "disabled": False},
|
||||
}
|
||||
],
|
||||
request_type="raw",
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
Reference in New Issue
Block a user