mirror of
https://github.com/SigNoz/signoz.git
synced 2026-11-17 13:51:10 +00:00
Compare commits
33 Commits
main
...
tvats-tele
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cb7f8ab84d | ||
|
|
3721e28826 | ||
|
|
9f888978dc | ||
|
|
907e59f317 | ||
|
|
4f423ac50f | ||
|
|
e0bfc56556 | ||
|
|
8fe86ffe3f | ||
|
|
0b45c8d786 | ||
|
|
6265c0c1c8 | ||
|
|
cba0354826 | ||
|
|
400a66c83d | ||
|
|
be6bf4e0a5 | ||
|
|
85c6c1547e | ||
|
|
02591cce68 | ||
|
|
5874b393c3 | ||
|
|
de57bb88f5 | ||
|
|
2b6bce9530 | ||
|
|
d8cdb9a3bf | ||
|
|
243b3f2636 | ||
|
|
fec811acf1 | ||
|
|
1367e0ab40 | ||
|
|
9436d63ef2 | ||
|
|
e68f368c0f | ||
|
|
1c18a61845 | ||
|
|
985ed61a35 | ||
|
|
b7b1cbca87 | ||
|
|
68c6da8db8 | ||
|
|
9a8c15fa6d | ||
|
|
857185b41d | ||
|
|
7f6a2611d0 | ||
|
|
0cf34784b5 | ||
|
|
48cffc91a2 | ||
|
|
8bd3d6276d |
@@ -205,6 +205,11 @@ telemetrystore:
|
||||
max_bytes_to_read: 0
|
||||
max_result_rows: 0
|
||||
ignore_data_skipping_indices: ""
|
||||
# Stores queries can run with, for example clickhouse_query. Each runs as the ClickHouse user in its dsn, under
|
||||
# that user's settings profile and grants.
|
||||
profiles: {}
|
||||
# clickhouse_query:
|
||||
# dsn: tcp://signoz_query:password@localhost:9000
|
||||
|
||||
##################### Prometheus #####################
|
||||
prometheus:
|
||||
|
||||
@@ -10,6 +10,10 @@ import (
|
||||
|
||||
const DefaultMaxConcurrentQueries = 8
|
||||
|
||||
// ClickHouseSQLProfile is the telemetrystore.clickhouse.profiles entry that user-written ClickHouse SQL
|
||||
// queries run with, once it is in the config.
|
||||
const ClickHouseSQLProfile = "clickhouse_query"
|
||||
|
||||
// Config represents the configuration for the querier.
|
||||
type Config struct {
|
||||
// CacheTTL is the TTL for cached query results
|
||||
|
||||
@@ -233,7 +233,7 @@ func (q *querier) buildQueries(
|
||||
if !ok {
|
||||
return nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid clickhouse query spec %T", query.Spec)
|
||||
}
|
||||
chSQLQuery := newchSQLQuery(q.logger, q.telemetryStore, chQuery, nil, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars)
|
||||
chSQLQuery := newchSQLQuery(q.logger, q.telemetryStore.WithProfile(ClickHouseSQLProfile), chQuery, nil, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType, tmplVars)
|
||||
queries[chQuery.Name] = chSQLQuery
|
||||
case qbtypes.QueryTypeTraceOperator:
|
||||
traceOpQuery, ok := query.Spec.(qbtypes.QueryBuilderTraceOperator)
|
||||
@@ -925,7 +925,7 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
|
||||
queryCopy := qt.query.Copy()
|
||||
argsCopy := make([]any, len(qt.args))
|
||||
copy(argsCopy, qt.args)
|
||||
return newchSQLQuery(q.logger, q.telemetryStore, queryCopy, argsCopy, timeRange, qt.kind, qt.vars)
|
||||
return newchSQLQuery(q.logger, q.telemetryStore.WithProfile(ClickHouseSQLProfile), queryCopy, argsCopy, timeRange, qt.kind, qt.vars)
|
||||
|
||||
// A builder query's window was moved into the shifted clock when it was
|
||||
// built, so a range taken from that window must not be shifted again.
|
||||
|
||||
@@ -26,6 +26,12 @@ var (
|
||||
ErrCodeNotAnAggregate = errors.MustNewCode("not_an_aggregate")
|
||||
ErrCodeIllegalAggregation = errors.MustNewCode("illegal_aggregation")
|
||||
ErrCodeInvalidRegexp = errors.MustNewCode("invalid_regexp")
|
||||
ErrCodeSettingNotAllowed = errors.MustNewCode("setting_not_allowed")
|
||||
ErrCodeSettingsProfileNotFound = errors.MustNewCode("settings_profile_not_found")
|
||||
ErrCodeResultTooLarge = errors.MustNewCode("result_too_large")
|
||||
ErrCodeReadTooLarge = errors.MustNewCode("read_too_large")
|
||||
ErrCodeReadonlyMode = errors.MustNewCode("readonly_mode")
|
||||
ErrCodeAccessDenied = errors.MustNewCode("access_denied")
|
||||
|
||||
ErrCodeQueryCanceled = errors.MustNewCode("query_canceled")
|
||||
ErrCodeQueryTimeout = errors.MustNewCode("query_timeout")
|
||||
@@ -111,11 +117,47 @@ var clickHouseExceptionWrappers = map[chproto.Error]func(cause error, ex *clickh
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeInvalidRegexp, "regular expression does not compile: %s", ex.Message).
|
||||
WithSuggestions("write the pattern in RE2 syntax, which has no lookaround or backreferences, and escape a backslash that is meant literally")
|
||||
},
|
||||
chproto.ErrTooManyRowsOrBytes: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeResultTooLarge, "query result too large: %s", ex.Message).
|
||||
WithSuggestions("add a LIMIT, narrow the time range or select fewer columns")
|
||||
},
|
||||
chproto.ErrTooManyRows: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeReadTooLarge, "query read too much data: %s", ex.Message).
|
||||
WithSuggestions("narrow the time range or add filters so the query reads less data")
|
||||
},
|
||||
chproto.ErrTooManyPartitions: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeReadTooLarge, "query read too much data: %s", ex.Message).
|
||||
WithSuggestions("narrow the time range or add filters so the query reads less data")
|
||||
},
|
||||
chproto.ErrTooManyBytes: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeReadTooLarge, "query read too much data: %s", ex.Message).
|
||||
WithSuggestions("narrow the time range or add filters so the query reads less data")
|
||||
},
|
||||
// A write, or a change to the readonly setting, under a read-only settings profile.
|
||||
chproto.ErrReadonly: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeReadonlyMode, "query not allowed in read-only mode: %s", ex.Message).
|
||||
WithSuggestions("run only SELECT queries and remove readonly from the SETTINGS clause")
|
||||
},
|
||||
// A query on a table outside the grants of the ClickHouse user it runs as.
|
||||
chproto.ErrAccessDenied: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapForbiddenf(cause, ErrCodeAccessDenied, "query not allowed: %s", ex.Message).
|
||||
WithSuggestions("query only the telemetry tables")
|
||||
},
|
||||
chproto.ErrThereIsNoProfile: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapNotFoundf(cause, ErrCodeSettingsProfileNotFound, "settings profile not found: %s", ex.Message)
|
||||
},
|
||||
// A SETTINGS clause went past a constraint of the query's settings profile.
|
||||
chproto.ErrSettingConstraintViolation: func(cause error, ex *clickhouse.Exception) error {
|
||||
return errors.WrapInvalidInputf(cause, ErrCodeSettingNotAllowed, "setting not allowed: %s", ex.Message).
|
||||
WithSuggestions("remove the setting from the SETTINGS clause of the query")
|
||||
},
|
||||
}
|
||||
|
||||
type provider struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
clickHouseConn clickhouse.Conn
|
||||
profile string
|
||||
profiles map[string]*provider
|
||||
cluster string
|
||||
hooks []telemetrystore.TelemetryStoreHook
|
||||
}
|
||||
@@ -129,21 +171,17 @@ func NewFactory(hookFactories ...factory.ProviderFactory[telemetrystore.Telemetr
|
||||
func New(ctx context.Context, providerSettings factory.ProviderSettings, config telemetrystore.Config, hookFactories ...factory.ProviderFactory[telemetrystore.TelemetryStoreHook, telemetrystore.Config]) (telemetrystore.TelemetryStore, error) {
|
||||
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/telemetrystore/clickhousetelemetrystore")
|
||||
|
||||
options, err := clickhouse.ParseDSN(config.Clickhouse.DSN)
|
||||
chConn, err := open(config.Clickhouse.DSN, config.Connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options.MaxIdleConns = config.Connection.MaxIdleConns
|
||||
options.MaxOpenConns = config.Connection.MaxOpenConns
|
||||
options.DialTimeout = config.Connection.DialTimeout
|
||||
// Read JSON columns natively: the driver mis-decodes JSON(max_dynamic_paths=0) columns without
|
||||
// the flattened serialization, and reading them as a string collapses a key stored at both a
|
||||
// scalar and an object path.
|
||||
options.Settings["output_format_native_use_flattened_dynamic_and_json_serialization"] = 1
|
||||
|
||||
chConn, err := clickhouse.Open(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
profileConns := make(map[string]clickhouse.Conn, len(config.Clickhouse.Profiles))
|
||||
for profile, profileConfig := range config.Clickhouse.Profiles {
|
||||
profileConns[profile], err = open(profileConfig.DSN, config.Connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
hooks := make([]telemetrystore.TelemetryStoreHook, len(hookFactories))
|
||||
@@ -178,49 +216,96 @@ func New(ctx context.Context, providerSettings factory.ProviderSettings, config
|
||||
cluster: config.Clickhouse.Cluster,
|
||||
hooks: hooks,
|
||||
}
|
||||
|
||||
withProfiles(p, profileConns)
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func (p *provider) ClickhouseDB() clickhouse.Conn {
|
||||
return p
|
||||
func open(dsn string, connection telemetrystore.ConnectionConfig) (clickhouse.Conn, error) {
|
||||
options, err := clickhouse.ParseDSN(dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options.MaxIdleConns = connection.MaxIdleConns
|
||||
options.MaxOpenConns = connection.MaxOpenConns
|
||||
options.DialTimeout = connection.DialTimeout
|
||||
// Read JSON columns natively: the driver mis-decodes JSON(max_dynamic_paths=0) columns without
|
||||
// the flattened serialization, and reading them as a string collapses a key stored at both a
|
||||
// scalar and an object path.
|
||||
options.Settings["output_format_native_use_flattened_dynamic_and_json_serialization"] = 1
|
||||
|
||||
return clickhouse.Open(options)
|
||||
}
|
||||
|
||||
func (p *provider) Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) {
|
||||
return RunExplainEstimate(ctx, p, stmt, args...)
|
||||
// withProfiles gives every profile a copy of main that runs its queries over the profile's pool.
|
||||
func withProfiles(main *provider, conns map[string]clickhouse.Conn) {
|
||||
main.profiles = make(map[string]*provider, len(conns))
|
||||
for profile, conn := range conns {
|
||||
profiled := *main
|
||||
profiled.profile = profile
|
||||
profiled.clickHouseConn = conn
|
||||
main.profiles[profile] = &profiled
|
||||
}
|
||||
}
|
||||
|
||||
func (p *provider) Plan(ctx context.Context, stmt string, args ...any) error {
|
||||
return RunExplainPlan(ctx, p, stmt, args...)
|
||||
func (provider *provider) WithProfile(profile string) telemetrystore.TelemetryStore {
|
||||
if store, ok := provider.profiles[profile]; ok {
|
||||
return store
|
||||
}
|
||||
return provider
|
||||
}
|
||||
|
||||
func (p *provider) Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) {
|
||||
return RunExplainIndexes(ctx, p, stmt, args...)
|
||||
func (provider *provider) ClickhouseDB() clickhouse.Conn {
|
||||
return provider
|
||||
}
|
||||
|
||||
func (p *provider) Cluster() string {
|
||||
return p.cluster
|
||||
func (provider *provider) Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) {
|
||||
return RunExplainEstimate(ctx, provider, stmt, args...)
|
||||
}
|
||||
|
||||
func (p *provider) Close() error {
|
||||
return p.clickHouseConn.Close()
|
||||
func (provider *provider) Plan(ctx context.Context, stmt string, args ...any) error {
|
||||
return RunExplainPlan(ctx, provider, stmt, args...)
|
||||
}
|
||||
|
||||
func (p *provider) Ping(ctx context.Context) error {
|
||||
return p.clickHouseConn.Ping(ctx)
|
||||
func (provider *provider) Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) {
|
||||
return RunExplainIndexes(ctx, provider, stmt, args...)
|
||||
}
|
||||
|
||||
func (p *provider) Stats() driver.Stats {
|
||||
return p.clickHouseConn.Stats()
|
||||
func (provider *provider) Cluster() string {
|
||||
return provider.cluster
|
||||
}
|
||||
|
||||
func (p *provider) Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error) {
|
||||
// Close closes the profile pools with the main one. A profile store leaves its pool to the main store.
|
||||
func (provider *provider) Close() error {
|
||||
if provider.profile != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
errs := make([]error, 0, len(provider.profiles)+1)
|
||||
for _, profiled := range provider.profiles {
|
||||
errs = append(errs, profiled.clickHouseConn.Close())
|
||||
}
|
||||
errs = append(errs, provider.clickHouseConn.Close())
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (provider *provider) Ping(ctx context.Context) error {
|
||||
return provider.clickHouseConn.Ping(ctx)
|
||||
}
|
||||
|
||||
func (provider *provider) Stats() driver.Stats {
|
||||
return provider.clickHouseConn.Stats()
|
||||
}
|
||||
|
||||
func (provider *provider) Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error) {
|
||||
event := telemetrystore.NewQueryEvent(query, args)
|
||||
|
||||
ctx = telemetrystore.WrapBeforeQuery(p.hooks, ctx, event)
|
||||
rows, err := p.clickHouseConn.Query(ctx, query, args...)
|
||||
ctx = telemetrystore.WrapBeforeQuery(provider.hooks, ctx, event)
|
||||
rows, err := provider.clickHouseConn.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
event.Err = err
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
return nil, castError(err)
|
||||
}
|
||||
|
||||
@@ -228,71 +313,71 @@ func (p *provider) Query(ctx context.Context, query string, args ...interface{})
|
||||
Rows: telemetrystore.WrapRows(rows),
|
||||
ctx: ctx,
|
||||
event: event,
|
||||
onClose: func() { telemetrystore.WrapAfterQuery(p.hooks, ctx, event) },
|
||||
onClose: func() { telemetrystore.WrapAfterQuery(provider.hooks, ctx, event) },
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *provider) QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row {
|
||||
func (provider *provider) QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row {
|
||||
event := telemetrystore.NewQueryEvent(query, args)
|
||||
|
||||
ctx = telemetrystore.WrapBeforeQuery(p.hooks, ctx, event)
|
||||
row := p.clickHouseConn.QueryRow(ctx, query, args...)
|
||||
ctx = telemetrystore.WrapBeforeQuery(provider.hooks, ctx, event)
|
||||
row := provider.clickHouseConn.QueryRow(ctx, query, args...)
|
||||
|
||||
if row == nil {
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
return nil
|
||||
}
|
||||
|
||||
event.Err = row.Err()
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
|
||||
return &rowWithCastError{Row: row}
|
||||
}
|
||||
|
||||
func (p *provider) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
func (provider *provider) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
|
||||
event := telemetrystore.NewQueryEvent(query, args)
|
||||
|
||||
ctx = telemetrystore.WrapBeforeQuery(p.hooks, ctx, event)
|
||||
err := p.clickHouseConn.Select(ctx, dest, query, args...)
|
||||
ctx = telemetrystore.WrapBeforeQuery(provider.hooks, ctx, event)
|
||||
err := provider.clickHouseConn.Select(ctx, dest, query, args...)
|
||||
|
||||
event.Err = err
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
|
||||
return castError(err)
|
||||
}
|
||||
|
||||
func (p *provider) Exec(ctx context.Context, query string, args ...interface{}) error {
|
||||
func (provider *provider) Exec(ctx context.Context, query string, args ...interface{}) error {
|
||||
event := telemetrystore.NewQueryEvent(query, args)
|
||||
|
||||
ctx = telemetrystore.WrapBeforeQuery(p.hooks, ctx, event)
|
||||
err := p.clickHouseConn.Exec(ctx, query, args...)
|
||||
ctx = telemetrystore.WrapBeforeQuery(provider.hooks, ctx, event)
|
||||
err := provider.clickHouseConn.Exec(ctx, query, args...)
|
||||
|
||||
event.Err = err
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
|
||||
return castError(err)
|
||||
}
|
||||
|
||||
func (p *provider) AsyncInsert(ctx context.Context, query string, wait bool, args ...interface{}) error {
|
||||
func (provider *provider) AsyncInsert(ctx context.Context, query string, wait bool, args ...interface{}) error {
|
||||
event := telemetrystore.NewQueryEvent(query, args)
|
||||
|
||||
ctx = telemetrystore.WrapBeforeQuery(p.hooks, ctx, event)
|
||||
err := p.clickHouseConn.Exec(clickhouse.Context(ctx, clickhouse.WithAsync(wait)), query, args...)
|
||||
ctx = telemetrystore.WrapBeforeQuery(provider.hooks, ctx, event)
|
||||
err := provider.clickHouseConn.Exec(clickhouse.Context(ctx, clickhouse.WithAsync(wait)), query, args...)
|
||||
|
||||
event.Err = err
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
|
||||
return castError(err)
|
||||
}
|
||||
|
||||
func (p *provider) PrepareBatch(ctx context.Context, query string, opts ...driver.PrepareBatchOption) (driver.Batch, error) {
|
||||
func (provider *provider) PrepareBatch(ctx context.Context, query string, opts ...driver.PrepareBatchOption) (driver.Batch, error) {
|
||||
event := telemetrystore.NewQueryEvent(query, nil)
|
||||
|
||||
ctx = telemetrystore.WrapBeforeQuery(p.hooks, ctx, event)
|
||||
batch, err := p.clickHouseConn.PrepareBatch(ctx, query, opts...)
|
||||
ctx = telemetrystore.WrapBeforeQuery(provider.hooks, ctx, event)
|
||||
batch, err := provider.clickHouseConn.PrepareBatch(ctx, query, opts...)
|
||||
|
||||
event.Err = err
|
||||
telemetrystore.WrapAfterQuery(p.hooks, ctx, event)
|
||||
telemetrystore.WrapAfterQuery(provider.hooks, ctx, event)
|
||||
|
||||
if batch == nil {
|
||||
return nil, castError(err)
|
||||
@@ -300,12 +385,12 @@ func (p *provider) PrepareBatch(ctx context.Context, query string, opts ...drive
|
||||
return &batchWithCastError{Batch: batch}, castError(err)
|
||||
}
|
||||
|
||||
func (p *provider) ServerVersion() (*driver.ServerVersion, error) {
|
||||
return p.clickHouseConn.ServerVersion()
|
||||
func (provider *provider) ServerVersion() (*driver.ServerVersion, error) {
|
||||
return provider.clickHouseConn.ServerVersion()
|
||||
}
|
||||
|
||||
func (p *provider) Contributors() []string {
|
||||
return p.clickHouseConn.Contributors()
|
||||
func (provider *provider) Contributors() []string {
|
||||
return provider.clickHouseConn.Contributors()
|
||||
}
|
||||
|
||||
func castError(err error) error {
|
||||
|
||||
74
pkg/telemetrystore/clickhousetelemetrystore/provider_test.go
Normal file
74
pkg/telemetrystore/clickhousetelemetrystore/provider_test.go
Normal file
@@ -0,0 +1,74 @@
|
||||
package clickhousetelemetrystore
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
chproto "github.com/ClickHouse/ch-go/proto"
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
cmock "github.com/SigNoz/clickhouse-go-mock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
)
|
||||
|
||||
func TestCastError_ProfileErrors(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
code chproto.Error
|
||||
want errors.Code
|
||||
notFound bool
|
||||
forbidden bool
|
||||
}{
|
||||
{name: "SettingConstraintViolation_SettingNotAllowed", code: chproto.ErrSettingConstraintViolation, want: ErrCodeSettingNotAllowed},
|
||||
{name: "TooManyRowsOrBytes_ResultTooLarge", code: chproto.ErrTooManyRowsOrBytes, want: ErrCodeResultTooLarge},
|
||||
{name: "TooManyRows_ReadTooLarge", code: chproto.ErrTooManyRows, want: ErrCodeReadTooLarge},
|
||||
{name: "TooManyBytes_ReadTooLarge", code: chproto.ErrTooManyBytes, want: ErrCodeReadTooLarge},
|
||||
{name: "TooManyPartitions_ReadTooLarge", code: chproto.ErrTooManyPartitions, want: ErrCodeReadTooLarge},
|
||||
{name: "Readonly_ReadonlyMode", code: chproto.ErrReadonly, want: ErrCodeReadonlyMode},
|
||||
{name: "AccessDenied_AccessDenied", code: chproto.ErrAccessDenied, want: ErrCodeAccessDenied, forbidden: true},
|
||||
{name: "ThereIsNoProfile_SettingsProfileNotFound", code: chproto.ErrThereIsNoProfile, want: ErrCodeSettingsProfileNotFound, notFound: true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
err := castError(&clickhouse.Exception{Code: int32(testCase.code), Message: testCase.name})
|
||||
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Asc(err, testCase.want), "got %v", err)
|
||||
assert.Equal(t, testCase.notFound, errors.Ast(err, errors.TypeNotFound), "got %v", err)
|
||||
assert.Equal(t, testCase.forbidden, errors.Ast(err, errors.TypeForbidden), "got %v", err)
|
||||
assert.Equal(t, !testCase.notFound && !testCase.forbidden, errors.Ast(err, errors.TypeInvalidInput), "got %v", err)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderWithProfile(t *testing.T) {
|
||||
conn, err := cmock.NewClickHouseWithQueryMatcher(&clickhouse.Options{}, sqlmock.QueryMatcherEqual)
|
||||
require.NoError(t, err)
|
||||
conn.ExpectClose()
|
||||
profileConn, err := cmock.NewClickHouseWithQueryMatcher(&clickhouse.Options{}, sqlmock.QueryMatcherEqual)
|
||||
require.NoError(t, err)
|
||||
profileConn.ExpectClose()
|
||||
|
||||
p := &provider{clickHouseConn: conn, cluster: "cluster"}
|
||||
withProfiles(p, map[string]clickhouse.Conn{"clickhouse_query": profileConn})
|
||||
|
||||
profiled, ok := p.WithProfile("clickhouse_query").(*provider)
|
||||
require.True(t, ok)
|
||||
assert.NotSame(t, p, profiled)
|
||||
assert.Equal(t, "clickhouse_query", profiled.profile)
|
||||
assert.Same(t, profileConn, profiled.clickHouseConn)
|
||||
assert.Same(t, conn, p.clickHouseConn)
|
||||
assert.Equal(t, "cluster", profiled.Cluster())
|
||||
assert.Empty(t, p.profile)
|
||||
|
||||
assert.Same(t, p, p.WithProfile("query_range"))
|
||||
assert.Same(t, profiled, profiled.WithProfile("clickhouse_query"))
|
||||
|
||||
require.NoError(t, profiled.Close())
|
||||
require.NoError(t, p.Close())
|
||||
assert.NoError(t, conn.ExpectationsWereMet())
|
||||
assert.NoError(t, profileConn.ExpectationsWereMet())
|
||||
}
|
||||
@@ -37,6 +37,15 @@ type ClickhouseConfig struct {
|
||||
|
||||
// QuerySettings is the query settings for clickhouse.
|
||||
QuerySettings QuerySettings `mapstructure:"settings"`
|
||||
|
||||
// Profiles are the stores TelemetryStore.WithProfile returns, by name. Each runs its queries as the
|
||||
// ClickHouse user in its DSN, under that user's settings profile and grants.
|
||||
Profiles map[string]ProfileConfig `mapstructure:"profiles"`
|
||||
}
|
||||
|
||||
type ProfileConfig struct {
|
||||
// DSN is the database source name of the ClickHouse user the profile's queries run as.
|
||||
DSN string `mapstructure:"dsn"`
|
||||
}
|
||||
|
||||
type QuerySettings struct {
|
||||
|
||||
@@ -11,6 +11,10 @@ type TelemetryStore interface {
|
||||
// ClickhouseDB returns the clickhouse connection, which can also EXPLAIN.
|
||||
ClickhouseDB() clickhouse.Conn
|
||||
|
||||
// WithProfile returns a store that runs every query as the profile's ClickHouse user, or this store when
|
||||
// the profile is not in the config.
|
||||
WithProfile(profile string) TelemetryStore
|
||||
|
||||
// Cluster returns the cluster name.
|
||||
Cluster() string
|
||||
|
||||
|
||||
@@ -31,31 +31,36 @@ func New(_ telemetrystore.Config, matcher sqlmock.QueryMatcher) *Provider {
|
||||
}
|
||||
|
||||
// ClickhouseDB returns the mock Clickhouse connection.
|
||||
func (p *Provider) ClickhouseDB() clickhouse.Conn {
|
||||
return p.clickhouseDB.(clickhouse.Conn)
|
||||
func (provider *Provider) ClickhouseDB() clickhouse.Conn {
|
||||
return provider.clickhouseDB.(clickhouse.Conn)
|
||||
}
|
||||
|
||||
// WithProfile returns the same mock, so expectations set on it apply to profile queries.
|
||||
func (provider *Provider) WithProfile(_ string) telemetrystore.TelemetryStore {
|
||||
return provider
|
||||
}
|
||||
|
||||
// Cluster returns the cluster name.
|
||||
func (p *Provider) Cluster() string {
|
||||
func (provider *Provider) Cluster() string {
|
||||
return "cluster"
|
||||
}
|
||||
|
||||
// Estimate runs EXPLAIN ESTIMATE against the mock connection.
|
||||
func (p *Provider) Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) {
|
||||
return clickhousetelemetrystore.RunExplainEstimate(ctx, p.clickhouseDB.(clickhouse.Conn), stmt, args...)
|
||||
func (provider *Provider) Estimate(ctx context.Context, stmt string, args ...any) ([]telemetrystoretypes.EstimateEntry, error) {
|
||||
return clickhousetelemetrystore.RunExplainEstimate(ctx, provider.clickhouseDB.(clickhouse.Conn), stmt, args...)
|
||||
}
|
||||
|
||||
// Plan runs EXPLAIN PLAN against the mock connection.
|
||||
func (p *Provider) Plan(ctx context.Context, stmt string, args ...any) error {
|
||||
return clickhousetelemetrystore.RunExplainPlan(ctx, p.clickhouseDB.(clickhouse.Conn), stmt, args...)
|
||||
func (provider *Provider) Plan(ctx context.Context, stmt string, args ...any) error {
|
||||
return clickhousetelemetrystore.RunExplainPlan(ctx, provider.clickhouseDB.(clickhouse.Conn), stmt, args...)
|
||||
}
|
||||
|
||||
// Indexes runs EXPLAIN indexes against the mock connection.
|
||||
func (p *Provider) Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) {
|
||||
return clickhousetelemetrystore.RunExplainIndexes(ctx, p.clickhouseDB.(clickhouse.Conn), stmt, args...)
|
||||
func (provider *Provider) Indexes(ctx context.Context, stmt string, args ...any) (telemetrystoretypes.Granules, bool, error) {
|
||||
return clickhousetelemetrystore.RunExplainIndexes(ctx, provider.clickhouseDB.(clickhouse.Conn), stmt, args...)
|
||||
}
|
||||
|
||||
// Mock returns the underlying Clickhouse mock instance for setting expectations.
|
||||
func (p *Provider) Mock() cmock.ClickConnMockCommon {
|
||||
return p.clickhouseDB
|
||||
func (provider *Provider) Mock() cmock.ClickConnMockCommon {
|
||||
return provider.clickhouseDB
|
||||
}
|
||||
|
||||
113
tests/fixtures/clickhouse.py
vendored
113
tests/fixtures/clickhouse.py
vendored
@@ -1,4 +1,7 @@
|
||||
import io
|
||||
import os
|
||||
import tarfile
|
||||
import time
|
||||
from collections.abc import Callable, Generator
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
@@ -595,3 +598,113 @@ def check_query_log(
|
||||
assert all(check_fn(q) for q in queries), f"query_log check failed for case '{case_name}'.\n" + "Queries:\n" + "\n---\n".join(queries)
|
||||
|
||||
return _check
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse_settings_profile", scope="package")
|
||||
def clickhouse_settings_profile(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[Callable[[str, str], None], Any]:
|
||||
"""
|
||||
Yields create(name, settings) that defines a settings profile from the XML body of its
|
||||
<profiles> entry. Profiles are users.d files, since the integration user has no access
|
||||
management; ClickHouse reloads them without a restart. They are removed on teardown.
|
||||
"""
|
||||
container = docker.from_env().containers.get(clickhouse.container.id)
|
||||
names: list[str] = []
|
||||
|
||||
def wait_for_profile(name: str, exists: bool) -> None:
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
count = clickhouse.conn.query(
|
||||
"SELECT count() FROM system.settings_profiles WHERE name = {name:String}",
|
||||
parameters={"name": name},
|
||||
).result_rows[0][0]
|
||||
if (count == 1) == exists:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError(f"settings profile {name} did not {'appear' if exists else 'disappear'}")
|
||||
|
||||
def create(name: str, settings: str) -> None:
|
||||
body = f"<clickhouse><profiles><{name}>{settings}</{name}></profiles></clickhouse>".encode()
|
||||
archive = io.BytesIO()
|
||||
with tarfile.open(fileobj=archive, mode="w") as tar:
|
||||
info = tarfile.TarInfo(f"{name}.xml")
|
||||
info.size = len(body)
|
||||
info.mode = 0o644
|
||||
tar.addfile(info, io.BytesIO(body))
|
||||
container.put_archive("/etc/clickhouse-server/users.d", archive.getvalue())
|
||||
names.append(name)
|
||||
wait_for_profile(name, exists=True)
|
||||
|
||||
yield create
|
||||
|
||||
for name in names:
|
||||
container.exec_run(["rm", "-f", f"/etc/clickhouse-server/users.d/{name}.xml"])
|
||||
for name in names:
|
||||
wait_for_profile(name, exists=False)
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse_user", scope="package")
|
||||
def clickhouse_user(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[Callable[[str, str, list[str]], str], Any]:
|
||||
"""
|
||||
Yields create(name, profile, grants) that defines a user whose default settings profile is
|
||||
profile, and returns its DSN on the docker network. Users are users.d files, like settings
|
||||
profiles. They are removed on teardown.
|
||||
"""
|
||||
container = docker.from_env().containers.get(clickhouse.container.id)
|
||||
password = "password"
|
||||
names: list[str] = []
|
||||
|
||||
def wait_for_user(name: str, exists: bool) -> None:
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
count = clickhouse.conn.query(
|
||||
"SELECT count() FROM system.users WHERE name = {name:String}",
|
||||
parameters={"name": name},
|
||||
).result_rows[0][0]
|
||||
if (count == 1) == exists:
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise TimeoutError(f"user {name} did not {'appear' if exists else 'disappear'}")
|
||||
|
||||
def create(name: str, profile: str, grants: list[str]) -> str:
|
||||
queries = "".join(f"<query>{grant}</query>" for grant in grants)
|
||||
body = (f"<clickhouse><users><{name}><password>{password}</password><profile>{profile}</profile><networks><ip>::/0</ip></networks><grants>{queries}</grants></{name}></users></clickhouse>").encode()
|
||||
archive = io.BytesIO()
|
||||
with tarfile.open(fileobj=archive, mode="w") as tar:
|
||||
info = tarfile.TarInfo(f"user-{name}.xml")
|
||||
info.size = len(body)
|
||||
info.mode = 0o644
|
||||
tar.addfile(info, io.BytesIO(body))
|
||||
container.put_archive("/etc/clickhouse-server/users.d", archive.getvalue())
|
||||
names.append(name)
|
||||
wait_for_user(name, exists=True)
|
||||
return f"tcp://{name}:{password}@{clickhouse.container.container_configs['9000'].address}:9000"
|
||||
|
||||
yield create
|
||||
|
||||
for name in names:
|
||||
container.exec_run(["rm", "-f", f"/etc/clickhouse-server/users.d/user-{name}.xml"])
|
||||
for name in names:
|
||||
wait_for_user(name, exists=False)
|
||||
|
||||
|
||||
@pytest.fixture(name="clickhouse_partitioned_table", scope="package")
|
||||
def clickhouse_partitioned_table(
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[str, Any]:
|
||||
"""
|
||||
Yields a MergeTree table with one partition per day for 5 days, where the first day holds
|
||||
300000 rows, for queries that hit limits on partitions or rows read.
|
||||
"""
|
||||
table = "default.integration_partitioned"
|
||||
clickhouse.conn.command(f"CREATE TABLE IF NOT EXISTS {table} (day Date, value UInt8) ENGINE = MergeTree PARTITION BY day ORDER BY value")
|
||||
clickhouse.conn.command(f"TRUNCATE TABLE {table}")
|
||||
clickhouse.conn.command(f"INSERT INTO {table} SELECT toDate('2026-01-01'), 1 FROM numbers(300000)")
|
||||
clickhouse.conn.command(f"INSERT INTO {table} SELECT toDate('2026-01-02') + number, 1 FROM numbers(4)")
|
||||
|
||||
yield table
|
||||
|
||||
clickhouse.conn.command(f"DROP TABLE IF EXISTS {table}")
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
|
||||
|
||||
def test_clickhouse_sql_runs_with_profile(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": "SELECT toFloat64(currentUser() = 'signoz_query') AS user, toFloat64(getSetting('readonly')) AS readonly, toFloat64(getSetting('max_result_rows')) AS max_result_rows", "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["data"]["data"]["results"][0]["data"][0] == [1, 2, 100]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"settings",
|
||||
[
|
||||
"max_result_rows = 1000",
|
||||
"max_result_rows = 0",
|
||||
"max_result_bytes = 1000000",
|
||||
"max_bytes_to_read = 10000000",
|
||||
"max_rows_to_read = 10000000",
|
||||
"max_partitions_to_read = 100",
|
||||
"max_partitions_to_read = -1",
|
||||
"profile = 'default', max_result_rows = 1000",
|
||||
"max_result_rows = 1000, profile = 'default'",
|
||||
],
|
||||
ids=[
|
||||
"result_rows_above_max",
|
||||
"result_rows_zero_is_unlimited",
|
||||
"result_bytes_above_max",
|
||||
"bytes_to_read_above_max",
|
||||
"rows_to_read_above_max",
|
||||
"partitions_to_read_above_max",
|
||||
"partitions_to_read_unlimited",
|
||||
"switch_profile_then_raise",
|
||||
"raise_then_switch_profile",
|
||||
],
|
||||
)
|
||||
def test_clickhouse_sql_settings_cannot_escape_profile(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
settings: str,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": f"SELECT toFloat64(1) AS value SETTINGS {settings}", "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
error = response.json()["error"]
|
||||
assert (error["code"], error["suggestions"]) == ("setting_not_allowed", ["remove the setting from the SETTINGS clause of the query"]), error
|
||||
|
||||
|
||||
def test_clickhouse_sql_switching_profile_keeps_limits(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": "SELECT toFloat64(getSetting('readonly')) AS readonly, toFloat64(getSetting('max_result_rows')) AS max_result_rows SETTINGS profile = 'default'", "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["data"]["data"]["results"][0]["data"][0] == [2, 100]
|
||||
|
||||
|
||||
RESULT_TOO_LARGE = "add a LIMIT, narrow the time range or select fewer columns"
|
||||
READ_TOO_LARGE = "narrow the time range or add filters so the query reads less data"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query,code,suggestion,reason",
|
||||
[
|
||||
("SELECT toFloat64(number) AS value FROM numbers(1000)", "result_too_large", RESULT_TOO_LARGE, "max rows"),
|
||||
("SELECT repeat(toString(number), 2000) AS text, toFloat64(number) AS value FROM numbers(90)", "result_too_large", RESULT_TOO_LARGE, "max bytes"),
|
||||
("SELECT toFloat64(count()) AS value FROM numbers(150000)", "read_too_large", READ_TOO_LARGE, "max bytes"),
|
||||
("SELECT toFloat64(sum(value)) AS value FROM {table} WHERE day = '2026-01-01'", "read_too_large", READ_TOO_LARGE, "max rows"),
|
||||
("SELECT toFloat64(sum(value)) AS value FROM {table} WHERE day > '2026-01-01'", "read_too_large", READ_TOO_LARGE, "partitions"),
|
||||
],
|
||||
ids=["result_rows", "result_bytes", "bytes_read", "rows_read", "partitions_read"],
|
||||
)
|
||||
def test_clickhouse_sql_over_profile_limit( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
clickhouse_partitioned_table: str,
|
||||
query: str,
|
||||
code: str,
|
||||
suggestion: str,
|
||||
reason: str,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": query.format(table=clickhouse_partitioned_table), "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
error = response.json()["error"]
|
||||
assert (error["code"], error["suggestions"]) == (code, [suggestion]), error
|
||||
assert reason in error["message"], error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,value",
|
||||
[("readonly", 2), ("max_result_rows", 50), ("max_threads", 1)],
|
||||
ids=["readonly_same_as_profile", "result_rows_below_max", "unconstrained_setting"],
|
||||
)
|
||||
def test_clickhouse_sql_settings_within_profile(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
name: str,
|
||||
value: int,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": f"SELECT toFloat64(getSetting('{name}')) AS value SETTINGS {name} = {value}", "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["data"]["data"]["results"][0]["data"][0][0] == value
|
||||
|
||||
|
||||
def test_clickhouse_sql_lower_limit_in_settings_is_enforced(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": "SELECT toFloat64(number) AS value FROM numbers(10) SETTINGS max_result_rows = 5", "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
error = response.json()["error"]
|
||||
assert (error["code"], error["suggestions"]) == ("result_too_large", [RESULT_TOO_LARGE]), error
|
||||
assert "max rows: 5" in error["message"], error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
"SELECT toFloat64(1) AS value SETTINGS readonly = 1",
|
||||
"SELECT toFloat64(1) AS value SETTINGS readonly = 0",
|
||||
],
|
||||
ids=["readonly_stricter", "readonly_lifted"],
|
||||
)
|
||||
def test_clickhouse_sql_readonly_profile_rejects(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
query: str,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": query, "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
error = response.json()["error"]
|
||||
assert (error["code"], error["suggestions"]) == ("readonly_mode", ["run only SELECT queries and remove readonly from the SETTINGS clause"]), error
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query",
|
||||
[
|
||||
"SELECT toFloat64(count()) AS value FROM system.query_log",
|
||||
"CREATE TABLE default.integration_readonly_probe (x UInt8) ENGINE = Memory",
|
||||
],
|
||||
ids=["read_outside_grants", "write"],
|
||||
)
|
||||
def test_clickhouse_sql_outside_grants_is_denied(
|
||||
signoz_clickhouse_query_profile: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
query: str,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
|
||||
response = querier.make_query_request(
|
||||
signoz_clickhouse_query_profile,
|
||||
get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD),
|
||||
int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
int(now.timestamp() * 1000),
|
||||
[{"type": "clickhouse_sql", "spec": {"name": "A", "query": query, "disabled": False}}],
|
||||
request_type=querier.RequestType.SCALAR,
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN, response.text
|
||||
error = response.json()["error"]
|
||||
assert (error["code"], error["suggestions"]) == ("access_denied", ["query only the telemetry tables"]), error
|
||||
56
tests/integration/tests/queriercommon/conftest.py
Normal file
56
tests/integration/tests/queriercommon/conftest.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
from testcontainers.core.container import Network
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.signoz import create_signoz
|
||||
|
||||
|
||||
@pytest.fixture(name="signoz_clickhouse_query_profile", scope="package")
|
||||
def signoz_clickhouse_query_profile(
|
||||
network: Network,
|
||||
migrator: types.Operation, # pylint: disable=unused-argument
|
||||
zeus: types.TestContainerDocker,
|
||||
gateway: types.TestContainerDocker,
|
||||
sqlstore: types.TestContainerSQL,
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
clickhouse_settings_profile: Callable[[str, str], None],
|
||||
clickhouse_user: Callable[[str, str, list[str]], str],
|
||||
request: pytest.FixtureRequest,
|
||||
pytestconfig: pytest.Config,
|
||||
) -> types.SigNoz:
|
||||
"""
|
||||
SigNoz whose clickhouse_sql queries run as signoz_query, a user with the clickhouse_query
|
||||
profile that can only read the default database.
|
||||
"""
|
||||
clickhouse_settings_profile(
|
||||
"clickhouse_query",
|
||||
"<readonly>2</readonly>"
|
||||
"<max_result_rows>100</max_result_rows>"
|
||||
"<max_result_bytes>100000</max_result_bytes>"
|
||||
"<max_bytes_to_read>1000000</max_bytes_to_read>"
|
||||
"<max_rows_to_read>200000</max_rows_to_read>"
|
||||
"<max_partitions_to_read>2</max_partitions_to_read>"
|
||||
"<constraints>"
|
||||
"<max_result_rows><min>1</min><max>100</max></max_result_rows>"
|
||||
"<max_result_bytes><min>1</min><max>100000</max></max_result_bytes>"
|
||||
"<max_bytes_to_read><min>1</min><max>1000000</max></max_bytes_to_read>"
|
||||
"<max_rows_to_read><min>1</min><max>200000</max></max_rows_to_read>"
|
||||
"<max_partitions_to_read><min>1</min><max>2</max></max_partitions_to_read>"
|
||||
"</constraints>",
|
||||
)
|
||||
dsn = clickhouse_user("signoz_query", "clickhouse_query", ["GRANT SELECT ON default.*"])
|
||||
return create_signoz(
|
||||
network=network,
|
||||
zeus=zeus,
|
||||
gateway=gateway,
|
||||
sqlstore=sqlstore,
|
||||
clickhouse=clickhouse,
|
||||
request=request,
|
||||
pytestconfig=pytestconfig,
|
||||
cache_key="signoz-clickhouse-query-profile",
|
||||
env_overrides={
|
||||
"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_PROFILES_CLICKHOUSE__QUERY_DSN": dsn,
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user