mirror of
https://github.com/SigNoz/signoz.git
synced 2026-10-30 21:11:11 +00:00
Compare commits
3 Commits
fix/scatte
...
tvats-quer
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
281c0b1d60 | ||
|
|
7fa73e9f5f | ||
|
|
dcf21a438f |
@@ -122,7 +122,7 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
return signoz.NewMeterReporterProviderFactories(), "noop"
|
||||
},
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
return querier.NewHandler(ps, q, a)
|
||||
return querier.NewHandler(ps, q, a, querier.NewBudget(config.Querier))
|
||||
},
|
||||
func(_ sqlstore.SQLStore, _ dashboard.Module, _ global.Global, _ zeus.Zeus, _ gateway.Gateway, _ licensing.Licensing, _ serviceaccount.Module, _ cloudintegration.Config) (cloudintegration.Module, error) {
|
||||
return implcloudintegration.NewModule(), nil
|
||||
|
||||
@@ -173,8 +173,9 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
return factories, "http"
|
||||
},
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
communityHandler := querier.NewHandler(ps, q, a)
|
||||
return eequerier.NewHandler(ps, q, communityHandler)
|
||||
queryBudget := querier.NewBudget(config.Querier)
|
||||
communityHandler := querier.NewHandler(ps, q, a, queryBudget)
|
||||
return eequerier.NewHandler(ps, q, communityHandler, queryBudget)
|
||||
},
|
||||
func(sqlStore sqlstore.SQLStore, dashboardModule dashboard.Module, global global.Global, zeus zeus.Zeus, gateway gateway.Gateway, licensing licensing.Licensing, serviceAccount serviceaccount.Module, config cloudintegration.Config) (cloudintegration.Module, error) {
|
||||
defStore := pkgcloudintegration.NewServiceDefinitionStore()
|
||||
|
||||
@@ -178,6 +178,8 @@ querier:
|
||||
# range with padding to include slightly delayed log exports. Logs only; set
|
||||
# to 0 to disable.
|
||||
log_trace_id_window_padding: 5m
|
||||
# The maximum time range of a query range request. Set to 0 to disable.
|
||||
max_time_range: 17520h
|
||||
|
||||
##################### TelemetryStore #####################
|
||||
telemetrystore:
|
||||
|
||||
@@ -21,13 +21,15 @@ type handler struct {
|
||||
set factory.ProviderSettings
|
||||
querier querier.Querier
|
||||
community querier.Handler
|
||||
budget querier.Budget
|
||||
}
|
||||
|
||||
func NewHandler(set factory.ProviderSettings, querier querier.Querier, community querier.Handler) querier.Handler {
|
||||
func NewHandler(set factory.ProviderSettings, querier querier.Querier, community querier.Handler, budget querier.Budget) querier.Handler {
|
||||
return &handler{
|
||||
set: set,
|
||||
querier: querier,
|
||||
community: community,
|
||||
budget: budget,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,6 +60,11 @@ func (h *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.budget.AdmitTimeRange(queryRangeRequest.Start, queryRangeRequest.End); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
orgID, err := valuer.NewUUID(claims.OrgID)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/http/binding"
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/exporttypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -25,10 +26,11 @@ import (
|
||||
|
||||
type handler struct {
|
||||
module rawdataexport.Module
|
||||
budget querier.Budget
|
||||
}
|
||||
|
||||
func NewHandler(module rawdataexport.Module) rawdataexport.Handler {
|
||||
return &handler{module: module}
|
||||
func NewHandler(module rawdataexport.Module, budget querier.Budget) rawdataexport.Handler {
|
||||
return &handler{module: module, budget: budget}
|
||||
}
|
||||
|
||||
func (handler *handler) ExportRawData(rw http.ResponseWriter, r *http.Request) {
|
||||
@@ -50,6 +52,11 @@ func (handler *handler) ExportRawData(rw http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if err := handler.budget.AdmitTimeRange(queryRangeRequest.Start, queryRangeRequest.End); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := validateAndApplyDefaultExportLimits(queryRangeRequest.CompositeQuery.Queries); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
|
||||
@@ -33,10 +33,11 @@ type handler struct {
|
||||
set factory.ProviderSettings
|
||||
analytics analytics.Analytics
|
||||
querier Querier
|
||||
budget Budget
|
||||
}
|
||||
|
||||
func NewHandler(set factory.ProviderSettings, querier Querier, analytics analytics.Analytics) Handler {
|
||||
return &handler{set: set, querier: querier, analytics: analytics}
|
||||
func NewHandler(set factory.ProviderSettings, querier Querier, analytics analytics.Analytics, budget Budget) Handler {
|
||||
return &handler{set: set, querier: querier, analytics: analytics, budget: budget}
|
||||
}
|
||||
|
||||
func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
|
||||
@@ -58,12 +59,16 @@ func (handler *handler) QueryRange(rw http.ResponseWriter, req *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the query request
|
||||
if err := queryRangeRequest.Validate(); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := handler.budget.AdmitTimeRange(queryRangeRequest.Start, queryRangeRequest.End); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
orgID, err := valuer.NewUUID(claims.OrgID)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
@@ -140,7 +145,9 @@ func (handler *handler) QueryRawStream(rw http.ResponseWriter, req *http.Request
|
||||
}
|
||||
// create the v5 request param
|
||||
queryRangeRequest := qbtypes.QueryRangeRequest{
|
||||
Start: start,
|
||||
// Coerce the window to epoch milliseconds up front (End may be 0 for the
|
||||
// open-ended stream, which ToMilliSecs leaves untouched).
|
||||
Start: qbtypes.ToMilliSecs(start),
|
||||
RequestType: qbtypes.RequestTypeRawStream,
|
||||
CompositeQuery: qbtypes.CompositeQuery{
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
|
||||
32
pkg/querier/budget.go
Normal file
32
pkg/querier/budget.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
)
|
||||
|
||||
// Budget is the admission policy applied to client-supplied query windows at the
|
||||
// API edge. It is deployment capacity, not well-formedness: a window it rejects
|
||||
// is still a valid QueryRangeRequest, which is why it is configurable and why
|
||||
// in-process callers — whose windows are derived, not client-supplied — never
|
||||
// consult it.
|
||||
type Budget struct {
|
||||
maxTimeRange time.Duration
|
||||
}
|
||||
|
||||
func NewBudget(config Config) Budget {
|
||||
return Budget{maxTimeRange: config.MaxTimeRange}
|
||||
}
|
||||
|
||||
// AdmitTimeRange rejects a window wider than the configured maximum. Both bounds
|
||||
// are epoch milliseconds; a zero maximum disables the limit.
|
||||
func (budget Budget) AdmitTimeRange(startMS, endMS uint64) error {
|
||||
if budget.maxTimeRange <= 0 || endMS <= startMS {
|
||||
return nil
|
||||
}
|
||||
if time.Duration(endMS-startMS)*time.Millisecond > budget.maxTimeRange {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "query time range must not exceed %d days", int64(budget.maxTimeRange/(24*time.Hour)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
|
||||
const DefaultMaxConcurrentQueries = 8
|
||||
|
||||
const DefaultMaxTimeRange = 2 * 365 * 24 * time.Hour
|
||||
|
||||
// Config represents the configuration for the querier.
|
||||
type Config struct {
|
||||
// CacheTTL is the TTL for cached query results
|
||||
@@ -20,6 +22,8 @@ type Config struct {
|
||||
MaxConcurrentQueries int `yaml:"max_concurrent_queries" mapstructure:"max_concurrent_queries"`
|
||||
// LogTraceIDWindowPadding is the padding added to narrowed down timerange from trace summary to logs with trace_id filter.
|
||||
LogTraceIDWindowPadding time.Duration `yaml:"log_trace_id_window_padding" mapstructure:"log_trace_id_window_padding"`
|
||||
// MaxTimeRange is the maximum allowed span between start and end of a query range request; 0 disables the limit.
|
||||
MaxTimeRange time.Duration `yaml:"max_time_range" mapstructure:"max_time_range"`
|
||||
|
||||
// Keys sit under querier.skip_resource_fingerprint.
|
||||
statementbuilder.Config `mapstructure:",squash" yaml:",squash"`
|
||||
@@ -37,6 +41,7 @@ func newConfig() factory.Config {
|
||||
FluxInterval: 5 * time.Minute,
|
||||
MaxConcurrentQueries: DefaultMaxConcurrentQueries,
|
||||
LogTraceIDWindowPadding: 5 * time.Minute,
|
||||
MaxTimeRange: DefaultMaxTimeRange,
|
||||
Config: statementbuilder.NewConfig(),
|
||||
}
|
||||
}
|
||||
@@ -55,6 +60,9 @@ func (c Config) Validate() error {
|
||||
if c.LogTraceIDWindowPadding < 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "log_trace_id_window_padding must not be negative, got %v", c.LogTraceIDWindowPadding)
|
||||
}
|
||||
if c.MaxTimeRange < 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "max_time_range must not be negative, got %v", c.MaxTimeRange)
|
||||
}
|
||||
// Embedded Validate is shadowed by this one; call it explicitly.
|
||||
if err := c.Config.Validate(); err != nil {
|
||||
return err
|
||||
|
||||
@@ -11,6 +11,12 @@ func makeBuckets(start, end uint64) []tsRange {
|
||||
startNS := querybuilder.ToNanoSecs(start)
|
||||
endNS := querybuilder.ToNanoSecs(end)
|
||||
|
||||
// A cursor that reaches the window boundary can leave end <= start; the
|
||||
// subtractions below are unsigned and would wrap.
|
||||
if endNS <= startNS {
|
||||
return nil
|
||||
}
|
||||
|
||||
if endNS-startNS <= uint64(hourNanos) {
|
||||
return []tsRange{{fromNS: startNS, toNS: endNS}}
|
||||
}
|
||||
@@ -20,9 +26,9 @@ func makeBuckets(start, end uint64) []tsRange {
|
||||
curEnd := endNS
|
||||
|
||||
for {
|
||||
curStart := curEnd - bucket
|
||||
if curStart < startNS {
|
||||
curStart = startNS
|
||||
curStart := startNS
|
||||
if curEnd-startNS > bucket {
|
||||
curStart = curEnd - bucket
|
||||
}
|
||||
out = append(out, tsRange{fromNS: curStart, toNS: curEnd})
|
||||
|
||||
|
||||
@@ -111,11 +111,10 @@ func New(
|
||||
|
||||
func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest) (*qbtypes.QueryRangeResponse, error) {
|
||||
|
||||
// Coerce the window to epoch milliseconds up front so every downstream
|
||||
// consumer (TimeRange, narrowWindowByTraceID, step interval, etc.) can
|
||||
// safely assume ms regardless of the resolution the caller sent.
|
||||
req.Start = querybuilder.ToMilliSecs(req.Start)
|
||||
req.End = querybuilder.ToMilliSecs(req.End)
|
||||
// Callers that build the request in-process never pass through
|
||||
// UnmarshalJSON, so normalise here too.
|
||||
req.Start = qbtypes.ToMilliSecs(req.Start)
|
||||
req.End = qbtypes.ToMilliSecs(req.End)
|
||||
|
||||
event := &qbtypes.QBEvent{
|
||||
Version: "v5",
|
||||
@@ -534,10 +533,8 @@ func familyTemporality(temporalities map[string]metrictypes.Temporality, names [
|
||||
|
||||
func (q *querier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qbtypes.QueryRangeRequest, client *qbtypes.RawStream) {
|
||||
|
||||
// Coerce the window to epoch milliseconds up front (End may be 0 for the
|
||||
// open-ended stream, which ToMilliSecs leaves untouched).
|
||||
req.Start = querybuilder.ToMilliSecs(req.Start)
|
||||
req.End = querybuilder.ToMilliSecs(req.End)
|
||||
req.Start = qbtypes.ToMilliSecs(req.Start)
|
||||
req.End = qbtypes.ToMilliSecs(req.End)
|
||||
|
||||
event := &qbtypes.QBEvent{
|
||||
Version: "v5",
|
||||
|
||||
@@ -33,28 +33,6 @@ func ToNanoSecs(epoch uint64) uint64 {
|
||||
return temp * uint64(math.Pow(10, float64(19-count)))
|
||||
}
|
||||
|
||||
// ToMilliSecs takes an epoch whose resolution is inferred from its magnitude
|
||||
// (s/ms/µs/ns) and returns it in milliseconds. A millisecond epoch for the
|
||||
// current era has 13 digits (e.g. ~1.7e12 in 2026), so the value is scaled so
|
||||
// its digit-width matches: smaller values (seconds) are scaled up, larger ones
|
||||
// (micro/nanoseconds) are scaled down. Zero is returned unchanged.
|
||||
func ToMilliSecs(epoch uint64) uint64 {
|
||||
if epoch == 0 {
|
||||
return 0
|
||||
}
|
||||
temp := epoch
|
||||
count := 0
|
||||
for epoch != 0 {
|
||||
epoch /= 10
|
||||
count++
|
||||
}
|
||||
const msDigits = 13
|
||||
if count < msDigits {
|
||||
return temp * uint64(math.Pow(10, float64(msDigits-count)))
|
||||
}
|
||||
return temp / uint64(math.Pow(10, float64(count-msDigits)))
|
||||
}
|
||||
|
||||
// TODO(srikanthccv): should these be rounded to nearest multiple of 60 instead of 5 if step > 60?
|
||||
// That would make graph look nice but "nice" but should be less important than the usefulness.
|
||||
func RecommendedStepInterval(start, end uint64) uint64 {
|
||||
|
||||
@@ -60,51 +60,3 @@ func TestToNanoSecs(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestToMilliSecs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
epoch uint64
|
||||
expected uint64
|
||||
}{
|
||||
{
|
||||
name: "10-digit Unix timestamp (seconds) - 2023-01-01 00:00:00 UTC",
|
||||
epoch: 1672531200, // seconds
|
||||
expected: 1672531200000, // * 10^3
|
||||
},
|
||||
{
|
||||
name: "13-digit Unix timestamp (milliseconds) - already ms",
|
||||
epoch: 1672531200000,
|
||||
expected: 1672531200000, // unchanged
|
||||
},
|
||||
{
|
||||
name: "16-digit Unix timestamp (microseconds)",
|
||||
epoch: 1672531200000000, // microseconds
|
||||
expected: 1672531200000, // / 10^3
|
||||
},
|
||||
{
|
||||
name: "19-digit Unix timestamp (nanoseconds)",
|
||||
epoch: 1672531200000000000, // nanoseconds
|
||||
expected: 1672531200000, // / 10^6
|
||||
},
|
||||
{
|
||||
name: "Unix epoch start - zero is unchanged",
|
||||
epoch: 0,
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "Recent timestamp in seconds - 2024-05-25 12:00:00 UTC",
|
||||
epoch: 1716638400,
|
||||
expected: 1716638400000,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := ToMilliSecs(tt.epoch)
|
||||
if result != tt.expected {
|
||||
t.Errorf("ToMilliSecs(%d) = %d, want %d", tt.epoch, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +97,7 @@ type Handlers struct {
|
||||
|
||||
func NewHandlers(
|
||||
modules Modules,
|
||||
config Config,
|
||||
providerSettings factory.ProviderSettings,
|
||||
analytics analytics.Analytics,
|
||||
querierHandler querier.Handler,
|
||||
@@ -120,7 +121,7 @@ func NewHandlers(
|
||||
Dashboard: impldashboard.NewHandler(modules.Dashboard, providerSettings, authz),
|
||||
QuickFilter: implquickfilter.NewHandler(modules.QuickFilter),
|
||||
TraceFunnel: impltracefunnel.NewHandler(modules.TraceFunnel),
|
||||
RawDataExport: implrawdataexport.NewHandler(modules.RawDataExport),
|
||||
RawDataExport: implrawdataexport.NewHandler(modules.RawDataExport, querier.NewBudget(config.Querier)),
|
||||
Services: implservices.NewHandler(modules.Services),
|
||||
MetricsExplorer: implmetricsexplorer.NewHandler(modules.MetricsExplorer),
|
||||
MetricReductionRule: implmetricreductionrule.NewHandler(modules.MetricReductionRule),
|
||||
|
||||
@@ -67,9 +67,9 @@ func TestNewHandlers(t *testing.T) {
|
||||
spanMapperModule := implspanmapper.NewModule(implspanmapper.NewStore(sqlstore), spanMapperRegistry, providerSettings)
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, nil, nil, nil, retentionGetter, flagger, tagModule, nil, spanMapperModule)
|
||||
|
||||
querierHandler := querier.NewHandler(providerSettings, nil, nil)
|
||||
querierHandler := querier.NewHandler(providerSettings, nil, nil, querier.Budget{})
|
||||
registryHandler := factory.NewHandler(nil)
|
||||
handlers := NewHandlers(modules, providerSettings, nil, querierHandler, nil, nil, nil, nil, nil, nil, nil, nil, registryHandler, alertmanager, nil, nil, nil)
|
||||
handlers := NewHandlers(modules, Config{}, providerSettings, nil, querierHandler, nil, nil, nil, nil, nil, nil, nil, nil, registryHandler, alertmanager, nil, nil, nil)
|
||||
reflectVal := reflect.ValueOf(handlers)
|
||||
for i := 0; i < reflectVal.NumField(); i++ {
|
||||
f := reflectVal.Field(i)
|
||||
|
||||
@@ -615,7 +615,7 @@ func New(
|
||||
registryHandler := factory.NewHandler(registry)
|
||||
subscriptionService := subscriptionCallback(zeus, licensing)
|
||||
|
||||
handlers := NewHandlers(modules, providerSettings, analytics, querierHandler, licensing, global, flagger, gateway, telemetryMetadataStore, authz, zeus, subscriptionService, registryHandler, alertmanager, prometheus, rulerInstance, statsAggregator)
|
||||
handlers := NewHandlers(modules, config, providerSettings, analytics, querierHandler, licensing, global, flagger, gateway, telemetryMetadataStore, authz, zeus, subscriptionService, registryHandler, alertmanager, prometheus, rulerInstance, statsAggregator)
|
||||
|
||||
// Initialize the API server (after registry so it can access service health)
|
||||
apiserverInstance, err := factory.NewProviderFromNamedMap(
|
||||
|
||||
@@ -673,6 +673,12 @@ func (r *QueryRangeRequest) UnmarshalJSON(data []byte) error {
|
||||
// Copy the decoded values back to the original struct
|
||||
*r = QueryRangeRequest(temp)
|
||||
|
||||
// Coerce the window to epoch milliseconds up front so every downstream
|
||||
// consumer (TimeRange, narrowWindowByTraceID, step interval, etc.) can
|
||||
// safely assume ms regardless of the resolution the caller sent.
|
||||
r.Start = ToMilliSecs(r.Start)
|
||||
r.End = ToMilliSecs(r.End)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
25
pkg/types/querybuildertypes/querybuildertypesv5/time.go
Normal file
25
pkg/types/querybuildertypes/querybuildertypesv5/time.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package querybuildertypesv5
|
||||
|
||||
import "math"
|
||||
|
||||
// ToMilliSecs takes an epoch whose resolution is inferred from its magnitude
|
||||
// (s/ms/µs/ns) and returns it in milliseconds. A millisecond epoch for the
|
||||
// current era has 13 digits (e.g. ~1.7e12 in 2026), so the value is scaled so
|
||||
// its digit-width matches: smaller values (seconds) are scaled up, larger ones
|
||||
// (micro/nanoseconds) are scaled down. Zero is returned unchanged.
|
||||
func ToMilliSecs(epoch uint64) uint64 {
|
||||
if epoch == 0 {
|
||||
return 0
|
||||
}
|
||||
temp := epoch
|
||||
count := 0
|
||||
for epoch != 0 {
|
||||
epoch /= 10
|
||||
count++
|
||||
}
|
||||
const msDigits = 13
|
||||
if count < msDigits {
|
||||
return temp * uint64(math.Pow(10, float64(msDigits-count)))
|
||||
}
|
||||
return temp / uint64(math.Pow(10, float64(count-msDigits)))
|
||||
}
|
||||
52
pkg/types/querybuildertypes/querybuildertypesv5/time_test.go
Normal file
52
pkg/types/querybuildertypes/querybuildertypesv5/time_test.go
Normal file
@@ -0,0 +1,52 @@
|
||||
package querybuildertypesv5
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestToMilliSecs(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
epoch uint64
|
||||
expected uint64
|
||||
}{
|
||||
{
|
||||
name: "SecondsEpoch_TenDigits_ScaledUp",
|
||||
epoch: 1672531200, // seconds, 2023-01-01 00:00:00 UTC
|
||||
expected: 1672531200000, // * 10^3
|
||||
},
|
||||
{
|
||||
name: "MilliSecondsEpoch_ThirteenDigits_Unchanged",
|
||||
epoch: 1672531200000,
|
||||
expected: 1672531200000, // unchanged
|
||||
},
|
||||
{
|
||||
name: "MicroSecondsEpoch_SixteenDigits_ScaledDown",
|
||||
epoch: 1672531200000000, // microseconds
|
||||
expected: 1672531200000, // / 10^3
|
||||
},
|
||||
{
|
||||
name: "NanoSecondsEpoch_NineteenDigits_ScaledDown",
|
||||
epoch: 1672531200000000000, // nanoseconds
|
||||
expected: 1672531200000, // / 10^6
|
||||
},
|
||||
{
|
||||
name: "Zero_Unchanged",
|
||||
epoch: 0,
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "SecondsEpoch_RecentTimestamp_ScaledUp",
|
||||
epoch: 1716638400, // 2024-05-25 12:00:00 UTC
|
||||
expected: 1716638400000,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testCase.expected, ToMilliSecs(testCase.epoch))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -5,12 +5,16 @@ import (
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// maxEndTimeAhead tolerates client clock skew; anything further ahead is a malformed timestamp.
|
||||
const maxEndTimeAhead = 24 * time.Hour
|
||||
|
||||
// getQueryIdentifier returns a friendly identifier for a query based on its type and name/content.
|
||||
func getQueryIdentifier(envelope QueryEnvelope, index int) string {
|
||||
name := envelope.GetQueryName()
|
||||
@@ -574,12 +578,8 @@ func (q *QueryBuilderQuery[T]) validateOrderByForAggregation() error {
|
||||
|
||||
// Validate validates the entire query range request.
|
||||
func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
|
||||
// Validate time range
|
||||
if r.RequestType != RequestTypeRawStream && r.Start >= r.End {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"start time must be before end time",
|
||||
)
|
||||
if err := r.validateTimeRange(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate request type
|
||||
@@ -631,8 +631,8 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
|
||||
// this so per-query errors can be attributed individually via QueryEnvelope.Validate
|
||||
// instead of failing fast like Validate does.
|
||||
func (r *QueryRangeRequest) ValidateRequestScope() ([]ValidationOption, error) {
|
||||
if r.RequestType != RequestTypeRawStream && r.Start >= r.End {
|
||||
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "start time must be before end time")
|
||||
if err := r.validateTimeRange(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var opts []ValidationOption
|
||||
@@ -699,6 +699,22 @@ func (e QueryEnvelope) Validate(opts ...ValidationOption) error {
|
||||
return validateQueryEnvelope(e, opts...)
|
||||
}
|
||||
|
||||
func (r *QueryRangeRequest) validateTimeRange() error {
|
||||
if r.RequestType == RequestTypeRawStream {
|
||||
return nil
|
||||
}
|
||||
if r.Start == 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "start time is required")
|
||||
}
|
||||
if r.Start >= r.End {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "start time must be before end time")
|
||||
}
|
||||
if r.End > uint64(time.Now().Add(maxEndTimeAhead).UnixMilli()) {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "end time %s is too far in the future", time.UnixMilli(int64(r.End)).UTC().Format(time.RFC3339))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateAllQueriesNotDisabled validates that at least one query in the composite query is enabled.
|
||||
func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
|
||||
for _, envelope := range r.CompositeQuery.Queries {
|
||||
|
||||
166
tests/integration/tests/queriercommon/09_time_range.py
Normal file
166
tests/integration/tests/queriercommon/09_time_range.py
Normal file
@@ -0,0 +1,166 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.metrics import Metrics
|
||||
from fixtures.querier import (
|
||||
Aggregation,
|
||||
BuilderQuery,
|
||||
MetricAggregation,
|
||||
get_all_warnings,
|
||||
get_error_message,
|
||||
get_series_values,
|
||||
make_query_request,
|
||||
)
|
||||
from fixtures.traces import TraceIdGenerator, Traces
|
||||
|
||||
TWO_YEARS = timedelta(days=730)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"start_ms,end_ms,message",
|
||||
[
|
||||
pytest.param(0, 1_700_000_000_000, "start time is required", id="zero_start"),
|
||||
pytest.param(1_700_000_000_000, 1_700_000_000_000, "start time must be before end time", id="start_equals_end"),
|
||||
# 2592000000 is 30 days in ms; read as an epoch in seconds it is year 2052.
|
||||
pytest.param(1_700_000_000_000, 2_592_000_000, "end time 2052-02-20T00:00:00Z is too far in the future", id="end_in_2052"),
|
||||
pytest.param(1_700_000_000_000 - int((TWO_YEARS + timedelta(days=1)).total_seconds() * 1000), 1_700_000_000_000, "query time range must not exceed 730 days", id="range_over_two_years"),
|
||||
],
|
||||
)
|
||||
def test_time_range_rejected(
|
||||
start_ms: int,
|
||||
end_ms: int,
|
||||
message: str,
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=start_ms,
|
||||
end_ms=end_ms,
|
||||
queries=[BuilderQuery(signal="logs", aggregations=[Aggregation(expression="count()")], step_interval=60).to_dict()],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert get_error_message(response.json()) == message
|
||||
|
||||
|
||||
def test_two_year_logs_timeseries_step_clamped(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
# A 1s step over 2 years is clamped to 42045s (at most ~1500 points); end is aligned to it so the bucket count is stable.
|
||||
step_seconds = 42045
|
||||
end = datetime.fromtimestamp(int(datetime.now(tz=UTC).timestamp()) // step_seconds * step_seconds, tz=UTC)
|
||||
insert_logs([Logs(timestamp=end - timedelta(minutes=1), resources={"service.name": "time-range-logs"}, body="two year span")])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((end - TWO_YEARS).timestamp() * 1000),
|
||||
end_ms=int(end.timestamp() * 1000),
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="logs",
|
||||
filter_expression="service.name = 'time-range-logs'",
|
||||
aggregations=[Aggregation(expression="count()")],
|
||||
step_interval=1,
|
||||
).to_dict()
|
||||
],
|
||||
format_options={"formatTableResultForUI": False, "fillGaps": True},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["meta"]["stepIntervals"] == {"A": step_seconds}
|
||||
assert get_all_warnings(response.json()) == [
|
||||
{"message": f"Query A is requesting aggregation interval 1 seconds, which is smaller than the minimum allowed interval of {step_seconds} seconds for selected time range. Using the minimum instead"},
|
||||
]
|
||||
values = get_series_values(response.json(), "A")
|
||||
assert len(values) == 1502
|
||||
assert sum(v["value"] for v in values) == 1
|
||||
|
||||
|
||||
def test_two_year_traces_timeseries_step_clamped(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
# Same clamp as logs: 42045s; end is aligned to it so the bucket count is stable.
|
||||
step_seconds = 42045
|
||||
end = datetime.fromtimestamp(int(datetime.now(tz=UTC).timestamp()) // step_seconds * step_seconds, tz=UTC)
|
||||
insert_traces([Traces(timestamp=end - timedelta(minutes=1), trace_id=TraceIdGenerator.trace_id(), span_id=TraceIdGenerator.span_id(), resources={"service.name": "time-range-traces"})])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((end - TWO_YEARS).timestamp() * 1000),
|
||||
end_ms=int(end.timestamp() * 1000),
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="traces",
|
||||
filter_expression="service.name = 'time-range-traces'",
|
||||
aggregations=[Aggregation(expression="count()")],
|
||||
step_interval=1,
|
||||
).to_dict()
|
||||
],
|
||||
format_options={"formatTableResultForUI": False, "fillGaps": True},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["meta"]["stepIntervals"] == {"A": step_seconds}
|
||||
assert get_all_warnings(response.json()) == [
|
||||
{"message": f"Query A is requesting aggregation interval 1 seconds, which is smaller than the minimum allowed interval of {step_seconds} seconds for selected time range. Using the minimum instead"},
|
||||
]
|
||||
values = get_series_values(response.json(), "A")
|
||||
assert len(values) == 1502
|
||||
assert sum(v["value"] for v in values) == 1
|
||||
|
||||
|
||||
def test_two_year_metrics_timeseries_step_clamped(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
# A 1s step over 2 years is clamped to 210600s (~300 points); end is aligned to it so the bucket count is stable.
|
||||
step_seconds = 210600
|
||||
metric_name = "time_range_two_year_gauge"
|
||||
end = datetime.fromtimestamp(int(datetime.now(tz=UTC).timestamp()) // step_seconds * step_seconds, tz=UTC)
|
||||
insert_metrics([Metrics(metric_name=metric_name, labels={"service": "time-range-metrics"}, timestamp=end - timedelta(minutes=1), value=1.0, temporality="Unspecified", type_="Gauge", is_monotonic=False)])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((end - TWO_YEARS).timestamp() * 1000),
|
||||
end_ms=int(end.timestamp() * 1000),
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="metrics",
|
||||
aggregations=[MetricAggregation(metric_name=metric_name, time_aggregation="latest", space_aggregation="sum", temporality="unspecified")],
|
||||
step_interval=1,
|
||||
).to_dict()
|
||||
],
|
||||
format_options={"formatTableResultForUI": False, "fillGaps": True},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["meta"]["stepIntervals"] == {"A": step_seconds}
|
||||
assert get_all_warnings(response.json()) == [
|
||||
{"message": f"Query A is requesting aggregation interval 1 seconds, which is smaller than the minimum allowed interval of {step_seconds} seconds for selected time range. Using the minimum instead"},
|
||||
]
|
||||
assert len(get_series_values(response.json(), "A")) == 301
|
||||
Reference in New Issue
Block a user