mirror of
https://github.com/SigNoz/signoz.git
synced 2026-03-05 21:32:00 +00:00
Compare commits
25 Commits
remove-v4-
...
like-op
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
96cebb095b | ||
|
|
a4d24d5055 | ||
|
|
36c5db177f | ||
|
|
2519f061d0 | ||
|
|
ce24ac217a | ||
|
|
215b16f7ef | ||
|
|
695ab0ae14 | ||
|
|
27802d8256 | ||
|
|
e411ecf92b | ||
|
|
029cd38e21 | ||
|
|
421f3bdd9e | ||
|
|
53a0b532bb | ||
|
|
03a13678d0 | ||
|
|
14d4751c96 | ||
|
|
b50d8136a5 | ||
|
|
4103fe4d2f | ||
|
|
ccc102dc7b | ||
|
|
2cd0ab8a28 | ||
|
|
b101ffeae1 | ||
|
|
d7f543ad4a | ||
|
|
f81c64f348 | ||
|
|
40956116dc | ||
|
|
b49a95b7b3 | ||
|
|
ae19bb2be2 | ||
|
|
749f52ff9d |
@@ -5,16 +5,23 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/ee/query-service/anomaly"
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/common"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/transition"
|
||||
"github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
querierV2 "github.com/SigNoz/signoz/pkg/query-service/app/querier/v2"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/queryBuilder"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/interfaces"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
||||
@@ -25,8 +32,6 @@ import (
|
||||
|
||||
querierV5 "github.com/SigNoz/signoz/pkg/querier"
|
||||
|
||||
"github.com/SigNoz/signoz/ee/query-service/anomaly"
|
||||
|
||||
anomalyV2 "github.com/SigNoz/signoz/ee/anomaly"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -43,12 +48,17 @@ type AnomalyRule struct {
|
||||
|
||||
reader interfaces.Reader
|
||||
|
||||
// querierV5 is the query builder v5 querier used for all alert rule evaluation
|
||||
// querierV2 is used for alerts created after the introduction of new metrics query builder
|
||||
querierV2 interfaces.Querier
|
||||
|
||||
// querierV5 is used for alerts migrated after the introduction of new query builder
|
||||
querierV5 querierV5.Querier
|
||||
|
||||
provider anomaly.Provider
|
||||
providerV2 anomalyV2.Provider
|
||||
|
||||
logger *slog.Logger
|
||||
version string
|
||||
logger *slog.Logger
|
||||
|
||||
seasonality anomaly.Seasonality
|
||||
}
|
||||
@@ -92,6 +102,34 @@ func NewAnomalyRule(
|
||||
|
||||
logger.Info("using seasonality", "seasonality", t.seasonality.String())
|
||||
|
||||
querierOptsV2 := querierV2.QuerierOptions{
|
||||
Reader: reader,
|
||||
Cache: cache,
|
||||
KeyGenerator: queryBuilder.NewKeyGenerator(),
|
||||
}
|
||||
|
||||
t.querierV2 = querierV2.NewQuerier(querierOptsV2)
|
||||
t.reader = reader
|
||||
if t.seasonality == anomaly.SeasonalityHourly {
|
||||
t.provider = anomaly.NewHourlyProvider(
|
||||
anomaly.WithCache[*anomaly.HourlyProvider](cache),
|
||||
anomaly.WithKeyGenerator[*anomaly.HourlyProvider](queryBuilder.NewKeyGenerator()),
|
||||
anomaly.WithReader[*anomaly.HourlyProvider](reader),
|
||||
)
|
||||
} else if t.seasonality == anomaly.SeasonalityDaily {
|
||||
t.provider = anomaly.NewDailyProvider(
|
||||
anomaly.WithCache[*anomaly.DailyProvider](cache),
|
||||
anomaly.WithKeyGenerator[*anomaly.DailyProvider](queryBuilder.NewKeyGenerator()),
|
||||
anomaly.WithReader[*anomaly.DailyProvider](reader),
|
||||
)
|
||||
} else if t.seasonality == anomaly.SeasonalityWeekly {
|
||||
t.provider = anomaly.NewWeeklyProvider(
|
||||
anomaly.WithCache[*anomaly.WeeklyProvider](cache),
|
||||
anomaly.WithKeyGenerator[*anomaly.WeeklyProvider](queryBuilder.NewKeyGenerator()),
|
||||
anomaly.WithReader[*anomaly.WeeklyProvider](reader),
|
||||
)
|
||||
}
|
||||
|
||||
if t.seasonality == anomaly.SeasonalityHourly {
|
||||
t.providerV2 = anomalyV2.NewHourlyProvider(
|
||||
anomalyV2.WithQuerier[*anomalyV2.HourlyProvider](querierV5),
|
||||
@@ -110,7 +148,7 @@ func NewAnomalyRule(
|
||||
}
|
||||
|
||||
t.querierV5 = querierV5
|
||||
t.reader = reader
|
||||
t.version = p.Version
|
||||
t.logger = logger
|
||||
return &t, nil
|
||||
}
|
||||
@@ -119,9 +157,36 @@ func (r *AnomalyRule) Type() ruletypes.RuleType {
|
||||
return RuleTypeAnomaly
|
||||
}
|
||||
|
||||
func (r *AnomalyRule) prepareQueryRange(ctx context.Context, ts time.Time) (*qbtypes.QueryRangeRequest, error) {
|
||||
func (r *AnomalyRule) prepareQueryRange(ctx context.Context, ts time.Time) (*v3.QueryRangeParamsV3, error) {
|
||||
|
||||
r.logger.InfoContext(ctx, "prepare query range request", "ts", ts.UnixMilli(), "eval_window", r.EvalWindow().Milliseconds(), "eval_delay", r.EvalDelay().Milliseconds())
|
||||
r.logger.InfoContext(
|
||||
ctx, "prepare query range request v4", "ts", ts.UnixMilli(), "eval_window", r.EvalWindow().Milliseconds(), "eval_delay", r.EvalDelay().Milliseconds(),
|
||||
)
|
||||
|
||||
st, en := r.Timestamps(ts)
|
||||
start := st.UnixMilli()
|
||||
end := en.UnixMilli()
|
||||
|
||||
compositeQuery := r.Condition().CompositeQuery
|
||||
|
||||
if compositeQuery.PanelType != v3.PanelTypeGraph {
|
||||
compositeQuery.PanelType = v3.PanelTypeGraph
|
||||
}
|
||||
|
||||
// default mode
|
||||
return &v3.QueryRangeParamsV3{
|
||||
Start: start,
|
||||
End: end,
|
||||
Step: int64(math.Max(float64(common.MinAllowedStepInterval(start, end)), 60)),
|
||||
CompositeQuery: compositeQuery,
|
||||
Variables: make(map[string]interface{}, 0),
|
||||
NoCache: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *AnomalyRule) prepareQueryRangeV5(ctx context.Context, ts time.Time) (*qbtypes.QueryRangeRequest, error) {
|
||||
|
||||
r.logger.InfoContext(ctx, "prepare query range request v5", "ts", ts.UnixMilli(), "eval_window", r.EvalWindow().Milliseconds(), "eval_delay", r.EvalDelay().Milliseconds())
|
||||
|
||||
startTs, endTs := r.Timestamps(ts)
|
||||
start, end := startTs.UnixMilli(), endTs.UnixMilli()
|
||||
@@ -150,6 +215,60 @@ func (r *AnomalyRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, t
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = r.PopulateTemporality(ctx, orgID, params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("internal error while setting temporality")
|
||||
}
|
||||
|
||||
anomalies, err := r.provider.GetAnomalies(ctx, orgID, &anomaly.GetAnomaliesRequest{
|
||||
Params: params,
|
||||
Seasonality: r.seasonality,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var queryResult *v3.Result
|
||||
for _, result := range anomalies.Results {
|
||||
if result.QueryName == r.GetSelectedQuery() {
|
||||
queryResult = result
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
hasData := len(queryResult.AnomalyScores) > 0
|
||||
if missingDataAlert := r.HandleMissingDataAlert(ctx, ts, hasData); missingDataAlert != nil {
|
||||
return ruletypes.Vector{*missingDataAlert}, nil
|
||||
}
|
||||
|
||||
var resultVector ruletypes.Vector
|
||||
|
||||
scoresJSON, _ := json.Marshal(queryResult.AnomalyScores)
|
||||
r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON))
|
||||
|
||||
for _, series := range queryResult.AnomalyScores {
|
||||
if !r.Condition().ShouldEval(series) {
|
||||
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||
continue
|
||||
}
|
||||
results, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{
|
||||
ActiveAlerts: r.ActiveAlertsLabelFP(),
|
||||
SendUnmatched: r.ShouldSendUnmatched(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resultVector = append(resultVector, results...)
|
||||
}
|
||||
return resultVector, nil
|
||||
}
|
||||
|
||||
func (r *AnomalyRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUID, ts time.Time) (ruletypes.Vector, error) {
|
||||
|
||||
params, err := r.prepareQueryRangeV5(ctx, ts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
anomalies, err := r.providerV2.GetAnomalies(ctx, orgID, &anomalyV2.AnomaliesRequest{
|
||||
Params: *params,
|
||||
@@ -171,25 +290,20 @@ func (r *AnomalyRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, t
|
||||
r.logger.WarnContext(ctx, "nil qb result", "ts", ts.UnixMilli())
|
||||
}
|
||||
|
||||
var anomalyScores []*qbtypes.TimeSeries
|
||||
if qbResult != nil {
|
||||
for _, bucket := range qbResult.Aggregations {
|
||||
anomalyScores = append(anomalyScores, bucket.AnomalyScores...)
|
||||
}
|
||||
}
|
||||
queryResult := transition.ConvertV5TimeSeriesDataToV4Result(qbResult)
|
||||
|
||||
hasData := len(anomalyScores) > 0
|
||||
hasData := len(queryResult.AnomalyScores) > 0
|
||||
if missingDataAlert := r.HandleMissingDataAlert(ctx, ts, hasData); missingDataAlert != nil {
|
||||
return ruletypes.Vector{*missingDataAlert}, nil
|
||||
}
|
||||
|
||||
var resultVector ruletypes.Vector
|
||||
|
||||
scoresJSON, _ := json.Marshal(anomalyScores)
|
||||
scoresJSON, _ := json.Marshal(queryResult.AnomalyScores)
|
||||
r.logger.InfoContext(ctx, "anomaly scores", "scores", string(scoresJSON))
|
||||
|
||||
// Filter out new series if newGroupEvalDelay is configured
|
||||
seriesToProcess := anomalyScores
|
||||
seriesToProcess := queryResult.AnomalyScores
|
||||
if r.ShouldSkipNewGroups() {
|
||||
filteredSeries, filterErr := r.BaseRule.FilterNewSeries(ctx, ts, seriesToProcess)
|
||||
// In case of error we log the error and continue with the original series
|
||||
@@ -202,7 +316,7 @@ func (r *AnomalyRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, t
|
||||
|
||||
for _, series := range seriesToProcess {
|
||||
if !r.Condition().ShouldEval(series) {
|
||||
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Values), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||
continue
|
||||
}
|
||||
results, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{
|
||||
@@ -223,7 +337,16 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (int, error) {
|
||||
|
||||
valueFormatter := formatter.FromUnit(r.Unit())
|
||||
|
||||
res, err := r.buildAndRunQuery(ctx, r.OrgID(), ts)
|
||||
var res ruletypes.Vector
|
||||
var err error
|
||||
|
||||
if r.version == "v5" {
|
||||
r.logger.InfoContext(ctx, "running v5 query")
|
||||
res, err = r.buildAndRunQueryV5(ctx, r.OrgID(), ts)
|
||||
} else {
|
||||
r.logger.InfoContext(ctx, "running v4 query")
|
||||
res, err = r.buildAndRunQuery(ctx, r.OrgID(), ts)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
@@ -8,27 +8,28 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
anomalyV2 "github.com/SigNoz/signoz/ee/anomaly"
|
||||
"github.com/SigNoz/signoz/ee/query-service/anomaly"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/clickhouseReader"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// mockAnomalyProviderV2 is a mock implementation of anomalyV2.Provider for testing.
|
||||
type mockAnomalyProviderV2 struct {
|
||||
responses []*anomalyV2.AnomaliesResponse
|
||||
// mockAnomalyProvider is a mock implementation of anomaly.Provider for testing.
|
||||
// We need this because the anomaly provider makes 6 different queries for various
|
||||
// time periods (current, past period, current season, past season, past 2 seasons,
|
||||
// past 3 seasons), making it cumbersome to create mock data.
|
||||
type mockAnomalyProvider struct {
|
||||
responses []*anomaly.GetAnomaliesResponse
|
||||
callCount int
|
||||
}
|
||||
|
||||
func (m *mockAnomalyProviderV2) GetAnomalies(ctx context.Context, orgID valuer.UUID, req *anomalyV2.AnomaliesRequest) (*anomalyV2.AnomaliesResponse, error) {
|
||||
func (m *mockAnomalyProvider) GetAnomalies(ctx context.Context, orgID valuer.UUID, req *anomaly.GetAnomaliesRequest) (*anomaly.GetAnomaliesResponse, error) {
|
||||
if m.callCount >= len(m.responses) {
|
||||
return &anomalyV2.AnomaliesResponse{Results: []*qbtypes.TimeSeriesData{}}, nil
|
||||
return &anomaly.GetAnomaliesResponse{Results: []*v3.Result{}}, nil
|
||||
}
|
||||
resp := m.responses[m.callCount]
|
||||
m.callCount++
|
||||
@@ -81,11 +82,11 @@ func TestAnomalyRule_NoData_AlertOnAbsent(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
responseNoData := &anomalyV2.AnomaliesResponse{
|
||||
Results: []*qbtypes.TimeSeriesData{
|
||||
responseNoData := &anomaly.GetAnomaliesResponse{
|
||||
Results: []*v3.Result{
|
||||
{
|
||||
QueryName: "A",
|
||||
Aggregations: []*qbtypes.AggregationBucket{},
|
||||
QueryName: "A",
|
||||
AnomalyScores: []*v3.Series{},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -128,8 +129,8 @@ func TestAnomalyRule_NoData_AlertOnAbsent(t *testing.T) {
|
||||
)
|
||||
require.NoError(t, err)
|
||||
|
||||
rule.providerV2 = &mockAnomalyProviderV2{
|
||||
responses: []*anomalyV2.AnomaliesResponse{responseNoData},
|
||||
rule.provider = &mockAnomalyProvider{
|
||||
responses: []*anomaly.GetAnomaliesResponse{responseNoData},
|
||||
}
|
||||
|
||||
alertsFound, err := rule.Eval(context.Background(), evalTime)
|
||||
@@ -189,11 +190,11 @@ func TestAnomalyRule_NoData_AbsentFor(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
responseNoData := &anomalyV2.AnomaliesResponse{
|
||||
Results: []*qbtypes.TimeSeriesData{
|
||||
responseNoData := &anomaly.GetAnomaliesResponse{
|
||||
Results: []*v3.Result{
|
||||
{
|
||||
QueryName: "A",
|
||||
Aggregations: []*qbtypes.AggregationBucket{},
|
||||
QueryName: "A",
|
||||
AnomalyScores: []*v3.Series{},
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -227,22 +228,16 @@ func TestAnomalyRule_NoData_AbsentFor(t *testing.T) {
|
||||
t1 := baseTime.Add(5 * time.Minute)
|
||||
t2 := t1.Add(c.timeBetweenEvals)
|
||||
|
||||
responseWithData := &anomalyV2.AnomaliesResponse{
|
||||
Results: []*qbtypes.TimeSeriesData{
|
||||
responseWithData := &anomaly.GetAnomaliesResponse{
|
||||
Results: []*v3.Result{
|
||||
{
|
||||
QueryName: "A",
|
||||
Aggregations: []*qbtypes.AggregationBucket{
|
||||
AnomalyScores: []*v3.Series{
|
||||
{
|
||||
AnomalyScores: []*qbtypes.TimeSeries{
|
||||
{
|
||||
Labels: []*qbtypes.Label{
|
||||
{Key: telemetrytypes.TelemetryFieldKey{Name: "test"}, Value: "label"},
|
||||
},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
{Timestamp: baseTime.UnixMilli(), Value: 1.0},
|
||||
{Timestamp: baseTime.Add(time.Minute).UnixMilli(), Value: 1.5},
|
||||
},
|
||||
},
|
||||
Labels: map[string]string{"test": "label"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: baseTime.UnixMilli(), Value: 1.0},
|
||||
{Timestamp: baseTime.Add(time.Minute).UnixMilli(), Value: 1.5},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -257,8 +252,8 @@ func TestAnomalyRule_NoData_AbsentFor(t *testing.T) {
|
||||
rule, err := NewAnomalyRule("test-anomaly-rule", valuer.GenerateUUID(), &postableRule, reader, nil, logger, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
rule.providerV2 = &mockAnomalyProviderV2{
|
||||
responses: []*anomalyV2.AnomaliesResponse{responseWithData, responseNoData},
|
||||
rule.provider = &mockAnomalyProvider{
|
||||
responses: []*anomaly.GetAnomaliesResponse{responseWithData, responseNoData},
|
||||
}
|
||||
|
||||
alertsFound1, err := rule.Eval(context.Background(), t1)
|
||||
|
||||
@@ -6,7 +6,7 @@ import logEvent from 'api/common/logEvent';
|
||||
import PromQLIcon from 'assets/Dashboard/PromQl';
|
||||
import { QueryBuilderV2 } from 'components/QueryBuilderV2/QueryBuilderV2';
|
||||
import { ALERTS_DATA_SOURCE_MAP } from 'constants/alerts';
|
||||
import { ENTITY_VERSION_V5 } from 'constants/app';
|
||||
import { ENTITY_VERSION_V4 } from 'constants/app';
|
||||
import { PANEL_TYPES } from 'constants/queryBuilder';
|
||||
import { QBShortcuts } from 'constants/shortcuts/QBShortcuts';
|
||||
import RunQueryBtn from 'container/QueryBuilder/components/RunQueryBtn/RunQueryBtn';
|
||||
@@ -63,8 +63,12 @@ function QuerySection({
|
||||
signalSource: signalSource === 'meter' ? 'meter' : '',
|
||||
}}
|
||||
showTraceOperator={alertType === AlertTypes.TRACES_BASED_ALERT}
|
||||
showFunctions
|
||||
version={ENTITY_VERSION_V5}
|
||||
showFunctions={
|
||||
(alertType === AlertTypes.METRICS_BASED_ALERT &&
|
||||
alertDef.version === ENTITY_VERSION_V4) ||
|
||||
alertType === AlertTypes.LOGS_BASED_ALERT
|
||||
}
|
||||
version={alertDef.version || 'v3'}
|
||||
onSignalSourceChange={handleSignalSourceChange}
|
||||
signalSourceChangeEnabled
|
||||
/>
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
@@ -96,10 +96,10 @@ var buildProcessorTestData = []struct {
|
||||
|
||||
func TestBuildLogParsingProcessors(t *testing.T) {
|
||||
for _, test := range buildProcessorTestData {
|
||||
Convey(test.Name, t, func() {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
err := updateProcessorConfigsInCollectorConf(test.agentConf, test.pipelineProcessor)
|
||||
So(err, ShouldBeNil)
|
||||
So(test.agentConf, ShouldResemble, test.outputConf)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.outputConf, test.agentConf)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -202,11 +202,11 @@ var BuildLogsPipelineTestData = []struct {
|
||||
|
||||
func TestBuildLogsPipeline(t *testing.T) {
|
||||
for _, test := range BuildLogsPipelineTestData {
|
||||
Convey(test.Name, t, func() {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
v, err := buildCollectorPipelineProcessorsList(test.currentPipeline, test.logsPipeline)
|
||||
So(err, ShouldBeNil)
|
||||
assert.NoError(t, err)
|
||||
fmt.Println(test.Name, "\n", test.currentPipeline, "\n", v, "\n", test.expectedPipeline)
|
||||
So(v, ShouldResemble, test.expectedPipeline)
|
||||
assert.Equal(t, test.expectedPipeline, v)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -239,19 +239,8 @@ func TestPipelineAliasCollisionsDontResultInDuplicateCollectorProcessors(t *test
|
||||
Alias: alias,
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -310,19 +299,8 @@ func TestPipelineRouterWorksEvenIfFirstOpIsDisabled(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -383,19 +361,8 @@ func TestPipeCharInAliasDoesntBreakCollectorConfig(t *testing.T) {
|
||||
Alias: "test|pipeline",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
|
||||
@@ -11,13 +11,13 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/agentConf"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/opamptypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/google/uuid"
|
||||
|
||||
@@ -140,15 +140,8 @@ func (ic *LogParsingPipelineController) getDefaultPipelines() ([]pipelinetypes.G
|
||||
Alias: "NormalizeBodyDefault",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "body",
|
||||
},
|
||||
Operator: v3.FilterOperatorExists,
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "body EXISTS",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
|
||||
@@ -9,14 +9,14 @@ import (
|
||||
|
||||
signozstanzahelper "github.com/SigNoz/signoz-otel-collector/processor/signozlogspipelineprocessor/stanza/operator/helper"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/google/uuid"
|
||||
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -203,10 +203,10 @@ var prepareProcessorTestData = []struct {
|
||||
|
||||
func TestPreparePipelineProcessor(t *testing.T) {
|
||||
for _, test := range prepareProcessorTestData {
|
||||
Convey(test.Name, t, func() {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
res, err := getOperators(test.Operators)
|
||||
So(err, ShouldBeNil)
|
||||
So(res, ShouldResemble, test.Output)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, test.Output, res)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -214,19 +214,8 @@ func TestPreparePipelineProcessor(t *testing.T) {
|
||||
func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
testPipelineFilter := &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
testPipelineFilter := &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
}
|
||||
makeTestPipeline := func(config []pipelinetypes.PipelineOperator) pipelinetypes.GettablePipeline {
|
||||
return pipelinetypes.GettablePipeline{
|
||||
@@ -470,19 +459,8 @@ func TestResourceFiltersWork(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "service",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeResource,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "nginx",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "resource.service = 'nginx'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -524,11 +502,11 @@ func TestResourceFiltersWork(t *testing.T) {
|
||||
func TestPipelineFilterWithStringOpsShouldNotSpamWarningsIfAttributeIsMissing(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
for _, operator := range []v3.FilterOperator{
|
||||
v3.FilterOperatorContains,
|
||||
v3.FilterOperatorNotContains,
|
||||
v3.FilterOperatorRegex,
|
||||
v3.FilterOperatorNotRegex,
|
||||
for _, operator := range []string{
|
||||
"CONTAINS",
|
||||
"NOT CONTAINS",
|
||||
"REGEXP",
|
||||
"NOT REGEXP",
|
||||
} {
|
||||
testPipeline := pipelinetypes.GettablePipeline{
|
||||
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
||||
@@ -540,19 +518,8 @@ func TestPipelineFilterWithStringOpsShouldNotSpamWarningsIfAttributeIsMissing(t
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "service",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeResource,
|
||||
},
|
||||
Operator: operator,
|
||||
Value: "nginx",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: fmt.Sprintf("resource.service %s 'nginx'", operator),
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -601,19 +568,8 @@ func TestAttributePathsContainingDollarDoNotBreakCollector(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "$test",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "test",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.$test = 'test'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -664,19 +620,8 @@ func TestMembershipOpInProcessorFieldExpressions(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "http.method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.http.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -755,7 +700,7 @@ func TestMembershipOpInProcessorFieldExpressions(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestContainsFilterIsCaseInsensitive(t *testing.T) {
|
||||
// The contains and ncontains query builder filters are case insensitive when querying logs.
|
||||
// The CONTAINS and NOT CONTAINS query builder filters are case insensitive when querying logs.
|
||||
// Pipeline filter should also behave in the same way.
|
||||
require := require.New(t)
|
||||
|
||||
@@ -773,18 +718,8 @@ func TestContainsFilterIsCaseInsensitive(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "body",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeUnspecified,
|
||||
IsColumn: true,
|
||||
},
|
||||
Operator: "contains",
|
||||
Value: "log",
|
||||
}},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "body CONTAINS 'log'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -806,18 +741,8 @@ func TestContainsFilterIsCaseInsensitive(t *testing.T) {
|
||||
Alias: "pipeline2",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "body",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeUnspecified,
|
||||
IsColumn: true,
|
||||
},
|
||||
Operator: "ncontains",
|
||||
Value: "ecom",
|
||||
}},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "body NOT CONTAINS 'ecom'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -25,19 +25,8 @@ func TestPipelinePreview(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -58,19 +47,8 @@ func TestPipelinePreview(t *testing.T) {
|
||||
Alias: "pipeline2",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
@@ -159,19 +137,8 @@ func TestGrokParsingProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{
|
||||
{
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -30,19 +30,8 @@ func TestRegexProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -97,19 +86,8 @@ func TestGrokProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -164,19 +142,8 @@ func TestJSONProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -230,20 +197,9 @@ func TestTraceParsingProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
}
|
||||
@@ -339,19 +295,8 @@ func TestAddProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -404,19 +349,8 @@ func TestRemoveProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -469,19 +403,8 @@ func TestCopyProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -535,19 +458,8 @@ func TestMoveProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
|
||||
@@ -7,8 +7,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -23,19 +23,8 @@ func TestSeverityParsingProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
@@ -141,19 +130,8 @@ func TestSeverityParsingProcessor(t *testing.T) {
|
||||
func TestNoCollectorErrorsFromSeverityParserForMismatchedLogs(t *testing.T) {
|
||||
require := require.New(t)
|
||||
|
||||
testPipelineFilter := &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
testPipelineFilter := &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
}
|
||||
makeTestPipeline := func(config []pipelinetypes.PipelineOperator) pipelinetypes.GettablePipeline {
|
||||
return pipelinetypes.GettablePipeline{
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
@@ -24,19 +24,8 @@ func TestTimestampParsingProcessor(t *testing.T) {
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
},
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "attribute.method = 'GET'",
|
||||
},
|
||||
Config: []pipelinetypes.PipelineOperator{},
|
||||
},
|
||||
|
||||
@@ -2,11 +2,19 @@ package queryBuilderToExpr
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
expr "github.com/antonmedv/expr"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
@@ -15,147 +23,635 @@ var (
|
||||
CodeExprCompilationFailed = errors.MustNewCode("expr_compilation_failed")
|
||||
)
|
||||
|
||||
var logOperatorsToExpr = map[v3.FilterOperator]string{
|
||||
v3.FilterOperatorEqual: "==",
|
||||
v3.FilterOperatorNotEqual: "!=",
|
||||
v3.FilterOperatorLessThan: "<",
|
||||
v3.FilterOperatorLessThanOrEq: "<=",
|
||||
v3.FilterOperatorGreaterThan: ">",
|
||||
v3.FilterOperatorGreaterThanOrEq: ">=",
|
||||
v3.FilterOperatorContains: "contains",
|
||||
v3.FilterOperatorNotContains: "not contains",
|
||||
v3.FilterOperatorRegex: "matches",
|
||||
v3.FilterOperatorNotRegex: "not matches",
|
||||
v3.FilterOperatorIn: "in",
|
||||
v3.FilterOperatorNotIn: "not in",
|
||||
v3.FilterOperatorExists: "in",
|
||||
v3.FilterOperatorNotExists: "not in",
|
||||
// we dont support like and nlike as of now.
|
||||
var logOperatorsToExpr = map[qbtypes.FilterOperator]string{
|
||||
qbtypes.FilterOperatorEqual: "==",
|
||||
qbtypes.FilterOperatorNotEqual: "!=",
|
||||
qbtypes.FilterOperatorLessThan: "<",
|
||||
qbtypes.FilterOperatorLessThanOrEq: "<=",
|
||||
qbtypes.FilterOperatorGreaterThan: ">",
|
||||
qbtypes.FilterOperatorGreaterThanOrEq: ">=",
|
||||
qbtypes.FilterOperatorContains: "contains",
|
||||
qbtypes.FilterOperatorNotContains: "not contains",
|
||||
qbtypes.FilterOperatorRegexp: "matches",
|
||||
qbtypes.FilterOperatorNotRegexp: "not matches",
|
||||
qbtypes.FilterOperatorIn: "in",
|
||||
qbtypes.FilterOperatorNotIn: "not in",
|
||||
qbtypes.FilterOperatorExists: "in",
|
||||
qbtypes.FilterOperatorNotExists: "not in",
|
||||
}
|
||||
|
||||
func getName(v v3.AttributeKey) string {
|
||||
if v.Type == v3.AttributeKeyTypeTag {
|
||||
return fmt.Sprintf(`attributes["%s"]`, v.Key)
|
||||
} else if v.Type == v3.AttributeKeyTypeResource {
|
||||
return fmt.Sprintf(`resource["%s"]`, v.Key)
|
||||
func getName(key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
if key == nil {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "field key is nil")
|
||||
}
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
return fmt.Sprintf(`attributes["%s"]`, key.Name), nil
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return fmt.Sprintf(`resource["%s"]`, key.Name), nil
|
||||
case telemetrytypes.FieldContextBody:
|
||||
return fmt.Sprintf("%s.%s", key.FieldContext.StringValue(), key.Name), nil
|
||||
default:
|
||||
return key.Name, nil
|
||||
}
|
||||
return v.Key
|
||||
}
|
||||
|
||||
func getTypeName(v v3.AttributeKeyType) string {
|
||||
if v == v3.AttributeKeyTypeTag {
|
||||
return "attributes"
|
||||
} else if v == v3.AttributeKeyTypeResource {
|
||||
return "resource"
|
||||
// exprVisitor is an ANTLR visitor that directly produces expr-lang expression strings,
|
||||
// eliminating the intermediate FilterExprNode / FilterCondition representation.
|
||||
type exprVisitor struct {
|
||||
errors []string
|
||||
}
|
||||
|
||||
func (v *exprVisitor) Visit(tree antlr.ParseTree) any {
|
||||
if tree == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
switch t := tree.(type) {
|
||||
case *grammar.QueryContext:
|
||||
return v.VisitQuery(t)
|
||||
case *grammar.ExpressionContext:
|
||||
return v.VisitExpression(t)
|
||||
case *grammar.OrExpressionContext:
|
||||
return v.VisitOrExpression(t)
|
||||
case *grammar.AndExpressionContext:
|
||||
return v.VisitAndExpression(t)
|
||||
case *grammar.UnaryExpressionContext:
|
||||
return v.VisitUnaryExpression(t)
|
||||
case *grammar.PrimaryContext:
|
||||
return v.VisitPrimary(t)
|
||||
case *grammar.ComparisonContext:
|
||||
return v.VisitComparison(t)
|
||||
case *grammar.InClauseContext:
|
||||
return v.VisitInClause(t)
|
||||
case *grammar.NotInClauseContext:
|
||||
return v.VisitNotInClause(t)
|
||||
case *grammar.ValueListContext:
|
||||
return v.VisitValueList(t)
|
||||
case *grammar.ValueContext:
|
||||
return v.VisitValue(t)
|
||||
case *grammar.KeyContext:
|
||||
return v.VisitKey(t)
|
||||
case *grammar.FunctionCallContext:
|
||||
return v.VisitFunctionCall(t)
|
||||
case *grammar.FunctionParamListContext:
|
||||
return v.VisitFunctionParamList(t)
|
||||
case *grammar.FunctionParamContext:
|
||||
return v.VisitFunctionParam(t)
|
||||
case *grammar.ArrayContext:
|
||||
return v.VisitArray(t)
|
||||
case *grammar.FullTextContext:
|
||||
return v.VisitFullText(t)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitQuery(ctx *grammar.QueryContext) any {
|
||||
return v.Visit(ctx.Expression())
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitExpression(ctx *grammar.ExpressionContext) any {
|
||||
return v.Visit(ctx.OrExpression())
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
|
||||
andExprs := ctx.AllAndExpression()
|
||||
var parts []string
|
||||
for _, andExpr := range andExprs {
|
||||
s, _ := v.Visit(andExpr).(string)
|
||||
if s != "" {
|
||||
parts = append(parts, s)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
return parts[0]
|
||||
}
|
||||
return strings.Join(parts, " or ")
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
|
||||
unaryExprs := ctx.AllUnaryExpression()
|
||||
var parts []string
|
||||
for _, unary := range unaryExprs {
|
||||
s, _ := v.Visit(unary).(string)
|
||||
if s != "" {
|
||||
parts = append(parts, s)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
return parts[0]
|
||||
}
|
||||
return strings.Join(parts, " and ")
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
|
||||
s, _ := v.Visit(ctx.Primary()).(string)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
if ctx.NOT() != nil {
|
||||
// VisitPrimary already wraps parenthesized sub-expressions (when the user
|
||||
// wrote explicit parens, i.e. Primary.OrExpression != nil) in '(...)'.
|
||||
// In that case, prepend "not " without adding another pair of parens to
|
||||
// avoid the double-wrapping: not ((expr)) → not (expr).
|
||||
if primaryCtx, ok := ctx.Primary().(*grammar.PrimaryContext); ok && primaryCtx.OrExpression() != nil {
|
||||
return "not " + s
|
||||
}
|
||||
return fmt.Sprintf("not (%s)", s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
|
||||
switch {
|
||||
case ctx.OrExpression() != nil:
|
||||
// Parenthesized sub-expression: wrap to preserve precedence.
|
||||
s, _ := v.Visit(ctx.OrExpression()).(string)
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("(%s)", s)
|
||||
case ctx.Comparison() != nil:
|
||||
return v.Visit(ctx.Comparison())
|
||||
case ctx.FunctionCall() != nil:
|
||||
return v.Visit(ctx.FunctionCall())
|
||||
case ctx.FullText() != nil:
|
||||
return v.Visit(ctx.FullText())
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
|
||||
key := v.buildKey(ctx.Key())
|
||||
if key == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Validate: fields without context must be intrinsic (top-level OTEL log fields).
|
||||
_, isIntrinsic := telemetrylogs.IntrinsicFields[key.Name]
|
||||
if key.FieldContext == telemetrytypes.FieldContextUnspecified && !isIntrinsic {
|
||||
v.errors = append(v.errors, fmt.Sprintf(
|
||||
"field %q in filter expression must include a context prefix (attribute., resource., body.) OR can be one of the following fields: %v",
|
||||
key.Name, maps.Keys(telemetrylogs.IntrinsicFields),
|
||||
))
|
||||
return ""
|
||||
}
|
||||
|
||||
// EXISTS / NOT EXISTS are structural and don't follow the standard value path.
|
||||
if ctx.EXISTS() != nil {
|
||||
return v.buildExistsExpr(key, ctx.NOT() != nil)
|
||||
}
|
||||
|
||||
// BETWEEN / NOT BETWEEN: two numeric values, expanded to range comparisons.
|
||||
if ctx.BETWEEN() != nil {
|
||||
return v.buildBetweenExpr(key, ctx.NOT() != nil, ctx.AllValue())
|
||||
}
|
||||
|
||||
// Determine operator from grammar tokens.
|
||||
notModifier := ctx.NOT() != nil
|
||||
var op qbtypes.FilterOperator
|
||||
switch {
|
||||
case ctx.EQUALS() != nil:
|
||||
op = qbtypes.FilterOperatorEqual
|
||||
case ctx.NOT_EQUALS() != nil || ctx.NEQ() != nil:
|
||||
op = qbtypes.FilterOperatorNotEqual
|
||||
case ctx.LT() != nil:
|
||||
op = qbtypes.FilterOperatorLessThan
|
||||
case ctx.LE() != nil:
|
||||
op = qbtypes.FilterOperatorLessThanOrEq
|
||||
case ctx.GT() != nil:
|
||||
op = qbtypes.FilterOperatorGreaterThan
|
||||
case ctx.GE() != nil:
|
||||
op = qbtypes.FilterOperatorGreaterThanOrEq
|
||||
case ctx.REGEXP() != nil:
|
||||
if notModifier {
|
||||
op = qbtypes.FilterOperatorNotRegexp
|
||||
} else {
|
||||
op = qbtypes.FilterOperatorRegexp
|
||||
}
|
||||
case ctx.CONTAINS() != nil:
|
||||
if notModifier {
|
||||
op = qbtypes.FilterOperatorNotContains
|
||||
} else {
|
||||
op = qbtypes.FilterOperatorContains
|
||||
}
|
||||
case ctx.InClause() != nil:
|
||||
op = qbtypes.FilterOperatorIn
|
||||
case ctx.NotInClause() != nil:
|
||||
op = qbtypes.FilterOperatorNotIn
|
||||
case ctx.LIKE() != nil:
|
||||
if notModifier {
|
||||
op = qbtypes.FilterOperatorNotLike
|
||||
} else {
|
||||
op = qbtypes.FilterOperatorLike
|
||||
}
|
||||
case ctx.ILIKE() != nil:
|
||||
if notModifier {
|
||||
op = qbtypes.FilterOperatorNotILike
|
||||
} else {
|
||||
op = qbtypes.FilterOperatorILike
|
||||
}
|
||||
default:
|
||||
v.errors = append(v.errors, fmt.Sprintf("unsupported comparison operator: %s", ctx.GetText()))
|
||||
return ""
|
||||
}
|
||||
|
||||
switch op {
|
||||
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorNotLike,
|
||||
qbtypes.FilterOperatorILike, qbtypes.FilterOperatorNotILike:
|
||||
// supported using functions
|
||||
default:
|
||||
if _, ok := logOperatorsToExpr[op]; !ok {
|
||||
v.errors = append(v.errors, fmt.Sprintf("operator not supported: %v", op))
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// Build the right-hand side value.
|
||||
var value any
|
||||
if op == qbtypes.FilterOperatorIn || op == qbtypes.FilterOperatorNotIn {
|
||||
value = v.buildValuesFromInClause(ctx.InClause(), ctx.NotInClause())
|
||||
} else {
|
||||
valuesCtx := ctx.AllValue()
|
||||
if len(valuesCtx) == 0 {
|
||||
v.errors = append(v.errors, "comparison operator requires a value")
|
||||
return ""
|
||||
}
|
||||
value = v.buildValue(valuesCtx[0])
|
||||
}
|
||||
|
||||
// Validate patterns eagerly.
|
||||
switch op {
|
||||
case qbtypes.FilterOperatorRegexp, qbtypes.FilterOperatorNotRegexp:
|
||||
str, ok := value.(string)
|
||||
if !ok {
|
||||
v.errors = append(v.errors, "value for regex operator must be a string")
|
||||
return ""
|
||||
}
|
||||
if _, err := regexp.Compile(str); err != nil {
|
||||
v.errors = append(v.errors, "value for regex operator must be a valid regex")
|
||||
return ""
|
||||
}
|
||||
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorNotLike,
|
||||
qbtypes.FilterOperatorILike, qbtypes.FilterOperatorNotILike:
|
||||
_, ok := value.(string)
|
||||
if !ok {
|
||||
v.errors = append(v.errors, "value for LIKE/NOT LIKE/ILIKE/ NOT ILIKE operator must be a string")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
filter, err := buildFilterExpr(key, op, value)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, err.Error())
|
||||
return ""
|
||||
}
|
||||
|
||||
if _, err := expr.Compile(filter); err != nil {
|
||||
v.errors = append(v.errors, err.Error())
|
||||
return ""
|
||||
}
|
||||
|
||||
return filter
|
||||
}
|
||||
|
||||
// buildExistsExpr produces the expr-lang string for EXISTS / NOT EXISTS checks.
|
||||
func (v *exprVisitor) buildExistsExpr(key *telemetrytypes.TelemetryFieldKey, isNotExists bool) string {
|
||||
op := qbtypes.FilterOperatorExists
|
||||
if isNotExists {
|
||||
op = qbtypes.FilterOperatorNotExists
|
||||
}
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextBody:
|
||||
// JSON body: check membership in fromJSON(body).
|
||||
// Need to quote the key for expr lang
|
||||
quoted := exprFormattedValue(key.Name)
|
||||
jsonMembership := fmt.Sprintf(
|
||||
`((type(body) == "string" && isJSON(body)) && %s %s %s)`,
|
||||
quoted, logOperatorsToExpr[op], "fromJSON(body)",
|
||||
)
|
||||
// Map body: nil check on the field.
|
||||
nilOp := qbtypes.FilterOperatorNotEqual
|
||||
if isNotExists {
|
||||
nilOp = qbtypes.FilterOperatorEqual
|
||||
}
|
||||
nilCheckFilter := fmt.Sprintf("%s.%s %s nil", key.FieldContext.StringValue(), key.Name, logOperatorsToExpr[nilOp])
|
||||
return fmt.Sprintf(`(%s or (type(body) == "map" && (%s)))`, jsonMembership, nilCheckFilter)
|
||||
case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource:
|
||||
// Check membership in the attributes / resource map.
|
||||
target := "resource"
|
||||
if key.FieldContext == telemetrytypes.FieldContextAttribute {
|
||||
target = "attributes"
|
||||
}
|
||||
return fmt.Sprintf("%q %s %s", key.Name, logOperatorsToExpr[op], target)
|
||||
default:
|
||||
// Intrinsic / top-level field: use a nil comparison.
|
||||
nilOp := qbtypes.FilterOperatorNotEqual
|
||||
if isNotExists {
|
||||
nilOp = qbtypes.FilterOperatorEqual
|
||||
}
|
||||
return fmt.Sprintf("%s %s nil", key.Name, logOperatorsToExpr[nilOp])
|
||||
}
|
||||
}
|
||||
|
||||
// buildBetweenExpr expands BETWEEN / NOT BETWEEN into range comparisons.
|
||||
// Only numeric values are accepted; strings and booleans are rejected.
|
||||
//
|
||||
// key BETWEEN lo AND hi → keyName != nil && keyName >= lo && keyName <= hi
|
||||
// key NOT BETWEEN lo AND hi → keyName != nil && (keyName < lo || keyName > hi)
|
||||
func (v *exprVisitor) buildBetweenExpr(key *telemetrytypes.TelemetryFieldKey, isNot bool, valuesCtx []grammar.IValueContext) string {
|
||||
if len(valuesCtx) != 2 {
|
||||
v.errors = append(v.errors, "BETWEEN operator requires exactly two values")
|
||||
return ""
|
||||
}
|
||||
|
||||
lo := v.buildValue(valuesCtx[0])
|
||||
hi := v.buildValue(valuesCtx[1])
|
||||
|
||||
for _, val := range []any{lo, hi} {
|
||||
switch val.(type) {
|
||||
case int64, float32, float64:
|
||||
// ok
|
||||
default:
|
||||
v.errors = append(v.errors, "BETWEEN operator requires numeric values")
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
keyName, err := getName(key)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, err.Error())
|
||||
return ""
|
||||
}
|
||||
|
||||
loStr := exprFormattedValue(lo)
|
||||
hiStr := exprFormattedValue(hi)
|
||||
|
||||
var filter string
|
||||
if isNot {
|
||||
filter = fmt.Sprintf("%s != nil && (%s < %s || %s > %s)", keyName, keyName, loStr, keyName, hiStr)
|
||||
} else {
|
||||
filter = fmt.Sprintf("%s != nil && %s >= %s && %s <= %s", keyName, keyName, loStr, keyName, hiStr)
|
||||
}
|
||||
|
||||
if _, err := expr.Compile(filter); err != nil {
|
||||
v.errors = append(v.errors, err.Error())
|
||||
return ""
|
||||
}
|
||||
|
||||
return filter
|
||||
}
|
||||
|
||||
// buildFilterExpr converts key + op + value into a final expr-lang string.
|
||||
func buildFilterExpr(key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) (string, error) {
|
||||
keyName, err := getName(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
fmtValue := exprFormattedValue(value)
|
||||
var filter string
|
||||
|
||||
switch op {
|
||||
case qbtypes.FilterOperatorContains, qbtypes.FilterOperatorNotContains:
|
||||
// contains / not contains must be case-insensitive to match query-time behaviour.
|
||||
filter = fmt.Sprintf("lower(%s) %s lower(%s)", keyName, logOperatorsToExpr[op], fmtValue)
|
||||
case qbtypes.FilterOperatorLike:
|
||||
filter = fmt.Sprintf(`type(%s) != "map" && like(string(%s), %s)`, keyName, keyName, fmtValue)
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
filter = fmt.Sprintf(`type(%s) != "map" && not like(string(%s), %s)`, keyName, keyName, fmtValue)
|
||||
case qbtypes.FilterOperatorILike:
|
||||
filter = fmt.Sprintf(`type(%s) != "map" && ilike(string(%s), %s)`, keyName, keyName, fmtValue)
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
filter = fmt.Sprintf(`type(%s) != "map" && not ilike(string(%s), %s)`, keyName, keyName, fmtValue)
|
||||
default:
|
||||
filter = fmt.Sprintf("%s %s %s", keyName, logOperatorsToExpr[op], fmtValue)
|
||||
}
|
||||
|
||||
// Avoid running operators on nil values (except equality, which handles nil fine).
|
||||
if op != qbtypes.FilterOperatorEqual && op != qbtypes.FilterOperatorNotEqual {
|
||||
filter = fmt.Sprintf("%s != nil && %s", keyName, filter)
|
||||
}
|
||||
|
||||
return filter, nil
|
||||
}
|
||||
|
||||
// buildKey turns a key grammar context into a TelemetryFieldKey.
|
||||
func (v *exprVisitor) buildKey(ctx grammar.IKeyContext) *telemetrytypes.TelemetryFieldKey {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
key := telemetrytypes.GetFieldKeyFromKeyText(ctx.GetText())
|
||||
return &key
|
||||
}
|
||||
|
||||
func (v *exprVisitor) buildValuesFromInClause(in grammar.IInClauseContext, notIn grammar.INotInClauseContext) []any {
|
||||
var ctxVal any
|
||||
if in != nil {
|
||||
ctxVal = v.VisitInClause(in)
|
||||
} else if notIn != nil {
|
||||
ctxVal = v.VisitNotInClause(notIn)
|
||||
}
|
||||
|
||||
switch ret := ctxVal.(type) {
|
||||
case []any:
|
||||
return ret
|
||||
case any:
|
||||
if ret != nil {
|
||||
return []any{ret}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitInClause(ctx grammar.IInClauseContext) any {
|
||||
if ctx.ValueList() != nil {
|
||||
return v.Visit(ctx.ValueList())
|
||||
}
|
||||
return v.Visit(ctx.Value())
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitNotInClause(ctx grammar.INotInClauseContext) any {
|
||||
if ctx.ValueList() != nil {
|
||||
return v.Visit(ctx.ValueList())
|
||||
}
|
||||
return v.Visit(ctx.Value())
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitValueList(ctx grammar.IValueListContext) any {
|
||||
values := ctx.AllValue()
|
||||
parts := make([]any, 0, len(values))
|
||||
for _, val := range values {
|
||||
parts = append(parts, v.Visit(val))
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitValue(ctx *grammar.ValueContext) any {
|
||||
return v.buildValue(ctx)
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitKey(ctx *grammar.KeyContext) any {
|
||||
return v.buildKey(ctx)
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitFunctionCall(_ *grammar.FunctionCallContext) any {
|
||||
v.errors = append(v.errors, "function calls are not supported in expr expressions")
|
||||
return ""
|
||||
}
|
||||
|
||||
func Parse(filters *v3.FilterSet) (string, error) {
|
||||
var res []string
|
||||
for _, v := range filters.Items {
|
||||
if _, ok := logOperatorsToExpr[v.Operator]; !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "operator not supported: %s", v.Operator)
|
||||
}
|
||||
|
||||
name := getName(v.Key)
|
||||
|
||||
var filter string
|
||||
|
||||
switch v.Operator {
|
||||
// uncomment following lines when new version of expr is used
|
||||
// case v3.FilterOperatorIn, v3.FilterOperatorNotIn:
|
||||
// filter = fmt.Sprintf("%s %s list%s", name, logOperatorsToExpr[v.Operator], exprFormattedValue(v.Value))
|
||||
|
||||
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
|
||||
// accustom log filters like `body.log.message EXISTS` into EXPR language
|
||||
// where User is attempting to check for keys present in JSON log body
|
||||
if strings.HasPrefix(v.Key.Key, "body.") {
|
||||
// if body is a string and is a valid JSON, then check if the key exists in the JSON
|
||||
filter = fmt.Sprintf(`((type(body) == "string" && isJSON(body)) && %s %s %s)`, exprFormattedValue(strings.TrimPrefix(v.Key.Key, "body.")), logOperatorsToExpr[v.Operator], "fromJSON(body)")
|
||||
|
||||
// if body is a map, then check if the key exists in the map
|
||||
operator := v3.FilterOperatorNotEqual
|
||||
if v.Operator == v3.FilterOperatorNotExists {
|
||||
operator = v3.FilterOperatorEqual
|
||||
}
|
||||
nilCheckFilter := fmt.Sprintf("%s %s nil", v.Key.Key, logOperatorsToExpr[operator])
|
||||
|
||||
// join the two filters with OR
|
||||
filter = fmt.Sprintf(`(%s or (type(body) == "map" && (%s)))`, filter, nilCheckFilter)
|
||||
} else if typ := getTypeName(v.Key.Type); typ != "" {
|
||||
filter = fmt.Sprintf("%s %s %s", exprFormattedValue(v.Key.Key), logOperatorsToExpr[v.Operator], typ)
|
||||
} else {
|
||||
// if type of key is not available; is considered as TOP LEVEL key in OTEL Log Data model hence
|
||||
// switch Exist and Not Exists operators with NOT EQUAL and EQUAL respectively
|
||||
operator := v3.FilterOperatorNotEqual
|
||||
if v.Operator == v3.FilterOperatorNotExists {
|
||||
operator = v3.FilterOperatorEqual
|
||||
}
|
||||
|
||||
filter = fmt.Sprintf("%s %s nil", v.Key.Key, logOperatorsToExpr[operator])
|
||||
}
|
||||
default:
|
||||
filter = fmt.Sprintf("%s %s %s", name, logOperatorsToExpr[v.Operator], exprFormattedValue(v.Value))
|
||||
|
||||
if v.Operator == v3.FilterOperatorContains || v.Operator == v3.FilterOperatorNotContains {
|
||||
// `contains` and `ncontains` should be case insensitive to match how they work when querying logs.
|
||||
filter = fmt.Sprintf(
|
||||
"lower(%s) %s lower(%s)",
|
||||
name, logOperatorsToExpr[v.Operator], exprFormattedValue(v.Value),
|
||||
)
|
||||
}
|
||||
|
||||
// Avoid running operators on nil values
|
||||
if v.Operator != v3.FilterOperatorEqual && v.Operator != v3.FilterOperatorNotEqual {
|
||||
filter = fmt.Sprintf("%s != nil && %s", name, filter)
|
||||
}
|
||||
}
|
||||
|
||||
// check if the filter is a correct expression language
|
||||
_, err := expr.Compile(filter)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
res = append(res, filter)
|
||||
}
|
||||
|
||||
// check the final filter
|
||||
q := strings.Join(res, " "+strings.ToLower(filters.Operator)+" ")
|
||||
_, err := expr.Compile(q)
|
||||
if err != nil {
|
||||
return "", errors.WrapInternalf(err, CodeExprCompilationFailed, "failed to compile expression: %s", q)
|
||||
}
|
||||
|
||||
return q, nil
|
||||
func (v *exprVisitor) VisitFunctionParamList(_ *grammar.FunctionParamListContext) any {
|
||||
v.errors = append(v.errors, "function calls are not supported in expr expressions")
|
||||
return ""
|
||||
}
|
||||
|
||||
func exprFormattedValue(v interface{}) string {
|
||||
func (v *exprVisitor) VisitFunctionParam(_ *grammar.FunctionParamContext) any {
|
||||
v.errors = append(v.errors, "function calls are not supported in expr expressions")
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitArray(_ *grammar.ArrayContext) any {
|
||||
v.errors = append(v.errors, "array literals are not supported in expr expressions")
|
||||
return ""
|
||||
}
|
||||
|
||||
func (v *exprVisitor) VisitFullText(_ *grammar.FullTextContext) any {
|
||||
v.errors = append(v.errors, "full-text search is not supported in expr expressions")
|
||||
return ""
|
||||
}
|
||||
|
||||
// buildValue converts a grammar VALUE token into a Go type.
|
||||
func (v *exprVisitor) buildValue(ctx grammar.IValueContext) any {
|
||||
switch {
|
||||
case ctx == nil:
|
||||
return nil
|
||||
case ctx.QUOTED_TEXT() != nil:
|
||||
return trimQuotes(ctx.QUOTED_TEXT().GetText())
|
||||
case ctx.NUMBER() != nil:
|
||||
text := ctx.NUMBER().GetText()
|
||||
if i, err := strconv.ParseInt(text, 10, 64); err == nil {
|
||||
return i
|
||||
}
|
||||
f, err := strconv.ParseFloat(text, 64)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to parse number %s", text))
|
||||
return nil
|
||||
}
|
||||
return f
|
||||
case ctx.BOOL() != nil:
|
||||
return strings.ToLower(ctx.BOOL().GetText()) == "true"
|
||||
case ctx.KEY() != nil:
|
||||
return ctx.KEY().GetText()
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// trimQuotes removes surrounding single or double quotes from a token and
|
||||
// unescapes embedded escape sequences produced by the ANTLR lexer.
|
||||
func trimQuotes(txt string) string {
|
||||
if len(txt) >= 2 {
|
||||
if (txt[0] == '"' && txt[len(txt)-1] == '"') ||
|
||||
(txt[0] == '\'' && txt[len(txt)-1] == '\'') {
|
||||
txt = txt[1 : len(txt)-1]
|
||||
}
|
||||
}
|
||||
txt = strings.ReplaceAll(txt, `\\`, `\`)
|
||||
txt = strings.ReplaceAll(txt, `\'`, `'`)
|
||||
return txt
|
||||
}
|
||||
|
||||
// Parse converts the QB filter Expression (query builder expression string) into
|
||||
// the Expr expression string used by the collector. It runs the ANTLR parser
|
||||
// directly and produces the output in a single visitor pass, without building an
|
||||
// intermediate FilterExprNode / FilterCondition tree.
|
||||
func Parse(filter *qbtypes.Filter) (string, error) {
|
||||
if filter == nil || strings.TrimSpace(filter.Expression) == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
input := antlr.NewInputStream(filter.Expression)
|
||||
lexer := grammar.NewFilterQueryLexer(input)
|
||||
|
||||
lexerErrorListener := querybuilder.NewErrorListener()
|
||||
lexer.RemoveErrorListeners()
|
||||
lexer.AddErrorListener(lexerErrorListener)
|
||||
|
||||
tokens := antlr.NewCommonTokenStream(lexer, 0)
|
||||
parserErrorListener := querybuilder.NewErrorListener()
|
||||
parser := grammar.NewFilterQueryParser(tokens)
|
||||
parser.RemoveErrorListeners()
|
||||
parser.AddErrorListener(parserErrorListener)
|
||||
|
||||
tree := parser.Query()
|
||||
|
||||
if len(parserErrorListener.SyntaxErrors) > 0 {
|
||||
combinedErrors := errors.Newf(
|
||||
errors.TypeInvalidInput,
|
||||
errors.CodeInvalidInput,
|
||||
"Found %d syntax errors while parsing the search expression.",
|
||||
len(parserErrorListener.SyntaxErrors),
|
||||
)
|
||||
additionals := make([]string, 0, len(parserErrorListener.SyntaxErrors))
|
||||
for _, err := range parserErrorListener.SyntaxErrors {
|
||||
if err.Error() != "" {
|
||||
additionals = append(additionals, err.Error())
|
||||
}
|
||||
}
|
||||
return "", combinedErrors.WithAdditional(additionals...)
|
||||
}
|
||||
|
||||
visitor := &exprVisitor{}
|
||||
result, _ := visitor.Visit(tree).(string)
|
||||
|
||||
if len(visitor.errors) > 0 {
|
||||
combinedErrors := errors.Newf(
|
||||
errors.TypeInvalidInput,
|
||||
errors.CodeInvalidInput,
|
||||
"Found %d errors while building expr expression.",
|
||||
len(visitor.errors),
|
||||
)
|
||||
return "", combinedErrors.WithAdditional(visitor.errors...)
|
||||
}
|
||||
|
||||
if _, err := expr.Compile(result); err != nil {
|
||||
return "", errors.WrapUnexpectedf(err, CodeExprCompilationFailed, "failed to compile expression: %s", result)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exprFormattedValue(v any) string {
|
||||
switch x := v.(type) {
|
||||
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
|
||||
return fmt.Sprintf("%d", x)
|
||||
case float32, float64:
|
||||
return fmt.Sprintf("%f", x)
|
||||
return fmt.Sprintf("%v", x)
|
||||
case string:
|
||||
return fmt.Sprintf("\"%s\"", quoteEscapedString(x))
|
||||
case bool:
|
||||
return fmt.Sprintf("%v", x)
|
||||
|
||||
case []interface{}:
|
||||
case []any:
|
||||
if len(x) == 0 {
|
||||
return ""
|
||||
}
|
||||
switch x[0].(type) {
|
||||
case string:
|
||||
str := "["
|
||||
for idx, sVal := range x {
|
||||
str += fmt.Sprintf("'%s'", quoteEscapedString(sVal.(string)))
|
||||
if idx != len(x)-1 {
|
||||
str += ","
|
||||
}
|
||||
parts := make([]string, len(x))
|
||||
for i, sVal := range x {
|
||||
parts[i] = fmt.Sprintf("'%s'", quoteEscapedString(sVal.(string)))
|
||||
}
|
||||
str += "]"
|
||||
return str
|
||||
return "[" + strings.Join(parts, ",") + "]"
|
||||
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64, float32, float64, bool:
|
||||
return strings.Join(strings.Fields(fmt.Sprint(x)), ",")
|
||||
default:
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"testing"
|
||||
|
||||
signozstanzahelper "github.com/SigNoz/signoz-otel-collector/processor/signozlogspipelineprocessor/stanza/operator/helper"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/expr-lang/expr/vm"
|
||||
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -13,185 +13,235 @@ import (
|
||||
func TestParseExpression(t *testing.T) {
|
||||
var testCases = []struct {
|
||||
Name string
|
||||
Query *v3.FilterSet
|
||||
Query *qbtypes.Filter
|
||||
Expr string
|
||||
ExpectError bool
|
||||
}{
|
||||
{
|
||||
Name: "equal",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "checkbody", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key = 'checkbody'",
|
||||
},
|
||||
Expr: `attributes["key"] == "checkbody"`,
|
||||
},
|
||||
{
|
||||
Name: "NOT equal (unary NOT)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "NOT (attribute.key = 'checkbody')",
|
||||
},
|
||||
Expr: `not (attributes["key"] == "checkbody")`,
|
||||
},
|
||||
{
|
||||
Name: "not equal",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "checkbody", Operator: "!="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key != 'checkbody'",
|
||||
},
|
||||
Expr: `attributes["key"] != "checkbody"`,
|
||||
},
|
||||
{
|
||||
Name: "less than",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key < 10",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] < 10`,
|
||||
},
|
||||
{
|
||||
Name: "greater than",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: ">"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key > 10",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] > 10`,
|
||||
},
|
||||
{
|
||||
Name: "less than equal",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key <= 10",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] <= 10`,
|
||||
},
|
||||
{
|
||||
Name: "greater than equal",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: ">="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key >= 10",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] >= 10`,
|
||||
},
|
||||
// case sensitive
|
||||
{
|
||||
Name: "body contains",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body contains 'checkbody'",
|
||||
},
|
||||
Expr: `body != nil && lower(body) contains lower("checkbody")`,
|
||||
},
|
||||
{
|
||||
Name: "body.log.message exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body.log.message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body.log.message exists",
|
||||
},
|
||||
Expr: `(((type(body) == "string" && isJSON(body)) && "log.message" in fromJSON(body)) or (type(body) == "map" && (body.log.message != nil)))`,
|
||||
},
|
||||
{
|
||||
Name: "body.log.message not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body.log.message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body.log.message not exists",
|
||||
},
|
||||
Expr: `(((type(body) == "string" && isJSON(body)) && "log.message" not in fromJSON(body)) or (type(body) == "map" && (body.log.message == nil)))`,
|
||||
},
|
||||
{
|
||||
Name: "body ncontains",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "ncontains"},
|
||||
}},
|
||||
Name: "body NOT CONTAINS",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body NOT CONTAINS 'checkbody'",
|
||||
},
|
||||
Expr: `body != nil && lower(body) not contains lower("checkbody")`,
|
||||
},
|
||||
{
|
||||
Name: "body regex",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex$", Operator: "regex"},
|
||||
}},
|
||||
Name: "body REGEXP",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body REGEXP '[0-1]+regex$'",
|
||||
},
|
||||
Expr: `body != nil && body matches "[0-1]+regex$"`,
|
||||
},
|
||||
{
|
||||
Name: "body not regex",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex$", Operator: "nregex"},
|
||||
}},
|
||||
Name: "body NOT REGEXP",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body NOT REGEXP '[0-1]+regex$'",
|
||||
},
|
||||
Expr: `body != nil && body not matches "[0-1]+regex$"`,
|
||||
},
|
||||
{
|
||||
Name: "regex with escape characters",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: `^Executing \[\S+@\S+:[0-9]+\] \S+".*`, Operator: "regex"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body REGEXP '^Executing \\[\\S+@\\S+:[0-9]+\\] \\S+\".*'",
|
||||
},
|
||||
Expr: `body != nil && body matches "^Executing \\[\\S+@\\S+:[0-9]+\\] \\S+\".*"`,
|
||||
},
|
||||
{
|
||||
Name: "invalid regex",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-9]++", Operator: "nregex"},
|
||||
}},
|
||||
Expr: `body != nil && lower(body) not matches "[0-9]++"`,
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body not REGEXP '[0-9]++'",
|
||||
},
|
||||
ExpectError: true,
|
||||
},
|
||||
{
|
||||
Name: "in",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{1, 2, 3, 4}, Operator: "in"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key in [1,2,3,4]",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] in [1,2,3,4]`,
|
||||
},
|
||||
{
|
||||
Name: "not in",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{"1", "2"}, Operator: "nin"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key not in ['1','2']",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] not in ['1','2']`,
|
||||
},
|
||||
{
|
||||
Name: "exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key exists",
|
||||
},
|
||||
Expr: `"key" in attributes`,
|
||||
},
|
||||
{
|
||||
Name: "not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key not exists",
|
||||
},
|
||||
Expr: `"key" not in attributes`,
|
||||
},
|
||||
{
|
||||
Name: "trace_id not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "trace_id not exists",
|
||||
},
|
||||
Expr: `trace_id == nil`,
|
||||
},
|
||||
{
|
||||
Name: "trace_id exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "trace_id exists",
|
||||
},
|
||||
Expr: `trace_id != nil`,
|
||||
},
|
||||
{
|
||||
Name: "span_id not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "span_id not exists",
|
||||
},
|
||||
Expr: `span_id == nil`,
|
||||
},
|
||||
{
|
||||
Name: "span_id exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "span_id exists",
|
||||
},
|
||||
Expr: `span_id != nil`,
|
||||
},
|
||||
{
|
||||
Name: "Multi filter",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<="},
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex$", Operator: "nregex"},
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key <= 10 and body not regexp '[0-1]+regex$' and attribute.key not exists",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] <= 10 and body != nil && body not matches "[0-1]+regex$" and "key" not in attributes`,
|
||||
},
|
||||
{
|
||||
Name: "incorrect multi filter",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<="},
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-9]++", Operator: "nregex"},
|
||||
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
|
||||
}},
|
||||
Expr: `attributes["key"] != nil && attributes["key"] <= 10 and body not matches "[0-9]++" and "key" not in attributes`,
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key <= 10 and body not regexp '[0-9]++' and attribute.key not exists",
|
||||
},
|
||||
ExpectError: true,
|
||||
},
|
||||
{
|
||||
Name: "attributes. is unsupported",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attributes.key = 'checkbody'",
|
||||
},
|
||||
ExpectError: true,
|
||||
},
|
||||
{
|
||||
Name: "LIKE",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key LIKE 'foo%'",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && like(string(attributes["key"]), "foo%")`,
|
||||
},
|
||||
{
|
||||
Name: "NOT LIKE",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key NOT LIKE 'foo%'",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && not like(string(attributes["key"]), "foo%")`,
|
||||
},
|
||||
{
|
||||
Name: "ILIKE",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key ILIKE 'FOO%'",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && ilike(string(attributes["key"]), "FOO%")`,
|
||||
},
|
||||
{
|
||||
Name: "NOT ILIKE",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.key NOT ILIKE 'FOO%'",
|
||||
},
|
||||
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && not ilike(string(attributes["key"]), "FOO%")`,
|
||||
},
|
||||
{
|
||||
Name: "body LIKE",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body LIKE 'Server%'",
|
||||
},
|
||||
Expr: `body != nil && type(body) != "map" && like(string(body), "Server%")`,
|
||||
},
|
||||
{
|
||||
Name: "body ILIKE",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body ILIKE 'server%'",
|
||||
},
|
||||
Expr: `body != nil && type(body) != "map" && ilike(string(body), "server%")`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range testCases {
|
||||
@@ -267,250 +317,360 @@ func TestExpressionVSEntry(t *testing.T) {
|
||||
|
||||
var testCases = []struct {
|
||||
Name string
|
||||
Query *v3.FilterSet
|
||||
Query *qbtypes.Filter
|
||||
ExpectedMatches []int
|
||||
}{
|
||||
{
|
||||
Name: "resource equal (env)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.env = 'prod'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 1, 4, 5, 9, 11, 13, 14, 17, 18},
|
||||
},
|
||||
{
|
||||
Name: "NOT resource equal (unary NOT)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "NOT (resource.env = 'prod')",
|
||||
},
|
||||
ExpectedMatches: []int{2, 3, 6, 7, 8, 10, 12, 15, 16, 19, 20},
|
||||
},
|
||||
{
|
||||
Name: "resource not equal (env)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "!="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.env != 'prod'",
|
||||
},
|
||||
ExpectedMatches: []int{2, 3, 6, 7, 8, 10, 12, 15, 16, 19, 20},
|
||||
},
|
||||
{
|
||||
Name: "attribute less than (numeric)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 8, Operator: "<"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.count < 8",
|
||||
},
|
||||
ExpectedMatches: []int{4},
|
||||
},
|
||||
{
|
||||
Name: "attribute greater than (numeric)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 8, Operator: ">"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.count > 8",
|
||||
},
|
||||
ExpectedMatches: []int{5},
|
||||
},
|
||||
{
|
||||
Name: "body contains (case insensitive)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body contains 'checkbody'",
|
||||
},
|
||||
ExpectedMatches: []int{2, 9, 10, 16},
|
||||
},
|
||||
{
|
||||
Name: "body ncontains",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "ncontains"},
|
||||
}},
|
||||
Name: "body NOT CONTAINS",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body NOT CONTAINS 'checkbody'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 1, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 17},
|
||||
},
|
||||
{
|
||||
Name: "body.msg (case insensitive)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body.msg", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: false}, Value: "checkbody", Operator: "contains"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body.msg contains 'checkbody'",
|
||||
},
|
||||
ExpectedMatches: []int{2, 9, 10, 18},
|
||||
},
|
||||
{
|
||||
Name: "body regex",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex", Operator: "regex"},
|
||||
}},
|
||||
Name: "body REGEXP",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body REGEXP '[0-1]+regex'",
|
||||
},
|
||||
ExpectedMatches: []int{4},
|
||||
},
|
||||
{
|
||||
Name: "body not regex",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex", Operator: "nregex"},
|
||||
}},
|
||||
Name: "body NOT REGEXP",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body NOT REGEXP '[0-1]+regex'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17},
|
||||
},
|
||||
// body.log.message exists/nexists: expr checks "log.message" in fromJSON(body); nested key
|
||||
// semantics depend on signoz stanza helper. Omitted here to avoid coupling to env shape.
|
||||
{
|
||||
Name: "body top-level key exists (body.msg)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body.msg", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body.msg exists",
|
||||
},
|
||||
ExpectedMatches: []int{0, 1, 2, 3, 4, 5, 9, 10, 11, 12, 18},
|
||||
},
|
||||
{
|
||||
Name: "body top-level key not exists (body.missing)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body.missing", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body.missing not exists",
|
||||
},
|
||||
ExpectedMatches: []int{0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 18, 20},
|
||||
},
|
||||
{
|
||||
Name: "attribute exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.service exists",
|
||||
},
|
||||
ExpectedMatches: []int{6, 7, 8, 15},
|
||||
},
|
||||
{
|
||||
Name: "attribute not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.service not exists",
|
||||
},
|
||||
ExpectedMatches: []int{0, 1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20},
|
||||
},
|
||||
{
|
||||
Name: "trace_id exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "trace_id exists",
|
||||
},
|
||||
ExpectedMatches: []int{1, 2, 5, 7, 12, 15, 19},
|
||||
},
|
||||
{
|
||||
Name: "trace_id not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "trace_id not exists",
|
||||
},
|
||||
ExpectedMatches: []int{0, 3, 4, 6, 8, 9, 10, 11, 13, 14, 16, 17, 18, 20},
|
||||
},
|
||||
{
|
||||
Name: "span_id exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "span_id exists",
|
||||
},
|
||||
ExpectedMatches: []int{1, 3, 5, 12, 17, 20},
|
||||
},
|
||||
{
|
||||
Name: "span_id not exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "span_id not exists",
|
||||
},
|
||||
ExpectedMatches: []int{0, 2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 18, 19},
|
||||
},
|
||||
{
|
||||
Name: "in (attribute in list)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{"info", "error"}, Operator: "in"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level in ['info', 'error']",
|
||||
},
|
||||
ExpectedMatches: []int{0, 1, 2, 14, 16, 20},
|
||||
},
|
||||
{
|
||||
Name: "not in (attribute not in list)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{"error", "warn"}, Operator: "nin"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level not in ['error', 'warn']",
|
||||
},
|
||||
ExpectedMatches: []int{0, 2, 3, 16, 20},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "staging", Operator: "="},
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "info", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.env = 'staging' and attribute.level = 'info'",
|
||||
},
|
||||
ExpectedMatches: []int{2, 16},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND (two attributes)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "auth", Operator: "="},
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.service = 'auth' and attribute.level not exists",
|
||||
},
|
||||
ExpectedMatches: []int{6, 7},
|
||||
},
|
||||
// Multi-filter variations: body + attribute, three conditions, trace/span + attribute
|
||||
{
|
||||
Name: "multi filter AND body contains + attribute",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "Connection", Operator: "contains"},
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body contains 'Connection' and resource.env = 'prod'",
|
||||
},
|
||||
ExpectedMatches: []int{14},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND body contains + service",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "login", Operator: "contains"},
|
||||
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "auth", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body contains 'login' and attribute.service = 'auth'",
|
||||
},
|
||||
ExpectedMatches: []int{6, 15},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND env + level (prod error)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "error", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.env = 'prod' and attribute.level = 'error'",
|
||||
},
|
||||
ExpectedMatches: []int{1, 14},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND three conditions (staging + checkbody + info)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "staging", Operator: "="},
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "info", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.env = 'staging' and body contains 'checkbody' and attribute.level = 'info'",
|
||||
},
|
||||
ExpectedMatches: []int{2, 16},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND trace_id exists + body contains",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "trace_id exists and body contains 'checkbody'",
|
||||
},
|
||||
ExpectedMatches: []int{2},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND span_id nexists + service auth",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "auth", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "span_id not exists and attribute.service = 'auth'",
|
||||
},
|
||||
ExpectedMatches: []int{6, 7, 15},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND body regex + attribute",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex", Operator: "regex"},
|
||||
{Key: v3.AttributeKey{Key: "code", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "200", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body regexp '[0-1]+regex' and attribute.code = '200'",
|
||||
},
|
||||
ExpectedMatches: []int{4},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND no trace_id + no span_id + env prod",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "trace_id not exists and span_id not exists and resource.env = 'prod'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 4, 9, 11, 13, 14, 18},
|
||||
},
|
||||
{
|
||||
Name: "multi filter AND level warn + body contains",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "warn", Operator: "="},
|
||||
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "disk", Operator: "contains"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level = 'warn' and body contains 'disk'",
|
||||
},
|
||||
ExpectedMatches: []int{17},
|
||||
},
|
||||
{
|
||||
Name: "no matches (attribute value not present)",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "never", Operator: "="},
|
||||
}},
|
||||
Name: "no matches (resource value not present)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.env = 'never'",
|
||||
},
|
||||
ExpectedMatches: []int{},
|
||||
},
|
||||
{
|
||||
Name: "attribute equal and trace_id exists",
|
||||
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
|
||||
{Key: v3.AttributeKey{Key: "code", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "404", Operator: "="},
|
||||
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
|
||||
}},
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.code = '404' and trace_id exists",
|
||||
},
|
||||
ExpectedMatches: []int{5},
|
||||
},
|
||||
// LIKE / NOT LIKE / ILIKE / NOT ILIKE – pattern and type-safety coverage
|
||||
{
|
||||
Name: "LIKE exact match (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level LIKE 'info'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 2, 16, 20},
|
||||
},
|
||||
{
|
||||
Name: "LIKE prefix match (body)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body LIKE 'Server%'",
|
||||
},
|
||||
ExpectedMatches: []int{13},
|
||||
},
|
||||
{
|
||||
Name: "LIKE suffix match (body)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body LIKE '%8080'",
|
||||
},
|
||||
ExpectedMatches: []int{13},
|
||||
},
|
||||
{
|
||||
Name: "LIKE substring with % (body)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body LIKE '%checkbody%'",
|
||||
},
|
||||
// map bodies excluded by type check; entries 18-20 have map body → no match
|
||||
ExpectedMatches: []int{2, 9, 16},
|
||||
},
|
||||
{
|
||||
Name: "LIKE single-char wildcard (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level LIKE 'inf_'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 2, 16, 20},
|
||||
},
|
||||
{
|
||||
Name: "LIKE prefix on attribute",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.service LIKE 'au%'",
|
||||
},
|
||||
ExpectedMatches: []int{6, 7, 15},
|
||||
},
|
||||
{
|
||||
Name: "LIKE pattern on resource",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "resource.host LIKE 'node-1%'",
|
||||
},
|
||||
ExpectedMatches: []int{1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
|
||||
},
|
||||
{
|
||||
Name: "NOT LIKE exact match (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level NOT LIKE 'info'",
|
||||
},
|
||||
ExpectedMatches: []int{1, 3, 14, 15, 17},
|
||||
},
|
||||
{
|
||||
Name: "NOT LIKE suffix pattern (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level NOT LIKE '%o'",
|
||||
},
|
||||
// level ending in 'o': info; not ending in 'o': error, debug, warn
|
||||
ExpectedMatches: []int{1, 3, 14, 15, 17},
|
||||
},
|
||||
{
|
||||
Name: "ILIKE case-insensitive exact (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level ILIKE 'INFO'",
|
||||
},
|
||||
ExpectedMatches: []int{0, 2, 16, 20},
|
||||
},
|
||||
{
|
||||
Name: "ILIKE case-insensitive prefix (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.status ILIKE 'ACT%'",
|
||||
},
|
||||
ExpectedMatches: []int{11},
|
||||
},
|
||||
{
|
||||
Name: "ILIKE case-insensitive substring (body)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "body ILIKE '%checkbody%'",
|
||||
},
|
||||
// map bodies excluded by type check
|
||||
ExpectedMatches: []int{2, 9, 10, 16},
|
||||
},
|
||||
{
|
||||
Name: "NOT ILIKE case-insensitive exact (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level NOT ILIKE 'INFO'",
|
||||
},
|
||||
ExpectedMatches: []int{1, 3, 14, 15, 17},
|
||||
},
|
||||
{
|
||||
Name: "NOT ILIKE case-insensitive (attribute)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.service NOT ILIKE 'API'",
|
||||
},
|
||||
ExpectedMatches: []int{6, 7, 15},
|
||||
},
|
||||
{
|
||||
Name: "LIKE numeric attribute converted via string()",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.count LIKE '%5%'",
|
||||
},
|
||||
// count int64(5) -> string "5" matches; count int64(10) -> "10" does not
|
||||
ExpectedMatches: []int{4},
|
||||
},
|
||||
{
|
||||
Name: "LIKE with multi filter (attribute + resource)",
|
||||
Query: &qbtypes.Filter{
|
||||
Expression: "attribute.level LIKE 'info' and resource.env = 'prod'",
|
||||
},
|
||||
ExpectedMatches: []int{0},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range testCases {
|
||||
|
||||
@@ -625,7 +625,7 @@ func (r *BaseRule) extractMetricAndGroupBys(ctx context.Context) (map[string][]s
|
||||
|
||||
// FilterNewSeries filters out items that are too new based on metadata first_seen timestamps.
|
||||
// Returns the filtered series (old ones) excluding new series that are still within the grace period.
|
||||
func (r *BaseRule) FilterNewSeries(ctx context.Context, ts time.Time, series []*qbtypes.TimeSeries) ([]*qbtypes.TimeSeries, error) {
|
||||
func (r *BaseRule) FilterNewSeries(ctx context.Context, ts time.Time, series []*v3.Series) ([]*v3.Series, error) {
|
||||
// Extract metric names and groupBy keys
|
||||
metricToGroupedFields, err := r.extractMetricAndGroupBys(ctx)
|
||||
if err != nil {
|
||||
@@ -642,7 +642,7 @@ func (r *BaseRule) FilterNewSeries(ctx context.Context, ts time.Time, series []*
|
||||
seriesIdxToLookupKeys := make(map[int][]telemetrytypes.MetricMetadataLookupKey) // series index -> lookup keys
|
||||
|
||||
for i := 0; i < len(series); i++ {
|
||||
metricLabelMap := series[i].LabelsMap()
|
||||
metricLabelMap := series[i].Labels
|
||||
|
||||
// Collect groupBy attribute-value pairs for this series
|
||||
seriesKeys := make([]telemetrytypes.MetricMetadataLookupKey, 0)
|
||||
@@ -689,7 +689,7 @@ func (r *BaseRule) FilterNewSeries(ctx context.Context, ts time.Time, series []*
|
||||
}
|
||||
|
||||
// Filter series based on first_seen + delay
|
||||
filteredSeries := make([]*qbtypes.TimeSeries, 0, len(series))
|
||||
filteredSeries := make([]*v3.Series, 0, len(series))
|
||||
evalTimeMs := ts.UnixMilli()
|
||||
newGroupEvalDelayMs := r.newGroupEvalDelay.Milliseconds()
|
||||
|
||||
@@ -727,7 +727,7 @@ func (r *BaseRule) FilterNewSeries(ctx context.Context, ts time.Time, series []*
|
||||
// Check if first_seen + delay has passed
|
||||
if maxFirstSeen+newGroupEvalDelayMs > evalTimeMs {
|
||||
// Still within grace period, skip this series
|
||||
r.logger.InfoContext(ctx, "Skipping new series", "rule_name", r.Name(), "series_idx", i, "max_first_seen", maxFirstSeen, "eval_time_ms", evalTimeMs, "delay_ms", newGroupEvalDelayMs, "labels", series[i].LabelsMap())
|
||||
r.logger.InfoContext(ctx, "Skipping new series", "rule_name", r.Name(), "series_idx", i, "max_first_seen", maxFirstSeen, "eval_time_ms", evalTimeMs, "delay_ms", newGroupEvalDelayMs, "labels", series[i].Labels)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -26,33 +26,33 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// createTestSeries creates a *qbtypes.TimeSeries with the given labels and optional values
|
||||
// createTestSeries creates a *v3.Series with the given labels and optional points
|
||||
// so we don't exactly need the points in the series because the labels are used to determine if the series is new or old
|
||||
// we use the labels to create a lookup key for the series and then check the first_seen timestamp for the series in the metadata table
|
||||
func createTestSeries(labels map[string]string, points []*qbtypes.TimeSeriesValue) *qbtypes.TimeSeries {
|
||||
func createTestSeries(labels map[string]string, points []v3.Point) *v3.Series {
|
||||
if points == nil {
|
||||
points = []*qbtypes.TimeSeriesValue{}
|
||||
points = []v3.Point{}
|
||||
}
|
||||
lbls := make([]*qbtypes.Label, 0, len(labels))
|
||||
for k, v := range labels {
|
||||
lbls = append(lbls, &qbtypes.Label{Key: telemetrytypes.TelemetryFieldKey{Name: k}, Value: v})
|
||||
}
|
||||
return &qbtypes.TimeSeries{
|
||||
Labels: lbls,
|
||||
Values: points,
|
||||
return &v3.Series{
|
||||
Labels: labels,
|
||||
Points: points,
|
||||
}
|
||||
}
|
||||
|
||||
// seriesEqual compares two *qbtypes.TimeSeries by their labels
|
||||
// seriesEqual compares two v3.Series by their labels
|
||||
// Returns true if the series have the same labels (order doesn't matter)
|
||||
func seriesEqual(s1, s2 *qbtypes.TimeSeries) bool {
|
||||
m1 := s1.LabelsMap()
|
||||
m2 := s2.LabelsMap()
|
||||
if len(m1) != len(m2) {
|
||||
func seriesEqual(s1, s2 *v3.Series) bool {
|
||||
if s1 == nil && s2 == nil {
|
||||
return true
|
||||
}
|
||||
if s1 == nil || s2 == nil {
|
||||
return false
|
||||
}
|
||||
for k, v := range m1 {
|
||||
if m2[k] != v {
|
||||
if len(s1.Labels) != len(s2.Labels) {
|
||||
return false
|
||||
}
|
||||
for k, v := range s1.Labels {
|
||||
if s2.Labels[k] != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -149,11 +149,11 @@ func createPostableRule(compositeQuery *v3.CompositeQuery) ruletypes.PostableRul
|
||||
type filterNewSeriesTestCase struct {
|
||||
name string
|
||||
compositeQuery *v3.CompositeQuery
|
||||
series []*qbtypes.TimeSeries
|
||||
series []*v3.Series
|
||||
firstSeenMap map[telemetrytypes.MetricMetadataLookupKey]int64
|
||||
newGroupEvalDelay valuer.TextDuration
|
||||
evalTime time.Time
|
||||
expectedFiltered []*qbtypes.TimeSeries // series that should be in the final filtered result (old enough)
|
||||
expectedFiltered []*v3.Series // series that should be in the final filtered result (old enough)
|
||||
expectError bool
|
||||
}
|
||||
|
||||
@@ -193,7 +193,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-old", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-new", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-missing", "env": "stage"}, nil),
|
||||
@@ -205,7 +205,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-old", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-missing", "env": "stage"}, nil),
|
||||
}, // svc-old and svc-missing should be included; svc-new is filtered out
|
||||
@@ -227,7 +227,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-new1", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-new2", "env": "stage"}, nil),
|
||||
},
|
||||
@@ -237,7 +237,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{}, // all should be filtered out (new series)
|
||||
expectedFiltered: []*v3.Series{}, // all should be filtered out (new series)
|
||||
},
|
||||
{
|
||||
name: "all old series - ClickHouse query",
|
||||
@@ -254,7 +254,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-old1", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-old2", "env": "stage"}, nil),
|
||||
},
|
||||
@@ -264,7 +264,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-old1", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-old2", "env": "stage"}, nil),
|
||||
}, // all should be included (old series)
|
||||
@@ -292,13 +292,13 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
},
|
||||
firstSeenMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
}, // early return, no filtering - all series included
|
||||
},
|
||||
@@ -322,13 +322,13 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
},
|
||||
firstSeenMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
}, // early return, no filtering - all series included
|
||||
},
|
||||
@@ -358,13 +358,13 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"status": "200"}, nil), // no service_name or env
|
||||
},
|
||||
firstSeenMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"status": "200"}, nil),
|
||||
}, // series included as we can't decide if it's new or old
|
||||
},
|
||||
@@ -385,7 +385,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-old", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-no-metadata", "env": "prod"}, nil),
|
||||
},
|
||||
@@ -393,7 +393,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
// svc-no-metadata has no entry in firstSeenMap
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-old", "env": "prod"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc-no-metadata", "env": "prod"}, nil),
|
||||
}, // both should be included - svc-old is old, svc-no-metadata can't be decided
|
||||
@@ -413,7 +413,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-partial", "env": "prod"}, nil),
|
||||
},
|
||||
// Only provide metadata for service_name, not env
|
||||
@@ -423,7 +423,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc-partial", "env": "prod"}, nil),
|
||||
}, // has some metadata, uses max first_seen which is old
|
||||
},
|
||||
@@ -453,11 +453,11 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{},
|
||||
series: []*v3.Series{},
|
||||
firstSeenMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{},
|
||||
expectedFiltered: []*v3.Series{},
|
||||
},
|
||||
{
|
||||
name: "zero delay - Builder",
|
||||
@@ -485,13 +485,13 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
},
|
||||
firstSeenMap: createFirstSeenMap("request_total", defaultGroupByFields, defaultEvalTime, defaultDelay, true, "svc1", "prod"),
|
||||
newGroupEvalDelay: valuer.TextDuration{}, // zero delay
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
}, // with zero delay, all series pass
|
||||
},
|
||||
@@ -526,7 +526,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
},
|
||||
firstSeenMap: mergeFirstSeenMaps(
|
||||
@@ -535,7 +535,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
},
|
||||
},
|
||||
@@ -565,7 +565,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1", "env": "prod"}, nil),
|
||||
},
|
||||
// service_name is old, env is new - should use max (new)
|
||||
@@ -575,7 +575,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{}, // max first_seen is new, so should be filtered out
|
||||
expectedFiltered: []*v3.Series{}, // max first_seen is new, so should be filtered out
|
||||
},
|
||||
{
|
||||
name: "Logs query - should skip filtering and return empty skip indexes",
|
||||
@@ -600,14 +600,14 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc2"}, nil),
|
||||
},
|
||||
firstSeenMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc2"}, nil),
|
||||
}, // Logs queries should return early, no filtering - all included
|
||||
@@ -635,14 +635,14 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
series: []*qbtypes.TimeSeries{
|
||||
series: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc2"}, nil),
|
||||
},
|
||||
firstSeenMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
|
||||
newGroupEvalDelay: defaultNewGroupEvalDelay,
|
||||
evalTime: defaultEvalTime,
|
||||
expectedFiltered: []*qbtypes.TimeSeries{
|
||||
expectedFiltered: []*v3.Series{
|
||||
createTestSeries(map[string]string{"service_name": "svc1"}, nil),
|
||||
createTestSeries(map[string]string{"service_name": "svc2"}, nil),
|
||||
}, // Traces queries should return early, no filtering - all included
|
||||
@@ -724,14 +724,14 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
|
||||
// Build a map to count occurrences of each unique label combination in expected series
|
||||
expectedCounts := make(map[string]int)
|
||||
for _, expected := range tt.expectedFiltered {
|
||||
key := labelsKey(expected.LabelsMap())
|
||||
key := labelsKey(expected.Labels)
|
||||
expectedCounts[key]++
|
||||
}
|
||||
|
||||
// Build a map to count occurrences of each unique label combination in filtered series
|
||||
actualCounts := make(map[string]int)
|
||||
for _, filtered := range filteredSeries {
|
||||
key := labelsKey(filtered.LabelsMap())
|
||||
key := labelsKey(filtered.Labels)
|
||||
actualCounts[key]++
|
||||
}
|
||||
|
||||
|
||||
@@ -12,18 +12,19 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/formatter"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/interfaces"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
qslabels "github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
)
|
||||
|
||||
type PromRule struct {
|
||||
*BaseRule
|
||||
version string
|
||||
prometheus prometheus.Prometheus
|
||||
}
|
||||
|
||||
@@ -47,6 +48,7 @@ func NewPromRule(
|
||||
|
||||
p := PromRule{
|
||||
BaseRule: baseRule,
|
||||
version: postableRule.Version,
|
||||
prometheus: prometheus,
|
||||
}
|
||||
p.logger = logger
|
||||
@@ -81,30 +83,48 @@ func (r *PromRule) GetSelectedQuery() string {
|
||||
}
|
||||
|
||||
func (r *PromRule) getPqlQuery() (string, error) {
|
||||
if len(r.ruleCondition.CompositeQuery.Queries) > 0 {
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
for _, item := range r.ruleCondition.CompositeQuery.Queries {
|
||||
switch item.Type {
|
||||
case qbtypes.QueryTypePromQL:
|
||||
promQuery, ok := item.Spec.(qbtypes.PromQuery)
|
||||
if !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query spec %T", item.Spec)
|
||||
}
|
||||
if promQuery.Name == selectedQuery {
|
||||
return promQuery.Query, nil
|
||||
if r.version == "v5" {
|
||||
if len(r.ruleCondition.CompositeQuery.Queries) > 0 {
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
for _, item := range r.ruleCondition.CompositeQuery.Queries {
|
||||
switch item.Type {
|
||||
case qbtypes.QueryTypePromQL:
|
||||
promQuery, ok := item.Spec.(qbtypes.PromQuery)
|
||||
if !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query spec %T", item.Spec)
|
||||
}
|
||||
if promQuery.Name == selectedQuery {
|
||||
return promQuery.Query, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("invalid promql rule setup")
|
||||
}
|
||||
return "", fmt.Errorf("invalid promql rule setup")
|
||||
|
||||
if r.ruleCondition.CompositeQuery.QueryType == v3.QueryTypePromQL {
|
||||
if len(r.ruleCondition.CompositeQuery.PromQueries) > 0 {
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
if promQuery, ok := r.ruleCondition.CompositeQuery.PromQueries[selectedQuery]; ok {
|
||||
query := promQuery.Query
|
||||
if query == "" {
|
||||
return query, fmt.Errorf("a promquery needs to be set for this rule to function")
|
||||
}
|
||||
return query, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("invalid promql rule query")
|
||||
}
|
||||
|
||||
func matrixToTimeSeries(res promql.Matrix) []*qbtypes.TimeSeries {
|
||||
result := make([]*qbtypes.TimeSeries, 0, len(res))
|
||||
func (r *PromRule) matrixToV3Series(res promql.Matrix) []*v3.Series {
|
||||
v3Series := make([]*v3.Series, 0, len(res))
|
||||
for _, series := range res {
|
||||
result = append(result, promSeriesToTimeSeries(series))
|
||||
commonSeries := toCommonSeries(series)
|
||||
v3Series = append(v3Series, &commonSeries)
|
||||
}
|
||||
return result
|
||||
return v3Series
|
||||
}
|
||||
|
||||
func (r *PromRule) buildAndRunQuery(ctx context.Context, ts time.Time) (ruletypes.Vector, error) {
|
||||
@@ -123,31 +143,31 @@ func (r *PromRule) buildAndRunQuery(ctx context.Context, ts time.Time) (ruletype
|
||||
return nil, err
|
||||
}
|
||||
|
||||
seriesToProcess := matrixToTimeSeries(res)
|
||||
matrixToProcess := r.matrixToV3Series(res)
|
||||
|
||||
hasData := len(seriesToProcess) > 0
|
||||
hasData := len(matrixToProcess) > 0
|
||||
if missingDataAlert := r.HandleMissingDataAlert(ctx, ts, hasData); missingDataAlert != nil {
|
||||
return ruletypes.Vector{*missingDataAlert}, nil
|
||||
}
|
||||
|
||||
// Filter out new series if newGroupEvalDelay is configured
|
||||
if r.ShouldSkipNewGroups() {
|
||||
filteredSeries, filterErr := r.BaseRule.FilterNewSeries(ctx, ts, seriesToProcess)
|
||||
filteredSeries, filterErr := r.BaseRule.FilterNewSeries(ctx, ts, matrixToProcess)
|
||||
// In case of error we log the error and continue with the original series
|
||||
if filterErr != nil {
|
||||
r.logger.ErrorContext(ctx, "Error filtering new series, ", "error", filterErr, "rule_name", r.Name())
|
||||
} else {
|
||||
seriesToProcess = filteredSeries
|
||||
matrixToProcess = filteredSeries
|
||||
}
|
||||
}
|
||||
|
||||
var resultVector ruletypes.Vector
|
||||
|
||||
for _, series := range seriesToProcess {
|
||||
for _, series := range matrixToProcess {
|
||||
if !r.Condition().ShouldEval(series) {
|
||||
r.logger.InfoContext(
|
||||
ctx, "not enough data points to evaluate series, skipping",
|
||||
"rule_id", r.ID(), "num_points", len(series.Values), "required_points", r.Condition().RequiredNumPoints,
|
||||
"rule_id", r.ID(), "num_points", len(series.Points), "required_points", r.Condition().RequiredNumPoints,
|
||||
)
|
||||
continue
|
||||
}
|
||||
@@ -434,25 +454,26 @@ func (r *PromRule) RunAlertQuery(ctx context.Context, qs string, start, end time
|
||||
}
|
||||
}
|
||||
|
||||
func promSeriesToTimeSeries(series promql.Series) *qbtypes.TimeSeries {
|
||||
ts := &qbtypes.TimeSeries{
|
||||
Labels: make([]*qbtypes.Label, 0, len(series.Metric)),
|
||||
Values: make([]*qbtypes.TimeSeriesValue, 0, len(series.Floats)),
|
||||
func toCommonSeries(series promql.Series) v3.Series {
|
||||
commonSeries := v3.Series{
|
||||
Labels: make(map[string]string),
|
||||
LabelsArray: make([]map[string]string, 0),
|
||||
Points: make([]v3.Point, 0),
|
||||
}
|
||||
|
||||
for _, lbl := range series.Metric {
|
||||
ts.Labels = append(ts.Labels, &qbtypes.Label{
|
||||
Key: telemetrytypes.TelemetryFieldKey{Name: lbl.Name},
|
||||
Value: lbl.Value,
|
||||
commonSeries.Labels[lbl.Name] = lbl.Value
|
||||
commonSeries.LabelsArray = append(commonSeries.LabelsArray, map[string]string{
|
||||
lbl.Name: lbl.Value,
|
||||
})
|
||||
}
|
||||
|
||||
for _, f := range series.Floats {
|
||||
ts.Values = append(ts.Values, &qbtypes.TimeSeriesValue{
|
||||
commonSeries.Points = append(commonSeries.Points, v3.Point{
|
||||
Timestamp: f.T,
|
||||
Value: f.F,
|
||||
})
|
||||
}
|
||||
|
||||
return ts
|
||||
return commonSeries
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
qslabels "github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
@@ -48,13 +47,9 @@ func TestPromRuleEval(t *testing.T) {
|
||||
RuleCondition: &ruletypes.RuleCondition{
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{
|
||||
Name: "A",
|
||||
Query: "dummy_query", // This is not used in the test
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {
|
||||
Query: "dummy_query", // This is not used in the test
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -67,7 +62,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp string
|
||||
matchType string
|
||||
target float64
|
||||
expectedAlertSample float64
|
||||
expectedAlertSample v3.Point
|
||||
expectedVectorValues []float64 // Expected values in result vector
|
||||
}{
|
||||
// Test cases for Equals Always
|
||||
@@ -85,7 +80,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "3", // Equals
|
||||
matchType: "2", // Always
|
||||
target: 0.0,
|
||||
expectedAlertSample: 0.0,
|
||||
expectedAlertSample: v3.Point{Value: 0.0},
|
||||
expectedVectorValues: []float64{0.0},
|
||||
},
|
||||
{
|
||||
@@ -150,7 +145,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "3", // Equals
|
||||
matchType: "1", // Once
|
||||
target: 0.0,
|
||||
expectedAlertSample: 0.0,
|
||||
expectedAlertSample: v3.Point{Value: 0.0},
|
||||
expectedVectorValues: []float64{0.0},
|
||||
},
|
||||
{
|
||||
@@ -167,7 +162,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "3", // Equals
|
||||
matchType: "1", // Once
|
||||
target: 0.0,
|
||||
expectedAlertSample: 0.0,
|
||||
expectedAlertSample: v3.Point{Value: 0.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -183,7 +178,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "3", // Equals
|
||||
matchType: "1", // Once
|
||||
target: 0.0,
|
||||
expectedAlertSample: 0.0,
|
||||
expectedAlertSample: v3.Point{Value: 0.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -216,7 +211,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "1", // Greater Than
|
||||
matchType: "2", // Always
|
||||
target: 1.5,
|
||||
expectedAlertSample: 2.0,
|
||||
expectedAlertSample: v3.Point{Value: 2.0},
|
||||
expectedVectorValues: []float64{2.0},
|
||||
},
|
||||
{
|
||||
@@ -233,7 +228,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "1", // Above
|
||||
matchType: "2", // Always
|
||||
target: 2.0,
|
||||
expectedAlertSample: 3.0,
|
||||
expectedAlertSample: v3.Point{Value: 3.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -249,7 +244,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "2", // Below
|
||||
matchType: "2", // Always
|
||||
target: 13.0,
|
||||
expectedAlertSample: 12.0,
|
||||
expectedAlertSample: v3.Point{Value: 12.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -281,7 +276,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "1", // Greater Than
|
||||
matchType: "1", // Once
|
||||
target: 4.5,
|
||||
expectedAlertSample: 10.0,
|
||||
expectedAlertSample: v3.Point{Value: 10.0},
|
||||
expectedVectorValues: []float64{10.0},
|
||||
},
|
||||
{
|
||||
@@ -344,7 +339,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "4", // Not Equals
|
||||
matchType: "2", // Always
|
||||
target: 0.0,
|
||||
expectedAlertSample: 1.0,
|
||||
expectedAlertSample: v3.Point{Value: 1.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -376,7 +371,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "4", // Not Equals
|
||||
matchType: "1", // Once
|
||||
target: 0.0,
|
||||
expectedAlertSample: 1.0,
|
||||
expectedAlertSample: v3.Point{Value: 1.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -407,7 +402,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "4", // Not Equals
|
||||
matchType: "1", // Once
|
||||
target: 0.0,
|
||||
expectedAlertSample: 1.0,
|
||||
expectedAlertSample: v3.Point{Value: 1.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -423,7 +418,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "4", // Not Equals
|
||||
matchType: "1", // Once
|
||||
target: 0.0,
|
||||
expectedAlertSample: 1.0,
|
||||
expectedAlertSample: v3.Point{Value: 1.0},
|
||||
},
|
||||
// Test cases for Less Than Always
|
||||
{
|
||||
@@ -440,7 +435,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "2", // Less Than
|
||||
matchType: "2", // Always
|
||||
target: 4,
|
||||
expectedAlertSample: 1.5,
|
||||
expectedAlertSample: v3.Point{Value: 1.5},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -472,7 +467,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "2", // Less Than
|
||||
matchType: "1", // Once
|
||||
target: 4,
|
||||
expectedAlertSample: 2.5,
|
||||
expectedAlertSample: v3.Point{Value: 2.5},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -504,7 +499,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "3", // Equals
|
||||
matchType: "3", // OnAverage
|
||||
target: 6.0,
|
||||
expectedAlertSample: 6.0,
|
||||
expectedAlertSample: v3.Point{Value: 6.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -535,7 +530,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "4", // Not Equals
|
||||
matchType: "3", // OnAverage
|
||||
target: 4.5,
|
||||
expectedAlertSample: 6.0,
|
||||
expectedAlertSample: v3.Point{Value: 6.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -566,7 +561,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "1", // Greater Than
|
||||
matchType: "3", // OnAverage
|
||||
target: 4.5,
|
||||
expectedAlertSample: 6.0,
|
||||
expectedAlertSample: v3.Point{Value: 6.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -582,7 +577,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "2", // Less Than
|
||||
matchType: "3", // OnAverage
|
||||
target: 12.0,
|
||||
expectedAlertSample: 6.0,
|
||||
expectedAlertSample: v3.Point{Value: 6.0},
|
||||
},
|
||||
// Test cases for InTotal
|
||||
{
|
||||
@@ -599,7 +594,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "3", // Equals
|
||||
matchType: "4", // InTotal
|
||||
target: 30.0,
|
||||
expectedAlertSample: 30.0,
|
||||
expectedAlertSample: v3.Point{Value: 30.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -626,7 +621,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "4", // Not Equals
|
||||
matchType: "4", // InTotal
|
||||
target: 9.0,
|
||||
expectedAlertSample: 10.0,
|
||||
expectedAlertSample: v3.Point{Value: 10.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -650,7 +645,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "1", // Greater Than
|
||||
matchType: "4", // InTotal
|
||||
target: 10.0,
|
||||
expectedAlertSample: 20.0,
|
||||
expectedAlertSample: v3.Point{Value: 20.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -675,7 +670,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
compareOp: "2", // Less Than
|
||||
matchType: "4", // InTotal
|
||||
target: 30.0,
|
||||
expectedAlertSample: 20.0,
|
||||
expectedAlertSample: v3.Point{Value: 20.0},
|
||||
},
|
||||
{
|
||||
values: pql.Series{
|
||||
@@ -713,7 +708,7 @@ func TestPromRuleEval(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
resultVectors, err := rule.Threshold.Eval(*promSeriesToTimeSeries(c.values), rule.Unit(), ruletypes.EvalData{})
|
||||
resultVectors, err := rule.Threshold.Eval(toCommonSeries(c.values), rule.Unit(), ruletypes.EvalData{})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Compare full result vector with expected vector
|
||||
@@ -729,12 +724,12 @@ func TestPromRuleEval(t *testing.T) {
|
||||
if len(resultVectors) > 0 {
|
||||
found := false
|
||||
for _, sample := range resultVectors {
|
||||
if sample.V == c.expectedAlertSample {
|
||||
if sample.V == c.expectedAlertSample.Value {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
assert.True(t, found, "Expected alert sample value %.2f not found in result vectors for case %d. Got values: %v", c.expectedAlertSample, idx, actualValues)
|
||||
assert.True(t, found, "Expected alert sample value %.2f not found in result vectors for case %d. Got values: %v", c.expectedAlertSample.Value, idx, actualValues)
|
||||
}
|
||||
} else {
|
||||
assert.Empty(t, resultVectors, "Expected no alert but got result vectors for case %d", idx)
|
||||
@@ -759,13 +754,9 @@ func TestPromRuleUnitCombinations(t *testing.T) {
|
||||
RuleCondition: &ruletypes.RuleCondition{
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{
|
||||
Name: "A",
|
||||
Query: "test_metric",
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {
|
||||
Query: "test_metric",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1022,13 +1013,9 @@ func _Enable_this_after_9146_issue_fix_is_merged_TestPromRuleNoData(t *testing.T
|
||||
RuleCondition: &ruletypes.RuleCondition{
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{
|
||||
Name: "A",
|
||||
Query: "test_metric",
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {
|
||||
Query: "test_metric",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1137,13 +1124,9 @@ func TestMultipleThresholdPromRule(t *testing.T) {
|
||||
RuleCondition: &ruletypes.RuleCondition{
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{
|
||||
Name: "A",
|
||||
Query: "test_metric",
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {
|
||||
Query: "test_metric",
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -1378,11 +1361,8 @@ func TestPromRule_NoData(t *testing.T) {
|
||||
MatchType: ruletypes.AtleastOnce,
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{Name: "A", Query: "test_metric"},
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {Query: "test_metric"},
|
||||
},
|
||||
},
|
||||
Thresholds: &ruletypes.RuleThresholdData{
|
||||
@@ -1506,11 +1486,8 @@ func TestPromRule_NoData_AbsentFor(t *testing.T) {
|
||||
Target: &target,
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{Name: "A", Query: "test_metric"},
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {Query: "test_metric"},
|
||||
},
|
||||
},
|
||||
Thresholds: &ruletypes.RuleThresholdData{
|
||||
@@ -1658,11 +1635,8 @@ func TestPromRuleEval_RequireMinPoints(t *testing.T) {
|
||||
MatchType: ruletypes.AtleastOnce,
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: v3.QueryTypePromQL,
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypePromQL,
|
||||
Spec: qbtypes.PromQuery{Name: "A", Query: "test_metric"},
|
||||
},
|
||||
PromQueries: map[string]*v3.PromQuery{
|
||||
"A": {Query: "test_metric"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -1,24 +1,38 @@
|
||||
package rules
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/contextlinks"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/common"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/postprocess"
|
||||
"github.com/SigNoz/signoz/pkg/transition"
|
||||
"github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/querier"
|
||||
querierV2 "github.com/SigNoz/signoz/pkg/query-service/app/querier/v2"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/queryBuilder"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/interfaces"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||
querytemplate "github.com/SigNoz/signoz/pkg/query-service/utils/queryTemplate"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/times"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/timestamp"
|
||||
|
||||
logsv3 "github.com/SigNoz/signoz/pkg/query-service/app/logs/v3"
|
||||
tracesV4 "github.com/SigNoz/signoz/pkg/query-service/app/traces/v4"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/formatter"
|
||||
|
||||
querierV5 "github.com/SigNoz/signoz/pkg/querier"
|
||||
@@ -28,9 +42,23 @@ import (
|
||||
|
||||
type ThresholdRule struct {
|
||||
*BaseRule
|
||||
// Ever since we introduced the new metrics query builder, the version is "v4"
|
||||
// for all the rules
|
||||
// if the version is "v3", then we use the old querier
|
||||
// if the version is "v4", then we use the new querierV2
|
||||
version string
|
||||
|
||||
// querierV5 is the query builder v5 querier used for all alert rule evaluation
|
||||
// querier is used for alerts created before the introduction of new metrics query builder
|
||||
querier interfaces.Querier
|
||||
// querierV2 is used for alerts created after the introduction of new metrics query builder
|
||||
querierV2 interfaces.Querier
|
||||
|
||||
// querierV5 is used for alerts migrated after the introduction of new query builder
|
||||
querierV5 querierV5.Querier
|
||||
|
||||
// used for attribute metadata enrichment for logs and traces
|
||||
logsKeys map[string]v3.AttributeKey
|
||||
spansKeys map[string]v3.AttributeKey
|
||||
}
|
||||
|
||||
var _ Rule = (*ThresholdRule)(nil)
|
||||
@@ -54,10 +82,25 @@ func NewThresholdRule(
|
||||
}
|
||||
|
||||
t := ThresholdRule{
|
||||
BaseRule: baseRule,
|
||||
querierV5: querierV5,
|
||||
BaseRule: baseRule,
|
||||
version: p.Version,
|
||||
}
|
||||
|
||||
querierOption := querier.QuerierOptions{
|
||||
Reader: reader,
|
||||
Cache: nil,
|
||||
KeyGenerator: queryBuilder.NewKeyGenerator(),
|
||||
}
|
||||
|
||||
querierOptsV2 := querierV2.QuerierOptions{
|
||||
Reader: reader,
|
||||
Cache: nil,
|
||||
KeyGenerator: queryBuilder.NewKeyGenerator(),
|
||||
}
|
||||
|
||||
t.querier = querier.NewQuerier(querierOption)
|
||||
t.querierV2 = querierV2.NewQuerier(querierOptsV2)
|
||||
t.querierV5 = querierV5
|
||||
t.reader = reader
|
||||
return &t, nil
|
||||
}
|
||||
@@ -77,9 +120,169 @@ func (r *ThresholdRule) Type() ruletypes.RuleType {
|
||||
return ruletypes.RuleTypeThreshold
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) prepareQueryRange(ctx context.Context, ts time.Time) (*qbtypes.QueryRangeRequest, error) {
|
||||
func (r *ThresholdRule) prepareQueryRange(ctx context.Context, ts time.Time) (*v3.QueryRangeParamsV3, error) {
|
||||
r.logger.InfoContext(
|
||||
ctx, "prepare query range request", "ts", ts.UnixMilli(), "eval_window", r.evalWindow.Milliseconds(), "eval_delay", r.evalDelay.Milliseconds(),
|
||||
ctx, "prepare query range request v4", "ts", ts.UnixMilli(), "eval_window", r.evalWindow.Milliseconds(), "eval_delay", r.evalDelay.Milliseconds(),
|
||||
)
|
||||
|
||||
startTs, endTs := r.Timestamps(ts)
|
||||
start, end := startTs.UnixMilli(), endTs.UnixMilli()
|
||||
|
||||
if r.ruleCondition.QueryType() == v3.QueryTypeClickHouseSQL {
|
||||
params := &v3.QueryRangeParamsV3{
|
||||
Start: start,
|
||||
End: end,
|
||||
Step: int64(math.Max(float64(common.MinAllowedStepInterval(start, end)), 60)),
|
||||
CompositeQuery: &v3.CompositeQuery{
|
||||
QueryType: r.ruleCondition.CompositeQuery.QueryType,
|
||||
PanelType: r.ruleCondition.CompositeQuery.PanelType,
|
||||
BuilderQueries: make(map[string]*v3.BuilderQuery),
|
||||
ClickHouseQueries: make(map[string]*v3.ClickHouseQuery),
|
||||
PromQueries: make(map[string]*v3.PromQuery),
|
||||
Unit: r.ruleCondition.CompositeQuery.Unit,
|
||||
},
|
||||
Variables: make(map[string]interface{}),
|
||||
NoCache: true,
|
||||
}
|
||||
querytemplate.AssignReservedVarsV3(params)
|
||||
for name, chQuery := range r.ruleCondition.CompositeQuery.ClickHouseQueries {
|
||||
if chQuery.Disabled {
|
||||
continue
|
||||
}
|
||||
tmpl := template.New("clickhouse-query")
|
||||
tmpl, err := tmpl.Parse(chQuery.Query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var query bytes.Buffer
|
||||
err = tmpl.Execute(&query, params.Variables)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
params.CompositeQuery.ClickHouseQueries[name] = &v3.ClickHouseQuery{
|
||||
Query: query.String(),
|
||||
Disabled: chQuery.Disabled,
|
||||
Legend: chQuery.Legend,
|
||||
}
|
||||
}
|
||||
return params, nil
|
||||
}
|
||||
|
||||
if r.ruleCondition.CompositeQuery != nil && r.ruleCondition.CompositeQuery.BuilderQueries != nil {
|
||||
for _, q := range r.ruleCondition.CompositeQuery.BuilderQueries {
|
||||
// If the step interval is less than the minimum allowed step interval, set it to the minimum allowed step interval
|
||||
if minStep := common.MinAllowedStepInterval(start, end); q.StepInterval < minStep {
|
||||
q.StepInterval = minStep
|
||||
}
|
||||
|
||||
q.SetShiftByFromFunc()
|
||||
|
||||
if q.DataSource == v3.DataSourceMetrics {
|
||||
// if the time range is greater than 1 day, and less than 1 week set the step interval to be multiple of 5 minutes
|
||||
// if the time range is greater than 1 week, set the step interval to be multiple of 30 mins
|
||||
if end-start >= 24*time.Hour.Milliseconds() && end-start < 7*24*time.Hour.Milliseconds() {
|
||||
q.StepInterval = int64(math.Round(float64(q.StepInterval)/300)) * 300
|
||||
} else if end-start >= 7*24*time.Hour.Milliseconds() {
|
||||
q.StepInterval = int64(math.Round(float64(q.StepInterval)/1800)) * 1800
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if r.ruleCondition.CompositeQuery.PanelType != v3.PanelTypeGraph {
|
||||
r.ruleCondition.CompositeQuery.PanelType = v3.PanelTypeGraph
|
||||
}
|
||||
|
||||
// default mode
|
||||
return &v3.QueryRangeParamsV3{
|
||||
Start: start,
|
||||
End: end,
|
||||
Step: int64(math.Max(float64(common.MinAllowedStepInterval(start, end)), 60)),
|
||||
CompositeQuery: r.ruleCondition.CompositeQuery,
|
||||
Variables: make(map[string]interface{}),
|
||||
NoCache: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) prepareLinksToLogs(ctx context.Context, ts time.Time, lbls labels.Labels) string {
|
||||
if r.version == "v5" {
|
||||
return r.prepareLinksToLogsV5(ctx, ts, lbls)
|
||||
}
|
||||
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
|
||||
qr, err := r.prepareQueryRange(ctx, ts)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
start := time.UnixMilli(qr.Start)
|
||||
end := time.UnixMilli(qr.End)
|
||||
|
||||
// TODO(srikanthccv): handle formula queries
|
||||
if selectedQuery < "A" || selectedQuery > "Z" {
|
||||
return ""
|
||||
}
|
||||
|
||||
q := r.ruleCondition.CompositeQuery.BuilderQueries[selectedQuery]
|
||||
if q == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
if q.DataSource != v3.DataSourceLogs {
|
||||
return ""
|
||||
}
|
||||
|
||||
queryFilter := []v3.FilterItem{}
|
||||
if q.Filters != nil {
|
||||
queryFilter = q.Filters.Items
|
||||
}
|
||||
|
||||
filterItems := contextlinks.PrepareFilters(lbls.Map(), queryFilter, q.GroupBy, r.logsKeys)
|
||||
|
||||
return contextlinks.PrepareLinksToLogs(start, end, filterItems)
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) prepareLinksToTraces(ctx context.Context, ts time.Time, lbls labels.Labels) string {
|
||||
if r.version == "v5" {
|
||||
return r.prepareLinksToTracesV5(ctx, ts, lbls)
|
||||
}
|
||||
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
|
||||
qr, err := r.prepareQueryRange(ctx, ts)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
start := time.UnixMilli(qr.Start)
|
||||
end := time.UnixMilli(qr.End)
|
||||
|
||||
// TODO(srikanthccv): handle formula queries
|
||||
if selectedQuery < "A" || selectedQuery > "Z" {
|
||||
return ""
|
||||
}
|
||||
|
||||
q := r.ruleCondition.CompositeQuery.BuilderQueries[selectedQuery]
|
||||
if q == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
if q.DataSource != v3.DataSourceTraces {
|
||||
return ""
|
||||
}
|
||||
|
||||
queryFilter := []v3.FilterItem{}
|
||||
if q.Filters != nil {
|
||||
queryFilter = q.Filters.Items
|
||||
}
|
||||
|
||||
filterItems := contextlinks.PrepareFilters(lbls.Map(), queryFilter, q.GroupBy, r.spansKeys)
|
||||
|
||||
return contextlinks.PrepareLinksToTraces(start, end, filterItems)
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) prepareQueryRangeV5(ctx context.Context, ts time.Time) (*qbtypes.QueryRangeRequest, error) {
|
||||
r.logger.InfoContext(
|
||||
ctx, "prepare query range request v5", "ts", ts.UnixMilli(), "eval_window", r.evalWindow.Milliseconds(), "eval_delay", r.evalDelay.Milliseconds(),
|
||||
)
|
||||
|
||||
startTs, endTs := r.Timestamps(ts)
|
||||
@@ -99,10 +302,10 @@ func (r *ThresholdRule) prepareQueryRange(ctx context.Context, ts time.Time) (*q
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) prepareLinksToLogs(ctx context.Context, ts time.Time, lbls labels.Labels) string {
|
||||
func (r *ThresholdRule) prepareLinksToLogsV5(ctx context.Context, ts time.Time, lbls labels.Labels) string {
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
|
||||
qr, err := r.prepareQueryRange(ctx, ts)
|
||||
qr, err := r.prepareQueryRangeV5(ctx, ts)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -139,10 +342,10 @@ func (r *ThresholdRule) prepareLinksToLogs(ctx context.Context, ts time.Time, lb
|
||||
return contextlinks.PrepareLinksToLogsV5(start, end, whereClause)
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) prepareLinksToTraces(ctx context.Context, ts time.Time, lbls labels.Labels) string {
|
||||
func (r *ThresholdRule) prepareLinksToTracesV5(ctx context.Context, ts time.Time, lbls labels.Labels) string {
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
|
||||
qr, err := r.prepareQueryRange(ctx, ts)
|
||||
qr, err := r.prepareQueryRangeV5(ctx, ts)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -188,6 +391,115 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = r.PopulateTemporality(ctx, orgID, params)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("internal error while setting temporality")
|
||||
}
|
||||
|
||||
if params.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
||||
hasLogsQuery := false
|
||||
hasTracesQuery := false
|
||||
for _, query := range params.CompositeQuery.BuilderQueries {
|
||||
if query.DataSource == v3.DataSourceLogs {
|
||||
hasLogsQuery = true
|
||||
}
|
||||
if query.DataSource == v3.DataSourceTraces {
|
||||
hasTracesQuery = true
|
||||
}
|
||||
}
|
||||
|
||||
if hasLogsQuery {
|
||||
// check if any enrichment is required for logs if yes then enrich them
|
||||
if logsv3.EnrichmentRequired(params) {
|
||||
logsFields, apiErr := r.reader.GetLogFieldsFromNames(ctx, logsv3.GetFieldNames(params.CompositeQuery))
|
||||
if apiErr != nil {
|
||||
return nil, apiErr.ToError()
|
||||
}
|
||||
logsKeys := model.GetLogFieldsV3(ctx, params, logsFields)
|
||||
r.logsKeys = logsKeys
|
||||
logsv3.Enrich(params, logsKeys)
|
||||
}
|
||||
}
|
||||
|
||||
if hasTracesQuery {
|
||||
spanKeys, err := r.reader.GetSpanAttributeKeysByNames(ctx, logsv3.GetFieldNames(params.CompositeQuery))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.spansKeys = spanKeys
|
||||
tracesV4.Enrich(params, spanKeys)
|
||||
}
|
||||
}
|
||||
|
||||
var results []*v3.Result
|
||||
var queryErrors map[string]error
|
||||
|
||||
if r.version == "v4" {
|
||||
results, queryErrors, err = r.querierV2.QueryRange(ctx, orgID, params)
|
||||
} else {
|
||||
results, queryErrors, err = r.querier.QueryRange(ctx, orgID, params)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
r.logger.ErrorContext(ctx, "failed to get alert query range result", "rule_name", r.Name(), "error", err, "query_errors", queryErrors)
|
||||
return nil, fmt.Errorf("internal error while querying")
|
||||
}
|
||||
|
||||
if params.CompositeQuery.QueryType == v3.QueryTypeBuilder {
|
||||
results, err = postprocess.PostProcessResult(results, params)
|
||||
if err != nil {
|
||||
r.logger.ErrorContext(ctx, "failed to post process result", "rule_name", r.Name(), "error", err)
|
||||
return nil, fmt.Errorf("internal error while post processing")
|
||||
}
|
||||
}
|
||||
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
|
||||
var queryResult *v3.Result
|
||||
for _, res := range results {
|
||||
if res.QueryName == selectedQuery {
|
||||
queryResult = res
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
hasData := queryResult != nil && len(queryResult.Series) > 0
|
||||
if missingDataAlert := r.HandleMissingDataAlert(ctx, ts, hasData); missingDataAlert != nil {
|
||||
return ruletypes.Vector{*missingDataAlert}, nil
|
||||
}
|
||||
|
||||
var resultVector ruletypes.Vector
|
||||
|
||||
if queryResult == nil {
|
||||
r.logger.WarnContext(ctx, "query result is nil", "rule_name", r.Name(), "query_name", selectedQuery)
|
||||
return resultVector, nil
|
||||
}
|
||||
|
||||
for _, series := range queryResult.Series {
|
||||
if !r.Condition().ShouldEval(series) {
|
||||
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||
continue
|
||||
}
|
||||
resultSeries, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{
|
||||
ActiveAlerts: r.ActiveAlertsLabelFP(),
|
||||
SendUnmatched: r.ShouldSendUnmatched(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resultVector = append(resultVector, resultSeries...)
|
||||
}
|
||||
|
||||
return resultVector, nil
|
||||
}
|
||||
|
||||
func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUID, ts time.Time) (ruletypes.Vector, error) {
|
||||
params, err := r.prepareQueryRangeV5(ctx, ts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var results []*v3.Result
|
||||
|
||||
v5Result, err := r.querierV5.QueryRange(ctx, orgID, params)
|
||||
if err != nil {
|
||||
@@ -195,24 +507,26 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID,
|
||||
return nil, fmt.Errorf("internal error while querying")
|
||||
}
|
||||
|
||||
for _, item := range v5Result.Data.Results {
|
||||
if tsData, ok := item.(*qbtypes.TimeSeriesData); ok {
|
||||
results = append(results, transition.ConvertV5TimeSeriesDataToV4Result(tsData))
|
||||
} else {
|
||||
// NOTE: should not happen but just to ensure we don't miss it if it happens for some reason
|
||||
r.logger.WarnContext(ctx, "expected qbtypes.TimeSeriesData but got", "item_type", reflect.TypeOf(item))
|
||||
}
|
||||
}
|
||||
|
||||
selectedQuery := r.GetSelectedQuery()
|
||||
|
||||
var queryResult *qbtypes.TimeSeriesData
|
||||
for _, item := range v5Result.Data.Results {
|
||||
if tsData, ok := item.(*qbtypes.TimeSeriesData); ok && tsData.QueryName == selectedQuery {
|
||||
queryResult = tsData
|
||||
var queryResult *v3.Result
|
||||
for _, res := range results {
|
||||
if res.QueryName == selectedQuery {
|
||||
queryResult = res
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var allSeries []*qbtypes.TimeSeries
|
||||
if queryResult != nil {
|
||||
for _, bucket := range queryResult.Aggregations {
|
||||
allSeries = append(allSeries, bucket.Series...)
|
||||
}
|
||||
}
|
||||
|
||||
hasData := len(allSeries) > 0
|
||||
hasData := queryResult != nil && len(queryResult.Series) > 0
|
||||
if missingDataAlert := r.HandleMissingDataAlert(ctx, ts, hasData); missingDataAlert != nil {
|
||||
return ruletypes.Vector{*missingDataAlert}, nil
|
||||
}
|
||||
@@ -225,7 +539,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID,
|
||||
}
|
||||
|
||||
// Filter out new series if newGroupEvalDelay is configured
|
||||
seriesToProcess := allSeries
|
||||
seriesToProcess := queryResult.Series
|
||||
if r.ShouldSkipNewGroups() {
|
||||
filteredSeries, filterErr := r.BaseRule.FilterNewSeries(ctx, ts, seriesToProcess)
|
||||
// In case of error we log the error and continue with the original series
|
||||
@@ -238,7 +552,7 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID,
|
||||
|
||||
for _, series := range seriesToProcess {
|
||||
if !r.Condition().ShouldEval(series) {
|
||||
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Values), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||
r.logger.InfoContext(ctx, "not enough data points to evaluate series, skipping", "ruleid", r.ID(), "numPoints", len(series.Points), "requiredPoints", r.Condition().RequiredNumPoints)
|
||||
continue
|
||||
}
|
||||
resultSeries, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{
|
||||
@@ -259,7 +573,16 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (int, error) {
|
||||
|
||||
valueFormatter := formatter.FromUnit(r.Unit())
|
||||
|
||||
res, err := r.buildAndRunQuery(ctx, r.orgID, ts)
|
||||
var res ruletypes.Vector
|
||||
var err error
|
||||
|
||||
if r.version == "v5" {
|
||||
r.logger.InfoContext(ctx, "running v5 query")
|
||||
res, err = r.buildAndRunQueryV5(ctx, r.orgID, ts)
|
||||
} else {
|
||||
r.logger.InfoContext(ctx, "running v4 query")
|
||||
res, err = r.buildAndRunQuery(ctx, r.orgID, ts)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -169,7 +169,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewAddAnonymousPublicDashboardTransactionFactory(sqlstore),
|
||||
sqlmigration.NewAddRootUserFactory(sqlstore, sqlschema),
|
||||
sqlmigration.NewAddUserEmailOrgIDIndexFactory(sqlstore, sqlschema),
|
||||
sqlmigration.NewMigrateRulesV4ToV5Factory(sqlstore, telemetryStore),
|
||||
sqlmigration.NewMigratePipelineFiltersV5Factory(sqlschema),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,194 +0,0 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/transition"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
type migrateRulesV4ToV5 struct {
|
||||
store sqlstore.SQLStore
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewMigrateRulesV4ToV5Factory(
|
||||
store sqlstore.SQLStore,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("migrate_rules_v4_to_v5"),
|
||||
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return &migrateRulesV4ToV5{
|
||||
store: store,
|
||||
telemetryStore: telemetryStore,
|
||||
logger: ps.Logger,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (migration *migrateRulesV4ToV5) Register(migrations *migrate.Migrations) error {
|
||||
if err := migrations.Register(migration.Up, migration.Down); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (migration *migrateRulesV4ToV5) getLogDuplicateKeys(ctx context.Context) ([]string, error) {
|
||||
query := `
|
||||
SELECT name
|
||||
FROM (
|
||||
SELECT DISTINCT name FROM signoz_logs.distributed_logs_attribute_keys
|
||||
INTERSECT
|
||||
SELECT DISTINCT name FROM signoz_logs.distributed_logs_resource_keys
|
||||
)
|
||||
ORDER BY name
|
||||
`
|
||||
|
||||
rows, err := migration.telemetryStore.ClickhouseDB().Query(ctx, query)
|
||||
if err != nil {
|
||||
migration.logger.WarnContext(ctx, "failed to query log duplicate keys", "error", err)
|
||||
return nil, nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var key string
|
||||
if err := rows.Scan(&key); err != nil {
|
||||
migration.logger.WarnContext(ctx, "failed to scan log duplicate key", "error", err)
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (migration *migrateRulesV4ToV5) getTraceDuplicateKeys(ctx context.Context) ([]string, error) {
|
||||
query := `
|
||||
SELECT tagKey
|
||||
FROM signoz_traces.distributed_span_attributes_keys
|
||||
WHERE tagType IN ('tag', 'resource')
|
||||
GROUP BY tagKey
|
||||
HAVING COUNT(DISTINCT tagType) > 1
|
||||
ORDER BY tagKey
|
||||
`
|
||||
|
||||
rows, err := migration.telemetryStore.ClickhouseDB().Query(ctx, query)
|
||||
if err != nil {
|
||||
migration.logger.WarnContext(ctx, "failed to query trace duplicate keys", "error", err)
|
||||
return nil, nil
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var key string
|
||||
if err := rows.Scan(&key); err != nil {
|
||||
migration.logger.WarnContext(ctx, "failed to scan trace duplicate key", "error", err)
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func (migration *migrateRulesV4ToV5) Up(ctx context.Context, db *bun.DB) error {
|
||||
logsKeys, err := migration.getLogDuplicateKeys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tracesKeys, err := migration.getTraceDuplicateKeys(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
var rules []struct {
|
||||
ID string `bun:"id"`
|
||||
Data map[string]any `bun:"data"`
|
||||
}
|
||||
|
||||
err = tx.NewSelect().
|
||||
Table("rule").
|
||||
Column("id", "data").
|
||||
Scan(ctx, &rules)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
alertsMigrator := transition.NewAlertMigrateV5(migration.logger, logsKeys, tracesKeys)
|
||||
|
||||
for _, rule := range rules {
|
||||
version, _ := rule.Data["version"].(string)
|
||||
if version == "v5" {
|
||||
continue
|
||||
}
|
||||
|
||||
migration.logger.InfoContext(ctx, "migrating rule v4 to v5", "rule_id", rule.ID, "current_version", version)
|
||||
|
||||
// Check if the queries envelope already exists and is non-empty
|
||||
hasQueriesEnvelope := false
|
||||
if condition, ok := rule.Data["condition"].(map[string]any); ok {
|
||||
if compositeQuery, ok := condition["compositeQuery"].(map[string]any); ok {
|
||||
if queries, ok := compositeQuery["queries"].([]any); ok && len(queries) > 0 {
|
||||
hasQueriesEnvelope = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if hasQueriesEnvelope {
|
||||
// Case 2: Already has queries envelope, just bump version
|
||||
migration.logger.InfoContext(ctx, "rule already has queries envelope, bumping version", "rule_id", rule.ID)
|
||||
rule.Data["version"] = "v5"
|
||||
} else {
|
||||
// Case 1: Old format, run full migration
|
||||
migration.logger.InfoContext(ctx, "rule has old format, running full migration", "rule_id", rule.ID)
|
||||
alertsMigrator.Migrate(ctx, rule.Data)
|
||||
// Force version to v5 regardless of Migrate return value
|
||||
rule.Data["version"] = "v5"
|
||||
}
|
||||
|
||||
dataJSON, err := json.Marshal(rule.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = tx.NewUpdate().
|
||||
Table("rule").
|
||||
Set("data = ?", string(dataJSON)).
|
||||
Where("id = ?", rule.ID).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (migration *migrateRulesV4ToV5) Down(ctx context.Context, db *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
182
pkg/sqlmigration/066_migrate_pipeline_filters_v5.go
Normal file
182
pkg/sqlmigration/066_migrate_pipeline_filters_v5.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/queryBuilderToExpr"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/transition"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
type migratePipelineFiltersV5 struct {
|
||||
sqlschema sqlschema.SQLSchema
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
func NewMigratePipelineFiltersV5Factory(
|
||||
sqlschema sqlschema.SQLSchema,
|
||||
) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("migrate_pipeline_filters_v5"),
|
||||
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return newMigratePipelineFiltersV5(ctx, c, sqlschema, ps.Logger)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func newMigratePipelineFiltersV5(
|
||||
_ context.Context,
|
||||
_ Config,
|
||||
sqlschema sqlschema.SQLSchema,
|
||||
logger *slog.Logger,
|
||||
) (SQLMigration, error) {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
|
||||
return &migratePipelineFiltersV5{
|
||||
sqlschema: sqlschema,
|
||||
logger: logger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (migration *migratePipelineFiltersV5) Register(migrations *migrate.Migrations) error {
|
||||
if err := migrations.Register(migration.Up, migration.Down); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// pipelineFilterRow is used only during migration to read filter_deprecated and write to filter.
|
||||
type pipelineFilterRow struct {
|
||||
ID string `bun:"id,pk,type:text"`
|
||||
FilterDeprecated string `bun:"filter_deprecated,type:text,notnull"`
|
||||
}
|
||||
|
||||
func (migration *migratePipelineFiltersV5) Up(ctx context.Context, db *bun.DB) error {
|
||||
table, uniqueConstraints, err := migration.sqlschema.GetTable(ctx, "pipelines")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
// 1. Rename existing filter column to filter_deprecated
|
||||
for _, sql := range migration.sqlschema.Operator().RenameColumn(table, &sqlschema.Column{Name: "filter"}, "filter_deprecated") {
|
||||
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Add new filter column (v5); existing rows get default ''
|
||||
for _, sql := range migration.sqlschema.Operator().AddColumn(table, uniqueConstraints, &sqlschema.Column{
|
||||
Name: "filter",
|
||||
DataType: sqlschema.DataTypeText,
|
||||
Nullable: false,
|
||||
}, "") {
|
||||
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Copy v5 filter data: read from filter_deprecated, migrate to v5 expression, write to filter
|
||||
var rows []pipelineFilterRow
|
||||
if err := tx.NewSelect().
|
||||
Table("pipelines").
|
||||
Column("id", "filter_deprecated").
|
||||
Scan(ctx, &rows); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
raw := strings.TrimSpace(r.FilterDeprecated)
|
||||
if raw == "" {
|
||||
return errors.NewInternalf(errors.CodeInternal, "filter_deprecated is empty")
|
||||
}
|
||||
|
||||
var filterSet v3.FilterSet
|
||||
if err := json.Unmarshal([]byte(raw), &filterSet); err != nil {
|
||||
return err
|
||||
}
|
||||
expr, migrated, err := transition.BuildFilterExpressionFromFilterSet(ctx, migration.logger, "logs", &filterSet)
|
||||
if err != nil || !migrated || strings.TrimSpace(expr) == "" {
|
||||
return err
|
||||
}
|
||||
|
||||
filter := &qbtypes.Filter{Expression: expr}
|
||||
if _, err = queryBuilderToExpr.Parse(filter); err != nil {
|
||||
return err
|
||||
}
|
||||
out, err := json.Marshal(filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := tx.NewUpdate().
|
||||
Table("pipelines").
|
||||
Set("filter = ?", string(out)).
|
||||
Where("id = ?", r.ID).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (migration *migratePipelineFiltersV5) Down(ctx context.Context, db *bun.DB) error {
|
||||
table, _, err := migration.sqlschema.GetTable(ctx, "pipelines")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
// 1. Drop the new filter column
|
||||
for _, sql := range migration.sqlschema.Operator().DropColumn(table, &sqlschema.Column{Name: "filter"}) {
|
||||
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Rename filter_deprecated back to filter
|
||||
for _, sql := range migration.sqlschema.Operator().RenameColumn(table, &sqlschema.Column{Name: "filter_deprecated"}, "filter") {
|
||||
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -118,6 +118,19 @@ func (column *Column) ToUpdateSQL(fmter SQLFormatter, tableName TableName, value
|
||||
return sql
|
||||
}
|
||||
|
||||
func (column *Column) ToRenameSQL(fmter SQLFormatter, tableName TableName, newName ColumnName) []byte {
|
||||
sql := []byte{}
|
||||
|
||||
sql = append(sql, "ALTER TABLE "...)
|
||||
sql = fmter.AppendIdent(sql, string(tableName))
|
||||
sql = append(sql, " RENAME COLUMN "...)
|
||||
sql = fmter.AppendIdent(sql, string(column.Name))
|
||||
sql = append(sql, " TO "...)
|
||||
sql = fmter.AppendIdent(sql, string(newName))
|
||||
|
||||
return sql
|
||||
}
|
||||
|
||||
func (column *Column) ToSetNotNullSQL(fmter SQLFormatter, tableName TableName) []byte {
|
||||
sql := []byte{}
|
||||
|
||||
|
||||
@@ -95,6 +95,19 @@ func (operator *Operator) DropColumn(table *Table, column *Column) [][]byte {
|
||||
return [][]byte{column.ToDropSQL(operator.fmter, table.Name, operator.support.ColumnIfNotExistsExists)}
|
||||
}
|
||||
|
||||
func (operator *Operator) RenameColumn(table *Table, column *Column, newName ColumnName) [][]byte {
|
||||
index := operator.findColumnByName(table, column.Name)
|
||||
// If the column does not exist, we do not need to rename it.
|
||||
if index == -1 {
|
||||
return [][]byte{}
|
||||
}
|
||||
|
||||
sql := column.ToRenameSQL(operator.fmter, table.Name, newName)
|
||||
table.Columns[index].Name = newName
|
||||
|
||||
return [][]byte{sql}
|
||||
}
|
||||
|
||||
func (operator *Operator) DropConstraint(table *Table, uniqueConstraints []*UniqueConstraint, constraint Constraint) [][]byte {
|
||||
// The name of the input constraint is not guaranteed to be the same as the name of the constraint in the database.
|
||||
// So we need to find the constraint in the database and drop it.
|
||||
|
||||
@@ -50,6 +50,9 @@ type SQLOperator interface {
|
||||
// Returns a list of SQL statements to drop a column from a table.
|
||||
DropColumn(*Table, *Column) [][]byte
|
||||
|
||||
// Returns a list of SQL statements to rename a column in a table.
|
||||
RenameColumn(*Table, *Column, ColumnName) [][]byte
|
||||
|
||||
// Returns a list of SQL statements to drop a constraint from a table.
|
||||
DropConstraint(*Table, []*UniqueConstraint, Constraint) [][]byte
|
||||
}
|
||||
|
||||
79
pkg/transition/filter_migrate.go
Normal file
79
pkg/transition/filter_migrate.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package transition
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
)
|
||||
|
||||
// BuildFilterExpressionFromFilterSet converts a v3-style FilterSet JSON
|
||||
// ({"op": "...", "items": [...]}) into a v5-style filter expression string
|
||||
// (for example: "attribute.http.method = 'GET' AND resource.env = 'prod'").
|
||||
//
|
||||
// It reuses migrateCommon.createFilterExpression so that all the existing
|
||||
// semantics around operators, variables, data types, and ambiguity handling
|
||||
// are preserved.
|
||||
//
|
||||
// dataSource determines which ambiguity set to use ("logs", "traces", etc.).
|
||||
// For log pipelines, pass "logs".
|
||||
//
|
||||
// Returns:
|
||||
// - expression: the generated filter expression string
|
||||
// - migrated: true if an expression was generated, false if there was
|
||||
// nothing to migrate (e.g. empty filters)
|
||||
// - err: non-nil only if the input JSON could not be parsed
|
||||
func BuildFilterExpressionFromFilterSet(
|
||||
ctx context.Context,
|
||||
logger *slog.Logger,
|
||||
dataSource string,
|
||||
filterSet *v3.FilterSet,
|
||||
) (expression string, migrated bool, err error) {
|
||||
if filterSet == nil {
|
||||
return "", false, nil
|
||||
}
|
||||
filterJSON, err := json.Marshal(filterSet)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
var filters map[string]any
|
||||
if err := json.Unmarshal([]byte(filterJSON), &filters); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
mc := NewMigrateCommon(logger)
|
||||
// add keys with type into ambiguity set to preserve context in generated expression
|
||||
for _, item := range filterSet.Items {
|
||||
if item.Key.Type == v3.AttributeKeyTypeUnspecified {
|
||||
continue
|
||||
}
|
||||
|
||||
mc.ambiguity[dataSource] = append(mc.ambiguity[dataSource], item.Key.Key)
|
||||
}
|
||||
|
||||
// Shape expected by migrateCommon.createFilterExpression:
|
||||
// queryData["dataSource"] == "logs" | "traces" | "metrics"
|
||||
// queryData["filters"] == map[string]any{"op": "...", "items": [...]}
|
||||
queryData := map[string]any{
|
||||
"dataSource": dataSource,
|
||||
"filters": filters,
|
||||
}
|
||||
|
||||
if !mc.createFilterExpression(ctx, queryData) {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
filterAny, ok := queryData["filter"].(map[string]any)
|
||||
if !ok {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
expr, ok := filterAny["expression"].(string)
|
||||
if !ok || expr == "" {
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
return expr, true, nil
|
||||
}
|
||||
220
pkg/transition/filter_migrate_test.go
Normal file
220
pkg/transition/filter_migrate_test.go
Normal file
@@ -0,0 +1,220 @@
|
||||
package transition
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
)
|
||||
|
||||
func TestBuildFilterExpressionFromFilterSet(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
logger := slog.Default()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
dataSource string
|
||||
filterSet *v3.FilterSet
|
||||
wantExpr string
|
||||
wantMigrated bool
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "empty filter JSON",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{},
|
||||
wantExpr: "",
|
||||
wantMigrated: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "empty items array",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
|
||||
wantExpr: "",
|
||||
wantMigrated: false,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "multiple filter items with AND operator",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "GET",
|
||||
},
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "environment", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantExpr: `(attribute.http.method = 'GET' AND resource.environment = 'prod')`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "multiple filter items with OR operator",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "OR",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "error",
|
||||
},
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "warn",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantExpr: `(attribute.level = 'error' OR attribute.level = 'warn')`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "in operator with array value",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorIn,
|
||||
Value: []string{"api", "web", "worker"},
|
||||
}},
|
||||
},
|
||||
wantExpr: `attribute.service.name IN ['api', 'web', 'worker']`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "contains operator",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{Key: "message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorContains,
|
||||
Value: "error",
|
||||
}},
|
||||
},
|
||||
wantExpr: `attribute.message CONTAINS 'error'`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "not exists operator",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{Key: "trace.id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorNotExists,
|
||||
Value: nil,
|
||||
}},
|
||||
},
|
||||
wantExpr: `attribute.trace.id NOT EXISTS`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "regex operator",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{Key: "body"},
|
||||
Operator: v3.FilterOperatorRegex,
|
||||
Value: ".*",
|
||||
}},
|
||||
},
|
||||
wantExpr: `body REGEXP '.*'`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "has operator",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{Key: "tags", DataType: v3.AttributeKeyDataTypeArrayString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorHas,
|
||||
Value: "production",
|
||||
}},
|
||||
},
|
||||
wantExpr: `has(attribute.tags, 'production')`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "complex filter with multiple operators",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "POST",
|
||||
},
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "http.status_code", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag},
|
||||
Operator: v3.FilterOperatorGreaterThanOrEq,
|
||||
Value: float64(400),
|
||||
},
|
||||
{
|
||||
Key: v3.AttributeKey{Key: "resource.env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "prod",
|
||||
},
|
||||
},
|
||||
},
|
||||
wantExpr: `(attribute.http.method = 'POST' AND attribute.http.status_code >= 400 AND resource.resource.env = 'prod')`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "filter with resource type (non-ambiguous key)",
|
||||
dataSource: "logs",
|
||||
filterSet: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "frontend",
|
||||
}},
|
||||
},
|
||||
wantExpr: `resource.service.name = 'frontend'`,
|
||||
wantMigrated: true,
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
expr, migrated, err := BuildFilterExpressionFromFilterSet(ctx, logger, tt.dataSource, tt.filterSet)
|
||||
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("BuildFilterExpressionFromFilterSet() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
|
||||
if migrated != tt.wantMigrated {
|
||||
t.Errorf("BuildFilterExpressionFromFilterSet() migrated = %v, want %v", migrated, tt.wantMigrated)
|
||||
}
|
||||
|
||||
if expr != tt.wantExpr {
|
||||
t.Errorf("BuildFilterExpressionFromFilterSet() expression = %v, want %v", expr, tt.wantExpr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,8 @@ type migrateCommon struct {
|
||||
|
||||
func NewMigrateCommon(logger *slog.Logger) *migrateCommon {
|
||||
return &migrateCommon{
|
||||
logger: logger,
|
||||
logger: logger,
|
||||
ambiguity: make(map[string][]string),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
102
pkg/transition/v5_to_v4.go
Normal file
102
pkg/transition/v5_to_v4.go
Normal file
@@ -0,0 +1,102 @@
|
||||
package transition
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
)
|
||||
|
||||
// ConvertV5TimeSeriesDataToV4Result converts v5 TimeSeriesData to v4 Result
|
||||
func ConvertV5TimeSeriesDataToV4Result(v5Data *qbtypes.TimeSeriesData) *v3.Result {
|
||||
if v5Data == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
result := &v3.Result{
|
||||
QueryName: v5Data.QueryName,
|
||||
Series: make([]*v3.Series, 0),
|
||||
}
|
||||
|
||||
toV4Series := func(ts *qbtypes.TimeSeries) *v3.Series {
|
||||
series := &v3.Series{
|
||||
Labels: make(map[string]string),
|
||||
LabelsArray: make([]map[string]string, 0),
|
||||
Points: make([]v3.Point, 0, len(ts.Values)),
|
||||
}
|
||||
|
||||
for _, label := range ts.Labels {
|
||||
valueStr := fmt.Sprintf("%v", label.Value)
|
||||
series.Labels[label.Key.Name] = valueStr
|
||||
}
|
||||
|
||||
if len(series.Labels) > 0 {
|
||||
series.LabelsArray = append(series.LabelsArray, series.Labels)
|
||||
}
|
||||
|
||||
for _, tsValue := range ts.Values {
|
||||
if tsValue.Partial {
|
||||
continue
|
||||
}
|
||||
|
||||
point := v3.Point{
|
||||
Timestamp: tsValue.Timestamp,
|
||||
Value: tsValue.Value,
|
||||
}
|
||||
series.Points = append(series.Points, point)
|
||||
}
|
||||
return series
|
||||
}
|
||||
|
||||
for _, aggBucket := range v5Data.Aggregations {
|
||||
for _, ts := range aggBucket.Series {
|
||||
result.Series = append(result.Series, toV4Series(ts))
|
||||
}
|
||||
|
||||
if len(aggBucket.AnomalyScores) != 0 {
|
||||
result.AnomalyScores = make([]*v3.Series, 0)
|
||||
for _, ts := range aggBucket.AnomalyScores {
|
||||
result.AnomalyScores = append(result.AnomalyScores, toV4Series(ts))
|
||||
}
|
||||
}
|
||||
|
||||
if len(aggBucket.PredictedSeries) != 0 {
|
||||
result.PredictedSeries = make([]*v3.Series, 0)
|
||||
for _, ts := range aggBucket.PredictedSeries {
|
||||
result.PredictedSeries = append(result.PredictedSeries, toV4Series(ts))
|
||||
}
|
||||
}
|
||||
|
||||
if len(aggBucket.LowerBoundSeries) != 0 {
|
||||
result.LowerBoundSeries = make([]*v3.Series, 0)
|
||||
for _, ts := range aggBucket.LowerBoundSeries {
|
||||
result.LowerBoundSeries = append(result.LowerBoundSeries, toV4Series(ts))
|
||||
}
|
||||
}
|
||||
|
||||
if len(aggBucket.UpperBoundSeries) != 0 {
|
||||
result.UpperBoundSeries = make([]*v3.Series, 0)
|
||||
for _, ts := range aggBucket.UpperBoundSeries {
|
||||
result.UpperBoundSeries = append(result.UpperBoundSeries, toV4Series(ts))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// ConvertV5TimeSeriesDataSliceToV4Results converts a slice of v5 TimeSeriesData to v4 QueryRangeResponse
|
||||
func ConvertV5TimeSeriesDataSliceToV4Results(v5DataSlice []*qbtypes.TimeSeriesData) *v3.QueryRangeResponse {
|
||||
response := &v3.QueryRangeResponse{
|
||||
ResultType: "matrix", // Time series data is typically "matrix" type
|
||||
Result: make([]*v3.Result, 0, len(v5DataSlice)),
|
||||
}
|
||||
|
||||
for _, v5Data := range v5DataSlice {
|
||||
if result := ConvertV5TimeSeriesDataToV4Result(v5Data); result != nil {
|
||||
response.Result = append(response.Result, result)
|
||||
}
|
||||
}
|
||||
|
||||
return response
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/queryBuilderToExpr"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
@@ -57,7 +57,7 @@ type StoreablePipeline struct {
|
||||
|
||||
type GettablePipeline struct {
|
||||
StoreablePipeline
|
||||
Filter *v3.FilterSet `json:"filter"`
|
||||
Filter *qbtypes.Filter `json:"filter"`
|
||||
Config []PipelineOperator `json:"config"`
|
||||
}
|
||||
|
||||
@@ -72,7 +72,7 @@ func (i *GettablePipeline) ParseRawConfig() error {
|
||||
}
|
||||
|
||||
func (i *GettablePipeline) ParseFilter() error {
|
||||
f := v3.FilterSet{}
|
||||
f := qbtypes.Filter{}
|
||||
err := json.Unmarshal([]byte(i.FilterString), &f)
|
||||
if err != nil {
|
||||
return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to parse filter")
|
||||
@@ -200,7 +200,7 @@ type PostablePipeline struct {
|
||||
Alias string `json:"alias"`
|
||||
Description string `json:"description"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Filter *v3.FilterSet `json:"filter"`
|
||||
Filter *qbtypes.Filter `json:"filter"`
|
||||
Config []PipelineOperator `json:"config"`
|
||||
}
|
||||
|
||||
@@ -218,6 +218,14 @@ func (p *PostablePipeline) IsValid() error {
|
||||
}
|
||||
|
||||
// check the filter
|
||||
if p.Filter == nil || strings.TrimSpace(p.Filter.Expression) == "" {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "filter.expression is required")
|
||||
}
|
||||
|
||||
// Validate that every field in the expression has an explicit context
|
||||
// (attribute., resource., body., etc) so later pipeline processing does not
|
||||
// need to guess. We do not validate that the field actually
|
||||
// exists – only that the context is specified.
|
||||
_, err := queryBuilderToExpr.Parse(p.Filter)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -3,24 +3,13 @@ package pipelinetypes
|
||||
import (
|
||||
"testing"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsValidPostablePipeline(t *testing.T) {
|
||||
validPipelineFilterSet := &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeTag,
|
||||
},
|
||||
Operator: "=",
|
||||
Value: "GET",
|
||||
},
|
||||
},
|
||||
validPipelineFilter := &qbtypes.Filter{
|
||||
Expression: `attribute.method = "GET"`,
|
||||
}
|
||||
|
||||
var correctQueriesTest = []struct {
|
||||
@@ -34,7 +23,7 @@ func TestIsValidPostablePipeline(t *testing.T) {
|
||||
Name: "pipeline 1",
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
Filter: validPipelineFilterSet,
|
||||
Filter: validPipelineFilter,
|
||||
Config: []PipelineOperator{},
|
||||
},
|
||||
IsValid: false,
|
||||
@@ -46,7 +35,7 @@ func TestIsValidPostablePipeline(t *testing.T) {
|
||||
Name: "pipeline 1",
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
Filter: validPipelineFilterSet,
|
||||
Filter: validPipelineFilter,
|
||||
Config: []PipelineOperator{},
|
||||
},
|
||||
IsValid: false,
|
||||
@@ -58,7 +47,7 @@ func TestIsValidPostablePipeline(t *testing.T) {
|
||||
Name: "pipeline 1",
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
Filter: validPipelineFilterSet,
|
||||
Filter: validPipelineFilter,
|
||||
Config: []PipelineOperator{},
|
||||
},
|
||||
IsValid: true,
|
||||
@@ -70,19 +59,21 @@ func TestIsValidPostablePipeline(t *testing.T) {
|
||||
Name: "pipeline 1",
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
Filter: &v3.FilterSet{
|
||||
Operator: "AND",
|
||||
Items: []v3.FilterItem{
|
||||
{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "method",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeUnspecified,
|
||||
},
|
||||
Operator: "regex",
|
||||
Value: "[0-9A-Z*",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "",
|
||||
},
|
||||
},
|
||||
IsValid: false,
|
||||
},
|
||||
{
|
||||
Name: "Filter without context prefix on field",
|
||||
Pipeline: PostablePipeline{
|
||||
OrderID: 1,
|
||||
Name: "pipeline 1",
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: `method = "GET"`,
|
||||
},
|
||||
},
|
||||
IsValid: false,
|
||||
@@ -94,19 +85,19 @@ func TestIsValidPostablePipeline(t *testing.T) {
|
||||
Name: "pipeline 1",
|
||||
Alias: "pipeline1",
|
||||
Enabled: true,
|
||||
Filter: validPipelineFilterSet,
|
||||
Filter: validPipelineFilter,
|
||||
},
|
||||
IsValid: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range correctQueriesTest {
|
||||
Convey(test.Name, t, func() {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
err := test.Pipeline.IsValid()
|
||||
if test.IsValid {
|
||||
So(err, ShouldBeNil)
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
So(err, ShouldBeError)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -365,12 +356,12 @@ var operatorTest = []struct {
|
||||
|
||||
func TestValidOperator(t *testing.T) {
|
||||
for _, test := range operatorTest {
|
||||
Convey(test.Name, t, func() {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
err := isValidOperator(test.Operator)
|
||||
if test.IsValid {
|
||||
So(err, ShouldBeNil)
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
So(err, ShouldBeError)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -76,33 +76,6 @@ type TimeSeries struct {
|
||||
Values []*TimeSeriesValue `json:"values"`
|
||||
}
|
||||
|
||||
// LabelsMap converts the label slice to a map[string]string for use in
|
||||
// alert evaluation helpers that operate on flat label maps.
|
||||
func (ts *TimeSeries) LabelsMap() map[string]string {
|
||||
if ts == nil {
|
||||
return nil
|
||||
}
|
||||
m := make(map[string]string, len(ts.Labels))
|
||||
for _, l := range ts.Labels {
|
||||
m[l.Key.Name] = fmt.Sprintf("%v", l.Value)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// NonPartialValues returns only the values where Partial is false.
|
||||
func (ts *TimeSeries) NonPartialValues() []*TimeSeriesValue {
|
||||
if ts == nil {
|
||||
return nil
|
||||
}
|
||||
result := make([]*TimeSeriesValue, 0, len(ts.Values))
|
||||
for _, v := range ts.Values {
|
||||
if !v.Partial {
|
||||
result = append(result, v)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
type Label struct {
|
||||
Key telemetrytypes.TelemetryFieldKey `json:"key"`
|
||||
Value any `json:"value"`
|
||||
|
||||
@@ -203,11 +203,11 @@ func (rc *RuleCondition) IsValid() bool {
|
||||
}
|
||||
|
||||
// ShouldEval checks if the further series should be evaluated at all for alerts.
|
||||
func (rc *RuleCondition) ShouldEval(series *qbtypes.TimeSeries) bool {
|
||||
func (rc *RuleCondition) ShouldEval(series *v3.Series) bool {
|
||||
if rc == nil {
|
||||
return true
|
||||
}
|
||||
return !rc.RequireMinPoints || len(series.NonPartialValues()) >= rc.RequiredNumPoints
|
||||
return !rc.RequireMinPoints || len(series.Points) >= rc.RequiredNumPoints
|
||||
}
|
||||
|
||||
// QueryType is a shorthand method to get query type
|
||||
|
||||
@@ -355,14 +355,6 @@ func (r *PostableRule) validate() error {
|
||||
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "composite query is required"))
|
||||
}
|
||||
|
||||
if r.Version != "" && r.Version != "v5" {
|
||||
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "only version v5 is supported, got %q", r.Version))
|
||||
}
|
||||
|
||||
if r.Version == "v5" && r.RuleCondition.CompositeQuery != nil && len(r.RuleCondition.CompositeQuery.Queries) == 0 {
|
||||
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "queries envelope is required in compositeQuery"))
|
||||
}
|
||||
|
||||
if isAllQueriesDisabled(r.RuleCondition.CompositeQuery) {
|
||||
errs = append(errs, signozError.NewInvalidInputf(signozError.CodeInvalidInput, "all queries are disabled in rule condition"))
|
||||
}
|
||||
|
||||
@@ -8,8 +8,6 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
func TestIsAllQueriesDisabled(t *testing.T) {
|
||||
@@ -623,9 +621,9 @@ func TestParseIntoRuleThresholdGeneration(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test that threshold can evaluate properly
|
||||
vector, err := threshold.Eval(qbtypes.TimeSeries{
|
||||
Values: []*qbtypes.TimeSeriesValue{{Value: 0.15, Timestamp: 1000}}, // 150ms in seconds
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "test"}, Value: "label"}},
|
||||
vector, err := threshold.Eval(v3.Series{
|
||||
Points: []v3.Point{{Value: 0.15, Timestamp: 1000}}, // 150ms in seconds
|
||||
Labels: map[string]string{"test": "label"},
|
||||
}, "", EvalData{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error in shouldAlert: %v", err)
|
||||
@@ -700,9 +698,9 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test with a value that should trigger both WARNING and CRITICAL thresholds
|
||||
vector, err := threshold.Eval(qbtypes.TimeSeries{
|
||||
Values: []*qbtypes.TimeSeriesValue{{Value: 95.0, Timestamp: 1000}}, // 95% CPU usage
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "service"}, Value: "test"}},
|
||||
vector, err := threshold.Eval(v3.Series{
|
||||
Points: []v3.Point{{Value: 95.0, Timestamp: 1000}}, // 95% CPU usage
|
||||
Labels: map[string]string{"service": "test"},
|
||||
}, "", EvalData{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error in shouldAlert: %v", err)
|
||||
@@ -710,9 +708,9 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) {
|
||||
|
||||
assert.Equal(t, 2, len(vector))
|
||||
|
||||
vector, err = threshold.Eval(qbtypes.TimeSeries{
|
||||
Values: []*qbtypes.TimeSeriesValue{{Value: 75.0, Timestamp: 1000}}, // 75% CPU usage
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "service"}, Value: "test"}},
|
||||
vector, err = threshold.Eval(v3.Series{
|
||||
Points: []v3.Point{{Value: 75.0, Timestamp: 1000}}, // 75% CPU usage
|
||||
Labels: map[string]string{"service": "test"},
|
||||
}, "", EvalData{})
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error in shouldAlert: %v", err)
|
||||
@@ -725,7 +723,7 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ruleJSON []byte
|
||||
series qbtypes.TimeSeries
|
||||
series v3.Series
|
||||
shouldAlert bool
|
||||
expectedValue float64
|
||||
}{
|
||||
@@ -753,9 +751,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`),
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: -2.1}, // below & at least once, should alert
|
||||
{Timestamp: 2000, Value: -2.3},
|
||||
},
|
||||
@@ -787,9 +785,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`), // below & at least once, no value below -2.0
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: -1.9},
|
||||
{Timestamp: 2000, Value: -1.8},
|
||||
},
|
||||
@@ -820,9 +818,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`), // above & at least once, should alert
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: 2.1}, // above 2.0, should alert
|
||||
{Timestamp: 2000, Value: 2.2},
|
||||
},
|
||||
@@ -854,9 +852,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`),
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: 1.1},
|
||||
{Timestamp: 2000, Value: 1.2},
|
||||
},
|
||||
@@ -887,9 +885,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`), // below and all the times
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: -2.1}, // all below -2
|
||||
{Timestamp: 2000, Value: -2.2},
|
||||
{Timestamp: 3000, Value: -2.5},
|
||||
@@ -922,9 +920,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`),
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: -3.0},
|
||||
{Timestamp: 2000, Value: -1.0}, // above -2, breaks condition
|
||||
{Timestamp: 3000, Value: -2.5},
|
||||
@@ -956,10 +954,10 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`),
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
{Timestamp: 1000, Value: -8.0}, // abs(-8) >= 7, alert
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: -8.0}, // abs(−8) >= 7, alert
|
||||
{Timestamp: 2000, Value: 5.0},
|
||||
},
|
||||
},
|
||||
@@ -990,9 +988,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`),
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: 80.0}, // below 90, should alert
|
||||
{Timestamp: 2000, Value: 85.0},
|
||||
},
|
||||
@@ -1024,9 +1022,9 @@ func TestAnomalyNegationEval(t *testing.T) {
|
||||
"selectedQuery": "A"
|
||||
}
|
||||
}`),
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{{Key: telemetrytypes.TelemetryFieldKey{Name: "host"}, Value: "server1"}},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"host": "server1"},
|
||||
Points: []v3.Point{
|
||||
{Timestamp: 1000, Value: 60.0}, // below, should alert
|
||||
{Timestamp: 2000, Value: 90.0},
|
||||
},
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/converter"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils/labels"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
@@ -83,7 +83,7 @@ func (eval EvalData) HasActiveAlert(sampleLabelFp uint64) bool {
|
||||
type RuleThreshold interface {
|
||||
// Eval runs the given series through the threshold rules
|
||||
// using the given EvalData and returns the matching series
|
||||
Eval(series qbtypes.TimeSeries, unit string, evalData EvalData) (Vector, error)
|
||||
Eval(series v3.Series, unit string, evalData EvalData) (Vector, error)
|
||||
GetRuleReceivers() []RuleReceivers
|
||||
}
|
||||
|
||||
@@ -122,11 +122,10 @@ func (r BasicRuleThresholds) Validate() error {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (r BasicRuleThresholds) Eval(series qbtypes.TimeSeries, unit string, evalData EvalData) (Vector, error) {
|
||||
func (r BasicRuleThresholds) Eval(series v3.Series, unit string, evalData EvalData) (Vector, error) {
|
||||
var resultVector Vector
|
||||
thresholds := []BasicRuleThreshold(r)
|
||||
sortThresholds(thresholds)
|
||||
seriesLabels := series.LabelsMap()
|
||||
for _, threshold := range thresholds {
|
||||
smpl, shouldAlert := threshold.shouldAlert(series, unit)
|
||||
if shouldAlert {
|
||||
@@ -138,15 +137,15 @@ func (r BasicRuleThresholds) Eval(series qbtypes.TimeSeries, unit string, evalDa
|
||||
resultVector = append(resultVector, smpl)
|
||||
continue
|
||||
} else if evalData.SendUnmatched {
|
||||
// Sanitise the series values to remove any NaN, Inf, or partial values
|
||||
values := filterValidValues(series.Values)
|
||||
if len(values) == 0 {
|
||||
// Sanitise the series points to remove any NaN or Inf values
|
||||
series.Points = removeGroupinSetPoints(series)
|
||||
if len(series.Points) == 0 {
|
||||
continue
|
||||
}
|
||||
// prepare the sample with the first value of the series
|
||||
// prepare the sample with the first point of the series
|
||||
smpl := Sample{
|
||||
Point: Point{T: values[0].Timestamp, V: values[0].Value},
|
||||
Metric: PrepareSampleLabelsForRule(seriesLabels, threshold.Name),
|
||||
Point: Point{T: series.Points[0].Timestamp, V: series.Points[0].Value},
|
||||
Metric: PrepareSampleLabelsForRule(series.Labels, threshold.Name),
|
||||
Target: *threshold.TargetValue,
|
||||
TargetUnit: threshold.TargetUnit,
|
||||
}
|
||||
@@ -161,7 +160,7 @@ func (r BasicRuleThresholds) Eval(series qbtypes.TimeSeries, unit string, evalDa
|
||||
if threshold.RecoveryTarget == nil {
|
||||
continue
|
||||
}
|
||||
sampleLabels := PrepareSampleLabelsForRule(seriesLabels, threshold.Name)
|
||||
sampleLabels := PrepareSampleLabelsForRule(series.Labels, threshold.Name)
|
||||
alertHash := sampleLabels.Hash()
|
||||
// check if alert is active and then check if recovery threshold matches
|
||||
if evalData.HasActiveAlert(alertHash) {
|
||||
@@ -256,23 +255,18 @@ func (b BasicRuleThreshold) Validate() error {
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (b BasicRuleThreshold) matchesRecoveryThreshold(series qbtypes.TimeSeries, ruleUnit string) (Sample, bool) {
|
||||
func (b BasicRuleThreshold) matchesRecoveryThreshold(series v3.Series, ruleUnit string) (Sample, bool) {
|
||||
return b.shouldAlertWithTarget(series, b.recoveryTarget(ruleUnit))
|
||||
}
|
||||
func (b BasicRuleThreshold) shouldAlert(series qbtypes.TimeSeries, ruleUnit string) (Sample, bool) {
|
||||
func (b BasicRuleThreshold) shouldAlert(series v3.Series, ruleUnit string) (Sample, bool) {
|
||||
return b.shouldAlertWithTarget(series, b.target(ruleUnit))
|
||||
}
|
||||
|
||||
// filterValidValues returns only the values that are valid for alert evaluation:
|
||||
// non-partial, non-NaN, non-Inf, and with non-negative timestamps.
|
||||
func filterValidValues(values []*qbtypes.TimeSeriesValue) []*qbtypes.TimeSeriesValue {
|
||||
var result []*qbtypes.TimeSeriesValue
|
||||
for _, v := range values {
|
||||
if v.Partial {
|
||||
continue
|
||||
}
|
||||
if v.Timestamp >= 0 && !math.IsNaN(v.Value) && !math.IsInf(v.Value, 0) {
|
||||
result = append(result, v)
|
||||
func removeGroupinSetPoints(series v3.Series) []v3.Point {
|
||||
var result []v3.Point
|
||||
for _, s := range series.Points {
|
||||
if s.Timestamp >= 0 && !math.IsNaN(s.Value) && !math.IsInf(s.Value, 0) {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
@@ -290,15 +284,15 @@ func PrepareSampleLabelsForRule(seriesLabels map[string]string, thresholdName st
|
||||
return lb.Labels()
|
||||
}
|
||||
|
||||
func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, target float64) (Sample, bool) {
|
||||
func (b BasicRuleThreshold) shouldAlertWithTarget(series v3.Series, target float64) (Sample, bool) {
|
||||
var shouldAlert bool
|
||||
var alertSmpl Sample
|
||||
lbls := PrepareSampleLabelsForRule(series.LabelsMap(), b.Name)
|
||||
lbls := PrepareSampleLabelsForRule(series.Labels, b.Name)
|
||||
|
||||
values := filterValidValues(series.Values)
|
||||
series.Points = removeGroupinSetPoints(series)
|
||||
|
||||
// nothing to evaluate
|
||||
if len(values) == 0 {
|
||||
if len(series.Points) == 0 {
|
||||
return alertSmpl, false
|
||||
}
|
||||
|
||||
@@ -306,7 +300,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
case AtleastOnce:
|
||||
// If any sample matches the condition, the rule is firing.
|
||||
if b.CompareOp == ValueIsAbove {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value > target {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
shouldAlert = true
|
||||
@@ -314,7 +308,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
}
|
||||
} else if b.CompareOp == ValueIsBelow {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value < target {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
shouldAlert = true
|
||||
@@ -322,7 +316,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
}
|
||||
} else if b.CompareOp == ValueIsEq {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value == target {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
shouldAlert = true
|
||||
@@ -330,7 +324,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
}
|
||||
} else if b.CompareOp == ValueIsNotEq {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value != target {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
shouldAlert = true
|
||||
@@ -338,7 +332,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
}
|
||||
} else if b.CompareOp == ValueOutsideBounds {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if math.Abs(smpl.Value) >= target {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
shouldAlert = true
|
||||
@@ -351,7 +345,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
shouldAlert = true
|
||||
alertSmpl = Sample{Point: Point{V: target}, Metric: lbls}
|
||||
if b.CompareOp == ValueIsAbove {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value <= target {
|
||||
shouldAlert = false
|
||||
break
|
||||
@@ -360,7 +354,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
// use min value from the series
|
||||
if shouldAlert {
|
||||
var minValue float64 = math.Inf(1)
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value < minValue {
|
||||
minValue = smpl.Value
|
||||
}
|
||||
@@ -368,7 +362,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
alertSmpl = Sample{Point: Point{V: minValue}, Metric: lbls}
|
||||
}
|
||||
} else if b.CompareOp == ValueIsBelow {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value >= target {
|
||||
shouldAlert = false
|
||||
break
|
||||
@@ -376,7 +370,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
if shouldAlert {
|
||||
var maxValue float64 = math.Inf(-1)
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value > maxValue {
|
||||
maxValue = smpl.Value
|
||||
}
|
||||
@@ -384,14 +378,14 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
alertSmpl = Sample{Point: Point{V: maxValue}, Metric: lbls}
|
||||
}
|
||||
} else if b.CompareOp == ValueIsEq {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value != target {
|
||||
shouldAlert = false
|
||||
break
|
||||
}
|
||||
}
|
||||
} else if b.CompareOp == ValueIsNotEq {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if smpl.Value == target {
|
||||
shouldAlert = false
|
||||
break
|
||||
@@ -399,7 +393,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
// use any non-inf or nan value from the series
|
||||
if shouldAlert {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if !math.IsInf(smpl.Value, 0) && !math.IsNaN(smpl.Value) {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
break
|
||||
@@ -407,7 +401,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
}
|
||||
}
|
||||
} else if b.CompareOp == ValueOutsideBounds {
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if math.Abs(smpl.Value) < target {
|
||||
alertSmpl = Sample{Point: Point{V: smpl.Value}, Metric: lbls}
|
||||
shouldAlert = false
|
||||
@@ -418,7 +412,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
case OnAverage:
|
||||
// If the average of all samples matches the condition, the rule is firing.
|
||||
var sum, count float64
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if math.IsNaN(smpl.Value) || math.IsInf(smpl.Value, 0) {
|
||||
continue
|
||||
}
|
||||
@@ -452,7 +446,7 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
// If the sum of all samples matches the condition, the rule is firing.
|
||||
var sum float64
|
||||
|
||||
for _, smpl := range values {
|
||||
for _, smpl := range series.Points {
|
||||
if math.IsNaN(smpl.Value) || math.IsInf(smpl.Value, 0) {
|
||||
continue
|
||||
}
|
||||
@@ -483,22 +477,21 @@ func (b BasicRuleThreshold) shouldAlertWithTarget(series qbtypes.TimeSeries, tar
|
||||
case Last:
|
||||
// If the last sample matches the condition, the rule is firing.
|
||||
shouldAlert = false
|
||||
lastValue := values[len(values)-1].Value
|
||||
alertSmpl = Sample{Point: Point{V: lastValue}, Metric: lbls}
|
||||
alertSmpl = Sample{Point: Point{V: series.Points[len(series.Points)-1].Value}, Metric: lbls}
|
||||
if b.CompareOp == ValueIsAbove {
|
||||
if lastValue > target {
|
||||
if series.Points[len(series.Points)-1].Value > target {
|
||||
shouldAlert = true
|
||||
}
|
||||
} else if b.CompareOp == ValueIsBelow {
|
||||
if lastValue < target {
|
||||
if series.Points[len(series.Points)-1].Value < target {
|
||||
shouldAlert = true
|
||||
}
|
||||
} else if b.CompareOp == ValueIsEq {
|
||||
if lastValue == target {
|
||||
if series.Points[len(series.Points)-1].Value == target {
|
||||
shouldAlert = true
|
||||
}
|
||||
} else if b.CompareOp == ValueIsNotEq {
|
||||
if lastValue != target {
|
||||
if series.Points[len(series.Points)-1].Value != target {
|
||||
shouldAlert = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,24 +6,16 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
)
|
||||
|
||||
func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
target := 100.0
|
||||
|
||||
makeLabel := func(name, value string) *qbtypes.Label {
|
||||
return &qbtypes.Label{Key: telemetrytypes.TelemetryFieldKey{Name: name}, Value: value}
|
||||
}
|
||||
makeValue := func(value float64, ts int64) *qbtypes.TimeSeriesValue {
|
||||
return &qbtypes.TimeSeriesValue{Value: value, Timestamp: ts}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
threshold BasicRuleThreshold
|
||||
series qbtypes.TimeSeries
|
||||
series v3.Series
|
||||
ruleUnit string
|
||||
shouldAlert bool
|
||||
}{
|
||||
@@ -36,9 +28,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(0.15, 1000)}, // 150ms in seconds
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.15, Timestamp: 1000}, // 150ms in seconds
|
||||
},
|
||||
},
|
||||
ruleUnit: "s",
|
||||
shouldAlert: true,
|
||||
@@ -52,9 +46,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(0.05, 1000)}, // 50ms in seconds
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.05, Timestamp: 1000}, // 50ms in seconds
|
||||
},
|
||||
},
|
||||
ruleUnit: "s",
|
||||
shouldAlert: false,
|
||||
@@ -68,9 +64,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(150000, 1000)}, // 150000ms = 150s
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 150000, Timestamp: 1000}, // 150000ms = 150s
|
||||
},
|
||||
},
|
||||
ruleUnit: "ms",
|
||||
shouldAlert: true,
|
||||
@@ -85,9 +83,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(0.15, 1000)}, // 0.15KiB ≈ 153.6 bytes
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.15, Timestamp: 1000}, // 0.15KiB ≈ 153.6 bytes
|
||||
},
|
||||
},
|
||||
ruleUnit: "kbytes",
|
||||
shouldAlert: true,
|
||||
@@ -101,9 +101,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(0.15, 1000)},
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.15, Timestamp: 1000},
|
||||
},
|
||||
},
|
||||
ruleUnit: "mbytes",
|
||||
shouldAlert: true,
|
||||
@@ -118,9 +120,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsBelow,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(0.05, 1000)}, // 50ms in seconds
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.05, Timestamp: 1000}, // 50ms in seconds
|
||||
},
|
||||
},
|
||||
ruleUnit: "s",
|
||||
shouldAlert: true,
|
||||
@@ -134,12 +138,12 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: OnAverage,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
makeValue(0.08, 1000), // 80ms
|
||||
makeValue(0.12, 2000), // 120ms
|
||||
makeValue(0.15, 3000), // 150ms
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.08, Timestamp: 1000}, // 80ms
|
||||
{Value: 0.12, Timestamp: 2000}, // 120ms
|
||||
{Value: 0.15, Timestamp: 3000}, // 150ms
|
||||
},
|
||||
},
|
||||
ruleUnit: "s",
|
||||
@@ -154,12 +158,12 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: InTotal,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
makeValue(0.04, 1000), // 40MB
|
||||
makeValue(0.05, 2000), // 50MB
|
||||
makeValue(0.03, 3000), // 30MB
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.04, Timestamp: 1000}, // 40MB
|
||||
{Value: 0.05, Timestamp: 2000}, // 50MB
|
||||
{Value: 0.03, Timestamp: 3000}, // 30MB
|
||||
},
|
||||
},
|
||||
ruleUnit: "decgbytes",
|
||||
@@ -174,12 +178,12 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AllTheTimes,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
makeValue(0.11, 1000), // 110ms
|
||||
makeValue(0.12, 2000), // 120ms
|
||||
makeValue(0.15, 3000), // 150ms
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.11, Timestamp: 1000}, // 110ms
|
||||
{Value: 0.12, Timestamp: 2000}, // 120ms
|
||||
{Value: 0.15, Timestamp: 3000}, // 150ms
|
||||
},
|
||||
},
|
||||
ruleUnit: "s",
|
||||
@@ -194,11 +198,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: Last,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{
|
||||
makeValue(0.15, 1000), // 150kB
|
||||
makeValue(0.05, 2000), // 50kB (last value)
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.15, Timestamp: 1000}, // 150kB
|
||||
{Value: 0.05, Timestamp: 2000}, // 50kB (last value)
|
||||
},
|
||||
},
|
||||
ruleUnit: "decmbytes",
|
||||
@@ -214,9 +218,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(0.15, 1000)},
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 0.15, Timestamp: 1000},
|
||||
},
|
||||
},
|
||||
ruleUnit: "KBs",
|
||||
shouldAlert: true,
|
||||
@@ -231,9 +237,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(150, 1000)}, // 150ms
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 150, Timestamp: 1000}, // 150ms
|
||||
},
|
||||
},
|
||||
ruleUnit: "ms",
|
||||
shouldAlert: true,
|
||||
@@ -248,9 +256,11 @@ func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) {
|
||||
MatchType: AtleastOnce,
|
||||
CompareOp: ValueIsAbove,
|
||||
},
|
||||
series: qbtypes.TimeSeries{
|
||||
Labels: []*qbtypes.Label{makeLabel("service", "test")},
|
||||
Values: []*qbtypes.TimeSeriesValue{makeValue(150, 1000)}, // 150 (unitless)
|
||||
series: v3.Series{
|
||||
Labels: map[string]string{"service": "test"},
|
||||
Points: []v3.Point{
|
||||
{Value: 150, Timestamp: 1000}, // 150 (unitless)
|
||||
},
|
||||
},
|
||||
ruleUnit: "",
|
||||
shouldAlert: true,
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
from http import HTTPStatus
|
||||
from typing import Callable
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.types import SigNoz
|
||||
|
||||
DUPLICATE_USER_EMAIL = "duplicate@integration.test"
|
||||
|
||||
|
||||
def test_duplicate_user_invite_rejected(
|
||||
signoz: SigNoz,
|
||||
get_token: Callable[[str, str], str],
|
||||
):
|
||||
"""
|
||||
Verify that the unique index on (email, org_id) in the users table prevents
|
||||
creating duplicate users. This invites a new user, accepts the invite, then
|
||||
tries to invite and accept the same email again expecting a failure.
|
||||
"""
|
||||
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Step 1: Invite a new user.
|
||||
initial_invite_response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/invite"),
|
||||
json={"email": DUPLICATE_USER_EMAIL, "role": "EDITOR"},
|
||||
headers={"Authorization": f"Bearer {admin_token}"},
|
||||
timeout=2,
|
||||
)
|
||||
assert initial_invite_response.status_code == HTTPStatus.CREATED
|
||||
initial_invite_token = initial_invite_response.json()["data"]["token"]
|
||||
|
||||
# Step 2: Accept the invite to create the user.
|
||||
initial_accept_response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/invite/accept"),
|
||||
json={"token": initial_invite_token, "password": "password123Z$"},
|
||||
timeout=2,
|
||||
)
|
||||
assert initial_accept_response.status_code == HTTPStatus.CREATED
|
||||
|
||||
# Step 3: Invite the same email again.
|
||||
duplicate_invite_response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/invite"),
|
||||
json={"email": DUPLICATE_USER_EMAIL, "role": "VIEWER"},
|
||||
headers={"Authorization": f"Bearer {admin_token}"},
|
||||
timeout=2,
|
||||
)
|
||||
|
||||
# The invite creation itself may be rejected if the app checks for existing users.
|
||||
if duplicate_invite_response.status_code != HTTPStatus.CREATED:
|
||||
assert duplicate_invite_response.status_code == HTTPStatus.CONFLICT
|
||||
return
|
||||
|
||||
duplicate_invite_token = duplicate_invite_response.json()["data"]["token"]
|
||||
|
||||
# Step 4: Accept the duplicate invite — should fail due to unique constraint.
|
||||
duplicate_accept_response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/invite/accept"),
|
||||
json={"token": duplicate_invite_token, "password": "password123Z$"},
|
||||
timeout=2,
|
||||
)
|
||||
assert duplicate_accept_response.status_code == HTTPStatus.CONFLICT
|
||||
Reference in New Issue
Block a user