Compare commits

...

25 Commits

Author SHA1 Message Date
Piyush Singariya
96cebb095b fix: only accounting primitive types 2026-03-05 17:44:20 +05:30
Piyush Singariya
a4d24d5055 fix: convert other types to string 2026-03-05 17:42:10 +05:30
Piyush Singariya
36c5db177f feat: operators LIKE/ILIKE support in logs pipelines 2026-03-05 17:27:55 +05:30
Piyush Singariya
2519f061d0 chore: replace dialect with operator 2026-03-05 01:19:51 +05:30
Piyush Singariya
ce24ac217a fix: minor fixes 2026-03-04 23:44:20 +05:30
Piyush Singariya
215b16f7ef feat: build expr format without intermediate layer 2026-03-04 23:37:22 +05:30
Piyush Singariya
695ab0ae14 fix: better migration 2026-02-24 11:59:19 +05:30
Piyush Singariya
27802d8256 chore: updated comment 2026-02-24 10:14:56 +05:30
Piyush Singariya
e411ecf92b fix: remove logicalopleaf 2026-02-24 10:12:00 +05:30
Piyush Singariya
029cd38e21 fix: test + rename files 2026-02-18 17:11:47 +05:30
Piyush Singariya
421f3bdd9e fix: go test 2026-02-18 17:01:20 +05:30
Piyush Singariya
53a0b532bb fix: go ci errors 2026-02-18 16:25:57 +05:30
Piyush Singariya
03a13678d0 Merge branch 'main' into lp-filter-v5 2026-02-18 16:14:33 +05:30
Piyush Singariya
14d4751c96 fix: tests and migration ran successfully 2026-02-18 16:10:20 +05:30
Piyush Singariya
b50d8136a5 Merge branch 'main' into lp-filter-v5 2026-02-18 15:34:37 +05:30
Piyush Singariya
4103fe4d2f fix: added test and some fixes 2026-02-18 14:29:52 +05:30
Piyush Singariya
ccc102dc7b fix: singular key fix 2026-02-10 14:56:40 +05:30
Piyush Singariya
2cd0ab8a28 fix: singular key in comparison condition 2026-02-10 14:53:51 +05:30
Piyush Singariya
b101ffeae1 fix: no attributes. in filter 2026-02-10 14:28:52 +05:30
Piyush Singariya
d7f543ad4a fix: working on migration 2026-02-10 14:22:51 +05:30
Piyush Singariya
f81c64f348 feat: address negate 2026-02-09 20:32:40 +05:30
Piyush Singariya
40956116dc fix: qbtoexpr migrated to v5 2026-02-09 20:24:32 +05:30
Piyush Singariya
b49a95b7b3 Merge branch 'main' into lp-filter-v5 2026-02-09 13:06:54 +05:30
Piyush Singariya
ae19bb2be2 fix: filter expression 2026-02-05 18:10:45 +05:30
Piyush Singariya
749f52ff9d chore: moving from v3 to v5 2026-02-05 17:48:57 +05:30
19 changed files with 1600 additions and 702 deletions

View File

@@ -7,9 +7,9 @@ import (
"github.com/SigNoz/signoz/pkg/query-service/constants"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
. "github.com/smartystreets/goconvey/convey"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
@@ -96,10 +96,10 @@ var buildProcessorTestData = []struct {
func TestBuildLogParsingProcessors(t *testing.T) {
for _, test := range buildProcessorTestData {
Convey(test.Name, t, func() {
t.Run(test.Name, func(t *testing.T) {
err := updateProcessorConfigsInCollectorConf(test.agentConf, test.pipelineProcessor)
So(err, ShouldBeNil)
So(test.agentConf, ShouldResemble, test.outputConf)
assert.NoError(t, err)
assert.Equal(t, test.outputConf, test.agentConf)
})
}
@@ -202,11 +202,11 @@ var BuildLogsPipelineTestData = []struct {
func TestBuildLogsPipeline(t *testing.T) {
for _, test := range BuildLogsPipelineTestData {
Convey(test.Name, t, func() {
t.Run(test.Name, func(t *testing.T) {
v, err := buildCollectorPipelineProcessorsList(test.currentPipeline, test.logsPipeline)
So(err, ShouldBeNil)
assert.NoError(t, err)
fmt.Println(test.Name, "\n", test.currentPipeline, "\n", v, "\n", test.expectedPipeline)
So(v, ShouldResemble, test.expectedPipeline)
assert.Equal(t, test.expectedPipeline, v)
})
}
}
@@ -239,19 +239,8 @@ func TestPipelineAliasCollisionsDontResultInDuplicateCollectorProcessors(t *test
Alias: alias,
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -310,19 +299,8 @@ func TestPipelineRouterWorksEvenIfFirstOpIsDisabled(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -383,19 +361,8 @@ func TestPipeCharInAliasDoesntBreakCollectorConfig(t *testing.T) {
Alias: "test|pipeline",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{

View File

@@ -11,13 +11,13 @@ import (
"github.com/SigNoz/signoz/pkg/query-service/agentConf"
"github.com/SigNoz/signoz/pkg/query-service/constants"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/utils"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/opamptypes"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/google/uuid"
@@ -140,15 +140,8 @@ func (ic *LogParsingPipelineController) getDefaultPipelines() ([]pipelinetypes.G
Alias: "NormalizeBodyDefault",
Enabled: true,
},
Filter: &v3.FilterSet{
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "body",
},
Operator: v3.FilterOperatorExists,
},
},
Filter: &qbtypes.Filter{
Expression: "body EXISTS",
},
Config: []pipelinetypes.PipelineOperator{
{

View File

@@ -9,14 +9,14 @@ import (
signozstanzahelper "github.com/SigNoz/signoz-otel-collector/processor/signozlogspipelineprocessor/stanza/operator/helper"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/utils"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/google/uuid"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
. "github.com/smartystreets/goconvey/convey"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -203,10 +203,10 @@ var prepareProcessorTestData = []struct {
func TestPreparePipelineProcessor(t *testing.T) {
for _, test := range prepareProcessorTestData {
Convey(test.Name, t, func() {
t.Run(test.Name, func(t *testing.T) {
res, err := getOperators(test.Operators)
So(err, ShouldBeNil)
So(res, ShouldResemble, test.Output)
assert.NoError(t, err)
assert.Equal(t, test.Output, res)
})
}
}
@@ -214,19 +214,8 @@ func TestPreparePipelineProcessor(t *testing.T) {
func TestNoCollectorErrorsFromProcessorsForMismatchedLogs(t *testing.T) {
require := require.New(t)
testPipelineFilter := &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
testPipelineFilter := &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
}
makeTestPipeline := func(config []pipelinetypes.PipelineOperator) pipelinetypes.GettablePipeline {
return pipelinetypes.GettablePipeline{
@@ -470,19 +459,8 @@ func TestResourceFiltersWork(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "service",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeResource,
},
Operator: "=",
Value: "nginx",
},
},
Filter: &qbtypes.Filter{
Expression: "resource.service = 'nginx'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -524,11 +502,11 @@ func TestResourceFiltersWork(t *testing.T) {
func TestPipelineFilterWithStringOpsShouldNotSpamWarningsIfAttributeIsMissing(t *testing.T) {
require := require.New(t)
for _, operator := range []v3.FilterOperator{
v3.FilterOperatorContains,
v3.FilterOperatorNotContains,
v3.FilterOperatorRegex,
v3.FilterOperatorNotRegex,
for _, operator := range []string{
"CONTAINS",
"NOT CONTAINS",
"REGEXP",
"NOT REGEXP",
} {
testPipeline := pipelinetypes.GettablePipeline{
StoreablePipeline: pipelinetypes.StoreablePipeline{
@@ -540,19 +518,8 @@ func TestPipelineFilterWithStringOpsShouldNotSpamWarningsIfAttributeIsMissing(t
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "service",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeResource,
},
Operator: operator,
Value: "nginx",
},
},
Filter: &qbtypes.Filter{
Expression: fmt.Sprintf("resource.service %s 'nginx'", operator),
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -601,19 +568,8 @@ func TestAttributePathsContainingDollarDoNotBreakCollector(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "$test",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "test",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.$test = 'test'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -664,19 +620,8 @@ func TestMembershipOpInProcessorFieldExpressions(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "http.method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.http.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -755,7 +700,7 @@ func TestMembershipOpInProcessorFieldExpressions(t *testing.T) {
}
func TestContainsFilterIsCaseInsensitive(t *testing.T) {
// The contains and ncontains query builder filters are case insensitive when querying logs.
// The CONTAINS and NOT CONTAINS query builder filters are case insensitive when querying logs.
// Pipeline filter should also behave in the same way.
require := require.New(t)
@@ -773,18 +718,8 @@ func TestContainsFilterIsCaseInsensitive(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{
Key: "body",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeUnspecified,
IsColumn: true,
},
Operator: "contains",
Value: "log",
}},
Filter: &qbtypes.Filter{
Expression: "body CONTAINS 'log'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -806,18 +741,8 @@ func TestContainsFilterIsCaseInsensitive(t *testing.T) {
Alias: "pipeline2",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{
Key: "body",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeUnspecified,
IsColumn: true,
},
Operator: "ncontains",
Value: "ecom",
}},
Filter: &qbtypes.Filter{
Expression: "body NOT CONTAINS 'ecom'",
},
Config: []pipelinetypes.PipelineOperator{
{

View File

@@ -7,8 +7,8 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/google/uuid"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
"github.com/stretchr/testify/require"
@@ -25,19 +25,8 @@ func TestPipelinePreview(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -58,19 +47,8 @@ func TestPipelinePreview(t *testing.T) {
Alias: "pipeline2",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{
@@ -159,19 +137,8 @@ func TestGrokParsingProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{
{

View File

@@ -10,8 +10,8 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/utils"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
"github.com/stretchr/testify/require"
)
@@ -30,19 +30,8 @@ func TestRegexProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -97,19 +86,8 @@ func TestGrokProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -164,19 +142,8 @@ func TestJSONProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -230,20 +197,9 @@ func TestTraceParsingProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
}
@@ -339,19 +295,8 @@ func TestAddProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -404,19 +349,8 @@ func TestRemoveProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -469,19 +403,8 @@ func TestCopyProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -535,19 +458,8 @@ func TestMoveProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},

View File

@@ -7,8 +7,8 @@ import (
"testing"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/require"
)
@@ -23,19 +23,8 @@ func TestSeverityParsingProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},
@@ -141,19 +130,8 @@ func TestSeverityParsingProcessor(t *testing.T) {
func TestNoCollectorErrorsFromSeverityParserForMismatchedLogs(t *testing.T) {
require := require.New(t)
testPipelineFilter := &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
testPipelineFilter := &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
}
makeTestPipeline := func(config []pipelinetypes.PipelineOperator) pipelinetypes.GettablePipeline {
return pipelinetypes.GettablePipeline{

View File

@@ -8,8 +8,8 @@ import (
"time"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/require"
)
@@ -24,19 +24,8 @@ func TestTimestampParsingProcessor(t *testing.T) {
Alias: "pipeline1",
Enabled: true,
},
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
Filter: &qbtypes.Filter{
Expression: "attribute.method = 'GET'",
},
Config: []pipelinetypes.PipelineOperator{},
},

View File

@@ -2,11 +2,19 @@ package queryBuilderToExpr
import (
"fmt"
"maps"
"reflect"
"regexp"
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
grammar "github.com/SigNoz/signoz/pkg/parser/grammar"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/antlr4-go/antlr/v4"
expr "github.com/antonmedv/expr"
"go.uber.org/zap"
)
@@ -15,147 +23,635 @@ var (
CodeExprCompilationFailed = errors.MustNewCode("expr_compilation_failed")
)
var logOperatorsToExpr = map[v3.FilterOperator]string{
v3.FilterOperatorEqual: "==",
v3.FilterOperatorNotEqual: "!=",
v3.FilterOperatorLessThan: "<",
v3.FilterOperatorLessThanOrEq: "<=",
v3.FilterOperatorGreaterThan: ">",
v3.FilterOperatorGreaterThanOrEq: ">=",
v3.FilterOperatorContains: "contains",
v3.FilterOperatorNotContains: "not contains",
v3.FilterOperatorRegex: "matches",
v3.FilterOperatorNotRegex: "not matches",
v3.FilterOperatorIn: "in",
v3.FilterOperatorNotIn: "not in",
v3.FilterOperatorExists: "in",
v3.FilterOperatorNotExists: "not in",
// we dont support like and nlike as of now.
var logOperatorsToExpr = map[qbtypes.FilterOperator]string{
qbtypes.FilterOperatorEqual: "==",
qbtypes.FilterOperatorNotEqual: "!=",
qbtypes.FilterOperatorLessThan: "<",
qbtypes.FilterOperatorLessThanOrEq: "<=",
qbtypes.FilterOperatorGreaterThan: ">",
qbtypes.FilterOperatorGreaterThanOrEq: ">=",
qbtypes.FilterOperatorContains: "contains",
qbtypes.FilterOperatorNotContains: "not contains",
qbtypes.FilterOperatorRegexp: "matches",
qbtypes.FilterOperatorNotRegexp: "not matches",
qbtypes.FilterOperatorIn: "in",
qbtypes.FilterOperatorNotIn: "not in",
qbtypes.FilterOperatorExists: "in",
qbtypes.FilterOperatorNotExists: "not in",
}
func getName(v v3.AttributeKey) string {
if v.Type == v3.AttributeKeyTypeTag {
return fmt.Sprintf(`attributes["%s"]`, v.Key)
} else if v.Type == v3.AttributeKeyTypeResource {
return fmt.Sprintf(`resource["%s"]`, v.Key)
func getName(key *telemetrytypes.TelemetryFieldKey) (string, error) {
if key == nil {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "field key is nil")
}
switch key.FieldContext {
case telemetrytypes.FieldContextAttribute:
return fmt.Sprintf(`attributes["%s"]`, key.Name), nil
case telemetrytypes.FieldContextResource:
return fmt.Sprintf(`resource["%s"]`, key.Name), nil
case telemetrytypes.FieldContextBody:
return fmt.Sprintf("%s.%s", key.FieldContext.StringValue(), key.Name), nil
default:
return key.Name, nil
}
return v.Key
}
func getTypeName(v v3.AttributeKeyType) string {
if v == v3.AttributeKeyTypeTag {
return "attributes"
} else if v == v3.AttributeKeyTypeResource {
return "resource"
// exprVisitor is an ANTLR visitor that directly produces expr-lang expression strings,
// eliminating the intermediate FilterExprNode / FilterCondition representation.
type exprVisitor struct {
errors []string
}
func (v *exprVisitor) Visit(tree antlr.ParseTree) any {
if tree == nil {
return ""
}
switch t := tree.(type) {
case *grammar.QueryContext:
return v.VisitQuery(t)
case *grammar.ExpressionContext:
return v.VisitExpression(t)
case *grammar.OrExpressionContext:
return v.VisitOrExpression(t)
case *grammar.AndExpressionContext:
return v.VisitAndExpression(t)
case *grammar.UnaryExpressionContext:
return v.VisitUnaryExpression(t)
case *grammar.PrimaryContext:
return v.VisitPrimary(t)
case *grammar.ComparisonContext:
return v.VisitComparison(t)
case *grammar.InClauseContext:
return v.VisitInClause(t)
case *grammar.NotInClauseContext:
return v.VisitNotInClause(t)
case *grammar.ValueListContext:
return v.VisitValueList(t)
case *grammar.ValueContext:
return v.VisitValue(t)
case *grammar.KeyContext:
return v.VisitKey(t)
case *grammar.FunctionCallContext:
return v.VisitFunctionCall(t)
case *grammar.FunctionParamListContext:
return v.VisitFunctionParamList(t)
case *grammar.FunctionParamContext:
return v.VisitFunctionParam(t)
case *grammar.ArrayContext:
return v.VisitArray(t)
case *grammar.FullTextContext:
return v.VisitFullText(t)
default:
return ""
}
}
func (v *exprVisitor) VisitQuery(ctx *grammar.QueryContext) any {
return v.Visit(ctx.Expression())
}
func (v *exprVisitor) VisitExpression(ctx *grammar.ExpressionContext) any {
return v.Visit(ctx.OrExpression())
}
func (v *exprVisitor) VisitOrExpression(ctx *grammar.OrExpressionContext) any {
andExprs := ctx.AllAndExpression()
var parts []string
for _, andExpr := range andExprs {
s, _ := v.Visit(andExpr).(string)
if s != "" {
parts = append(parts, s)
}
}
if len(parts) == 0 {
return ""
}
if len(parts) == 1 {
return parts[0]
}
return strings.Join(parts, " or ")
}
func (v *exprVisitor) VisitAndExpression(ctx *grammar.AndExpressionContext) any {
unaryExprs := ctx.AllUnaryExpression()
var parts []string
for _, unary := range unaryExprs {
s, _ := v.Visit(unary).(string)
if s != "" {
parts = append(parts, s)
}
}
if len(parts) == 0 {
return ""
}
if len(parts) == 1 {
return parts[0]
}
return strings.Join(parts, " and ")
}
func (v *exprVisitor) VisitUnaryExpression(ctx *grammar.UnaryExpressionContext) any {
s, _ := v.Visit(ctx.Primary()).(string)
if s == "" {
return ""
}
if ctx.NOT() != nil {
// VisitPrimary already wraps parenthesized sub-expressions (when the user
// wrote explicit parens, i.e. Primary.OrExpression != nil) in '(...)'.
// In that case, prepend "not " without adding another pair of parens to
// avoid the double-wrapping: not ((expr)) → not (expr).
if primaryCtx, ok := ctx.Primary().(*grammar.PrimaryContext); ok && primaryCtx.OrExpression() != nil {
return "not " + s
}
return fmt.Sprintf("not (%s)", s)
}
return s
}
func (v *exprVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any {
switch {
case ctx.OrExpression() != nil:
// Parenthesized sub-expression: wrap to preserve precedence.
s, _ := v.Visit(ctx.OrExpression()).(string)
if s == "" {
return ""
}
return fmt.Sprintf("(%s)", s)
case ctx.Comparison() != nil:
return v.Visit(ctx.Comparison())
case ctx.FunctionCall() != nil:
return v.Visit(ctx.FunctionCall())
case ctx.FullText() != nil:
return v.Visit(ctx.FullText())
default:
return ""
}
}
func (v *exprVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := v.buildKey(ctx.Key())
if key == nil {
return ""
}
// Validate: fields without context must be intrinsic (top-level OTEL log fields).
_, isIntrinsic := telemetrylogs.IntrinsicFields[key.Name]
if key.FieldContext == telemetrytypes.FieldContextUnspecified && !isIntrinsic {
v.errors = append(v.errors, fmt.Sprintf(
"field %q in filter expression must include a context prefix (attribute., resource., body.) OR can be one of the following fields: %v",
key.Name, maps.Keys(telemetrylogs.IntrinsicFields),
))
return ""
}
// EXISTS / NOT EXISTS are structural and don't follow the standard value path.
if ctx.EXISTS() != nil {
return v.buildExistsExpr(key, ctx.NOT() != nil)
}
// BETWEEN / NOT BETWEEN: two numeric values, expanded to range comparisons.
if ctx.BETWEEN() != nil {
return v.buildBetweenExpr(key, ctx.NOT() != nil, ctx.AllValue())
}
// Determine operator from grammar tokens.
notModifier := ctx.NOT() != nil
var op qbtypes.FilterOperator
switch {
case ctx.EQUALS() != nil:
op = qbtypes.FilterOperatorEqual
case ctx.NOT_EQUALS() != nil || ctx.NEQ() != nil:
op = qbtypes.FilterOperatorNotEqual
case ctx.LT() != nil:
op = qbtypes.FilterOperatorLessThan
case ctx.LE() != nil:
op = qbtypes.FilterOperatorLessThanOrEq
case ctx.GT() != nil:
op = qbtypes.FilterOperatorGreaterThan
case ctx.GE() != nil:
op = qbtypes.FilterOperatorGreaterThanOrEq
case ctx.REGEXP() != nil:
if notModifier {
op = qbtypes.FilterOperatorNotRegexp
} else {
op = qbtypes.FilterOperatorRegexp
}
case ctx.CONTAINS() != nil:
if notModifier {
op = qbtypes.FilterOperatorNotContains
} else {
op = qbtypes.FilterOperatorContains
}
case ctx.InClause() != nil:
op = qbtypes.FilterOperatorIn
case ctx.NotInClause() != nil:
op = qbtypes.FilterOperatorNotIn
case ctx.LIKE() != nil:
if notModifier {
op = qbtypes.FilterOperatorNotLike
} else {
op = qbtypes.FilterOperatorLike
}
case ctx.ILIKE() != nil:
if notModifier {
op = qbtypes.FilterOperatorNotILike
} else {
op = qbtypes.FilterOperatorILike
}
default:
v.errors = append(v.errors, fmt.Sprintf("unsupported comparison operator: %s", ctx.GetText()))
return ""
}
switch op {
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorNotLike,
qbtypes.FilterOperatorILike, qbtypes.FilterOperatorNotILike:
// supported using functions
default:
if _, ok := logOperatorsToExpr[op]; !ok {
v.errors = append(v.errors, fmt.Sprintf("operator not supported: %v", op))
return ""
}
}
// Build the right-hand side value.
var value any
if op == qbtypes.FilterOperatorIn || op == qbtypes.FilterOperatorNotIn {
value = v.buildValuesFromInClause(ctx.InClause(), ctx.NotInClause())
} else {
valuesCtx := ctx.AllValue()
if len(valuesCtx) == 0 {
v.errors = append(v.errors, "comparison operator requires a value")
return ""
}
value = v.buildValue(valuesCtx[0])
}
// Validate patterns eagerly.
switch op {
case qbtypes.FilterOperatorRegexp, qbtypes.FilterOperatorNotRegexp:
str, ok := value.(string)
if !ok {
v.errors = append(v.errors, "value for regex operator must be a string")
return ""
}
if _, err := regexp.Compile(str); err != nil {
v.errors = append(v.errors, "value for regex operator must be a valid regex")
return ""
}
case qbtypes.FilterOperatorLike, qbtypes.FilterOperatorNotLike,
qbtypes.FilterOperatorILike, qbtypes.FilterOperatorNotILike:
_, ok := value.(string)
if !ok {
v.errors = append(v.errors, "value for LIKE/NOT LIKE/ILIKE/ NOT ILIKE operator must be a string")
return ""
}
}
filter, err := buildFilterExpr(key, op, value)
if err != nil {
v.errors = append(v.errors, err.Error())
return ""
}
if _, err := expr.Compile(filter); err != nil {
v.errors = append(v.errors, err.Error())
return ""
}
return filter
}
// buildExistsExpr produces the expr-lang string for EXISTS / NOT EXISTS checks.
func (v *exprVisitor) buildExistsExpr(key *telemetrytypes.TelemetryFieldKey, isNotExists bool) string {
op := qbtypes.FilterOperatorExists
if isNotExists {
op = qbtypes.FilterOperatorNotExists
}
switch key.FieldContext {
case telemetrytypes.FieldContextBody:
// JSON body: check membership in fromJSON(body).
// Need to quote the key for expr lang
quoted := exprFormattedValue(key.Name)
jsonMembership := fmt.Sprintf(
`((type(body) == "string" && isJSON(body)) && %s %s %s)`,
quoted, logOperatorsToExpr[op], "fromJSON(body)",
)
// Map body: nil check on the field.
nilOp := qbtypes.FilterOperatorNotEqual
if isNotExists {
nilOp = qbtypes.FilterOperatorEqual
}
nilCheckFilter := fmt.Sprintf("%s.%s %s nil", key.FieldContext.StringValue(), key.Name, logOperatorsToExpr[nilOp])
return fmt.Sprintf(`(%s or (type(body) == "map" && (%s)))`, jsonMembership, nilCheckFilter)
case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource:
// Check membership in the attributes / resource map.
target := "resource"
if key.FieldContext == telemetrytypes.FieldContextAttribute {
target = "attributes"
}
return fmt.Sprintf("%q %s %s", key.Name, logOperatorsToExpr[op], target)
default:
// Intrinsic / top-level field: use a nil comparison.
nilOp := qbtypes.FilterOperatorNotEqual
if isNotExists {
nilOp = qbtypes.FilterOperatorEqual
}
return fmt.Sprintf("%s %s nil", key.Name, logOperatorsToExpr[nilOp])
}
}
// buildBetweenExpr expands BETWEEN / NOT BETWEEN into range comparisons.
// Only numeric values are accepted; strings and booleans are rejected.
//
// key BETWEEN lo AND hi → keyName != nil && keyName >= lo && keyName <= hi
// key NOT BETWEEN lo AND hi → keyName != nil && (keyName < lo || keyName > hi)
func (v *exprVisitor) buildBetweenExpr(key *telemetrytypes.TelemetryFieldKey, isNot bool, valuesCtx []grammar.IValueContext) string {
if len(valuesCtx) != 2 {
v.errors = append(v.errors, "BETWEEN operator requires exactly two values")
return ""
}
lo := v.buildValue(valuesCtx[0])
hi := v.buildValue(valuesCtx[1])
for _, val := range []any{lo, hi} {
switch val.(type) {
case int64, float32, float64:
// ok
default:
v.errors = append(v.errors, "BETWEEN operator requires numeric values")
return ""
}
}
keyName, err := getName(key)
if err != nil {
v.errors = append(v.errors, err.Error())
return ""
}
loStr := exprFormattedValue(lo)
hiStr := exprFormattedValue(hi)
var filter string
if isNot {
filter = fmt.Sprintf("%s != nil && (%s < %s || %s > %s)", keyName, keyName, loStr, keyName, hiStr)
} else {
filter = fmt.Sprintf("%s != nil && %s >= %s && %s <= %s", keyName, keyName, loStr, keyName, hiStr)
}
if _, err := expr.Compile(filter); err != nil {
v.errors = append(v.errors, err.Error())
return ""
}
return filter
}
// buildFilterExpr converts key + op + value into a final expr-lang string.
func buildFilterExpr(key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) (string, error) {
keyName, err := getName(key)
if err != nil {
return "", err
}
fmtValue := exprFormattedValue(value)
var filter string
switch op {
case qbtypes.FilterOperatorContains, qbtypes.FilterOperatorNotContains:
// contains / not contains must be case-insensitive to match query-time behaviour.
filter = fmt.Sprintf("lower(%s) %s lower(%s)", keyName, logOperatorsToExpr[op], fmtValue)
case qbtypes.FilterOperatorLike:
filter = fmt.Sprintf(`type(%s) != "map" && like(string(%s), %s)`, keyName, keyName, fmtValue)
case qbtypes.FilterOperatorNotLike:
filter = fmt.Sprintf(`type(%s) != "map" && not like(string(%s), %s)`, keyName, keyName, fmtValue)
case qbtypes.FilterOperatorILike:
filter = fmt.Sprintf(`type(%s) != "map" && ilike(string(%s), %s)`, keyName, keyName, fmtValue)
case qbtypes.FilterOperatorNotILike:
filter = fmt.Sprintf(`type(%s) != "map" && not ilike(string(%s), %s)`, keyName, keyName, fmtValue)
default:
filter = fmt.Sprintf("%s %s %s", keyName, logOperatorsToExpr[op], fmtValue)
}
// Avoid running operators on nil values (except equality, which handles nil fine).
if op != qbtypes.FilterOperatorEqual && op != qbtypes.FilterOperatorNotEqual {
filter = fmt.Sprintf("%s != nil && %s", keyName, filter)
}
return filter, nil
}
// buildKey turns a key grammar context into a TelemetryFieldKey.
func (v *exprVisitor) buildKey(ctx grammar.IKeyContext) *telemetrytypes.TelemetryFieldKey {
if ctx == nil {
return nil
}
key := telemetrytypes.GetFieldKeyFromKeyText(ctx.GetText())
return &key
}
func (v *exprVisitor) buildValuesFromInClause(in grammar.IInClauseContext, notIn grammar.INotInClauseContext) []any {
var ctxVal any
if in != nil {
ctxVal = v.VisitInClause(in)
} else if notIn != nil {
ctxVal = v.VisitNotInClause(notIn)
}
switch ret := ctxVal.(type) {
case []any:
return ret
case any:
if ret != nil {
return []any{ret}
}
}
return nil
}
func (v *exprVisitor) VisitInClause(ctx grammar.IInClauseContext) any {
if ctx.ValueList() != nil {
return v.Visit(ctx.ValueList())
}
return v.Visit(ctx.Value())
}
func (v *exprVisitor) VisitNotInClause(ctx grammar.INotInClauseContext) any {
if ctx.ValueList() != nil {
return v.Visit(ctx.ValueList())
}
return v.Visit(ctx.Value())
}
func (v *exprVisitor) VisitValueList(ctx grammar.IValueListContext) any {
values := ctx.AllValue()
parts := make([]any, 0, len(values))
for _, val := range values {
parts = append(parts, v.Visit(val))
}
return parts
}
func (v *exprVisitor) VisitValue(ctx *grammar.ValueContext) any {
return v.buildValue(ctx)
}
func (v *exprVisitor) VisitKey(ctx *grammar.KeyContext) any {
return v.buildKey(ctx)
}
func (v *exprVisitor) VisitFunctionCall(_ *grammar.FunctionCallContext) any {
v.errors = append(v.errors, "function calls are not supported in expr expressions")
return ""
}
func Parse(filters *v3.FilterSet) (string, error) {
var res []string
for _, v := range filters.Items {
if _, ok := logOperatorsToExpr[v.Operator]; !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "operator not supported: %s", v.Operator)
}
name := getName(v.Key)
var filter string
switch v.Operator {
// uncomment following lines when new version of expr is used
// case v3.FilterOperatorIn, v3.FilterOperatorNotIn:
// filter = fmt.Sprintf("%s %s list%s", name, logOperatorsToExpr[v.Operator], exprFormattedValue(v.Value))
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
// accustom log filters like `body.log.message EXISTS` into EXPR language
// where User is attempting to check for keys present in JSON log body
if strings.HasPrefix(v.Key.Key, "body.") {
// if body is a string and is a valid JSON, then check if the key exists in the JSON
filter = fmt.Sprintf(`((type(body) == "string" && isJSON(body)) && %s %s %s)`, exprFormattedValue(strings.TrimPrefix(v.Key.Key, "body.")), logOperatorsToExpr[v.Operator], "fromJSON(body)")
// if body is a map, then check if the key exists in the map
operator := v3.FilterOperatorNotEqual
if v.Operator == v3.FilterOperatorNotExists {
operator = v3.FilterOperatorEqual
}
nilCheckFilter := fmt.Sprintf("%s %s nil", v.Key.Key, logOperatorsToExpr[operator])
// join the two filters with OR
filter = fmt.Sprintf(`(%s or (type(body) == "map" && (%s)))`, filter, nilCheckFilter)
} else if typ := getTypeName(v.Key.Type); typ != "" {
filter = fmt.Sprintf("%s %s %s", exprFormattedValue(v.Key.Key), logOperatorsToExpr[v.Operator], typ)
} else {
// if type of key is not available; is considered as TOP LEVEL key in OTEL Log Data model hence
// switch Exist and Not Exists operators with NOT EQUAL and EQUAL respectively
operator := v3.FilterOperatorNotEqual
if v.Operator == v3.FilterOperatorNotExists {
operator = v3.FilterOperatorEqual
}
filter = fmt.Sprintf("%s %s nil", v.Key.Key, logOperatorsToExpr[operator])
}
default:
filter = fmt.Sprintf("%s %s %s", name, logOperatorsToExpr[v.Operator], exprFormattedValue(v.Value))
if v.Operator == v3.FilterOperatorContains || v.Operator == v3.FilterOperatorNotContains {
// `contains` and `ncontains` should be case insensitive to match how they work when querying logs.
filter = fmt.Sprintf(
"lower(%s) %s lower(%s)",
name, logOperatorsToExpr[v.Operator], exprFormattedValue(v.Value),
)
}
// Avoid running operators on nil values
if v.Operator != v3.FilterOperatorEqual && v.Operator != v3.FilterOperatorNotEqual {
filter = fmt.Sprintf("%s != nil && %s", name, filter)
}
}
// check if the filter is a correct expression language
_, err := expr.Compile(filter)
if err != nil {
return "", err
}
res = append(res, filter)
}
// check the final filter
q := strings.Join(res, " "+strings.ToLower(filters.Operator)+" ")
_, err := expr.Compile(q)
if err != nil {
return "", errors.WrapInternalf(err, CodeExprCompilationFailed, "failed to compile expression: %s", q)
}
return q, nil
func (v *exprVisitor) VisitFunctionParamList(_ *grammar.FunctionParamListContext) any {
v.errors = append(v.errors, "function calls are not supported in expr expressions")
return ""
}
func exprFormattedValue(v interface{}) string {
func (v *exprVisitor) VisitFunctionParam(_ *grammar.FunctionParamContext) any {
v.errors = append(v.errors, "function calls are not supported in expr expressions")
return ""
}
func (v *exprVisitor) VisitArray(_ *grammar.ArrayContext) any {
v.errors = append(v.errors, "array literals are not supported in expr expressions")
return ""
}
func (v *exprVisitor) VisitFullText(_ *grammar.FullTextContext) any {
v.errors = append(v.errors, "full-text search is not supported in expr expressions")
return ""
}
// buildValue converts a grammar VALUE token into a Go type.
func (v *exprVisitor) buildValue(ctx grammar.IValueContext) any {
switch {
case ctx == nil:
return nil
case ctx.QUOTED_TEXT() != nil:
return trimQuotes(ctx.QUOTED_TEXT().GetText())
case ctx.NUMBER() != nil:
text := ctx.NUMBER().GetText()
if i, err := strconv.ParseInt(text, 10, 64); err == nil {
return i
}
f, err := strconv.ParseFloat(text, 64)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to parse number %s", text))
return nil
}
return f
case ctx.BOOL() != nil:
return strings.ToLower(ctx.BOOL().GetText()) == "true"
case ctx.KEY() != nil:
return ctx.KEY().GetText()
default:
return nil
}
}
// trimQuotes removes surrounding single or double quotes from a token and
// unescapes embedded escape sequences produced by the ANTLR lexer.
func trimQuotes(txt string) string {
if len(txt) >= 2 {
if (txt[0] == '"' && txt[len(txt)-1] == '"') ||
(txt[0] == '\'' && txt[len(txt)-1] == '\'') {
txt = txt[1 : len(txt)-1]
}
}
txt = strings.ReplaceAll(txt, `\\`, `\`)
txt = strings.ReplaceAll(txt, `\'`, `'`)
return txt
}
// Parse converts the QB filter Expression (query builder expression string) into
// the Expr expression string used by the collector. It runs the ANTLR parser
// directly and produces the output in a single visitor pass, without building an
// intermediate FilterExprNode / FilterCondition tree.
func Parse(filter *qbtypes.Filter) (string, error) {
if filter == nil || strings.TrimSpace(filter.Expression) == "" {
return "", nil
}
input := antlr.NewInputStream(filter.Expression)
lexer := grammar.NewFilterQueryLexer(input)
lexerErrorListener := querybuilder.NewErrorListener()
lexer.RemoveErrorListeners()
lexer.AddErrorListener(lexerErrorListener)
tokens := antlr.NewCommonTokenStream(lexer, 0)
parserErrorListener := querybuilder.NewErrorListener()
parser := grammar.NewFilterQueryParser(tokens)
parser.RemoveErrorListeners()
parser.AddErrorListener(parserErrorListener)
tree := parser.Query()
if len(parserErrorListener.SyntaxErrors) > 0 {
combinedErrors := errors.Newf(
errors.TypeInvalidInput,
errors.CodeInvalidInput,
"Found %d syntax errors while parsing the search expression.",
len(parserErrorListener.SyntaxErrors),
)
additionals := make([]string, 0, len(parserErrorListener.SyntaxErrors))
for _, err := range parserErrorListener.SyntaxErrors {
if err.Error() != "" {
additionals = append(additionals, err.Error())
}
}
return "", combinedErrors.WithAdditional(additionals...)
}
visitor := &exprVisitor{}
result, _ := visitor.Visit(tree).(string)
if len(visitor.errors) > 0 {
combinedErrors := errors.Newf(
errors.TypeInvalidInput,
errors.CodeInvalidInput,
"Found %d errors while building expr expression.",
len(visitor.errors),
)
return "", combinedErrors.WithAdditional(visitor.errors...)
}
if _, err := expr.Compile(result); err != nil {
return "", errors.WrapUnexpectedf(err, CodeExprCompilationFailed, "failed to compile expression: %s", result)
}
return result, nil
}
func exprFormattedValue(v any) string {
switch x := v.(type) {
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
return fmt.Sprintf("%d", x)
case float32, float64:
return fmt.Sprintf("%f", x)
return fmt.Sprintf("%v", x)
case string:
return fmt.Sprintf("\"%s\"", quoteEscapedString(x))
case bool:
return fmt.Sprintf("%v", x)
case []interface{}:
case []any:
if len(x) == 0 {
return ""
}
switch x[0].(type) {
case string:
str := "["
for idx, sVal := range x {
str += fmt.Sprintf("'%s'", quoteEscapedString(sVal.(string)))
if idx != len(x)-1 {
str += ","
}
parts := make([]string, len(x))
for i, sVal := range x {
parts[i] = fmt.Sprintf("'%s'", quoteEscapedString(sVal.(string)))
}
str += "]"
return str
return "[" + strings.Join(parts, ",") + "]"
case uint8, uint16, uint32, uint64, int, int8, int16, int32, int64, float32, float64, bool:
return strings.Join(strings.Fields(fmt.Sprint(x)), ",")
default:

View File

@@ -4,7 +4,7 @@ import (
"testing"
signozstanzahelper "github.com/SigNoz/signoz-otel-collector/processor/signozlogspipelineprocessor/stanza/operator/helper"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/expr-lang/expr/vm"
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/entry"
"github.com/stretchr/testify/assert"
@@ -13,185 +13,235 @@ import (
func TestParseExpression(t *testing.T) {
var testCases = []struct {
Name string
Query *v3.FilterSet
Query *qbtypes.Filter
Expr string
ExpectError bool
}{
{
Name: "equal",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "checkbody", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key = 'checkbody'",
},
Expr: `attributes["key"] == "checkbody"`,
},
{
Name: "NOT equal (unary NOT)",
Query: &qbtypes.Filter{
Expression: "NOT (attribute.key = 'checkbody')",
},
Expr: `not (attributes["key"] == "checkbody")`,
},
{
Name: "not equal",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "checkbody", Operator: "!="},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key != 'checkbody'",
},
Expr: `attributes["key"] != "checkbody"`,
},
{
Name: "less than",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key < 10",
},
Expr: `attributes["key"] != nil && attributes["key"] < 10`,
},
{
Name: "greater than",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: ">"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key > 10",
},
Expr: `attributes["key"] != nil && attributes["key"] > 10`,
},
{
Name: "less than equal",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<="},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key <= 10",
},
Expr: `attributes["key"] != nil && attributes["key"] <= 10`,
},
{
Name: "greater than equal",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: ">="},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key >= 10",
},
Expr: `attributes["key"] != nil && attributes["key"] >= 10`,
},
// case sensitive
{
Name: "body contains",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
}},
Query: &qbtypes.Filter{
Expression: "body contains 'checkbody'",
},
Expr: `body != nil && lower(body) contains lower("checkbody")`,
},
{
Name: "body.log.message exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body.log.message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "body.log.message exists",
},
Expr: `(((type(body) == "string" && isJSON(body)) && "log.message" in fromJSON(body)) or (type(body) == "map" && (body.log.message != nil)))`,
},
{
Name: "body.log.message not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body.log.message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "body.log.message not exists",
},
Expr: `(((type(body) == "string" && isJSON(body)) && "log.message" not in fromJSON(body)) or (type(body) == "map" && (body.log.message == nil)))`,
},
{
Name: "body ncontains",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "ncontains"},
}},
Name: "body NOT CONTAINS",
Query: &qbtypes.Filter{
Expression: "body NOT CONTAINS 'checkbody'",
},
Expr: `body != nil && lower(body) not contains lower("checkbody")`,
},
{
Name: "body regex",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex$", Operator: "regex"},
}},
Name: "body REGEXP",
Query: &qbtypes.Filter{
Expression: "body REGEXP '[0-1]+regex$'",
},
Expr: `body != nil && body matches "[0-1]+regex$"`,
},
{
Name: "body not regex",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex$", Operator: "nregex"},
}},
Name: "body NOT REGEXP",
Query: &qbtypes.Filter{
Expression: "body NOT REGEXP '[0-1]+regex$'",
},
Expr: `body != nil && body not matches "[0-1]+regex$"`,
},
{
Name: "regex with escape characters",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: `^Executing \[\S+@\S+:[0-9]+\] \S+".*`, Operator: "regex"},
}},
Query: &qbtypes.Filter{
Expression: "body REGEXP '^Executing \\[\\S+@\\S+:[0-9]+\\] \\S+\".*'",
},
Expr: `body != nil && body matches "^Executing \\[\\S+@\\S+:[0-9]+\\] \\S+\".*"`,
},
{
Name: "invalid regex",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-9]++", Operator: "nregex"},
}},
Expr: `body != nil && lower(body) not matches "[0-9]++"`,
Query: &qbtypes.Filter{
Expression: "body not REGEXP '[0-9]++'",
},
ExpectError: true,
},
{
Name: "in",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{1, 2, 3, 4}, Operator: "in"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key in [1,2,3,4]",
},
Expr: `attributes["key"] != nil && attributes["key"] in [1,2,3,4]`,
},
{
Name: "not in",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{"1", "2"}, Operator: "nin"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key not in ['1','2']",
},
Expr: `attributes["key"] != nil && attributes["key"] not in ['1','2']`,
},
{
Name: "exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key exists",
},
Expr: `"key" in attributes`,
},
{
Name: "not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key not exists",
},
Expr: `"key" not in attributes`,
},
{
Name: "trace_id not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "trace_id not exists",
},
Expr: `trace_id == nil`,
},
{
Name: "trace_id exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "trace_id exists",
},
Expr: `trace_id != nil`,
},
{
Name: "span_id not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "span_id not exists",
},
Expr: `span_id == nil`,
},
{
Name: "span_id exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "span_id exists",
},
Expr: `span_id != nil`,
},
{
Name: "Multi filter",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<="},
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex$", Operator: "nregex"},
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.key <= 10 and body not regexp '[0-1]+regex$' and attribute.key not exists",
},
Expr: `attributes["key"] != nil && attributes["key"] <= 10 and body != nil && body not matches "[0-1]+regex$" and "key" not in attributes`,
},
{
Name: "incorrect multi filter",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: 10, Operator: "<="},
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-9]++", Operator: "nregex"},
{Key: v3.AttributeKey{Key: "key", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
}},
Expr: `attributes["key"] != nil && attributes["key"] <= 10 and body not matches "[0-9]++" and "key" not in attributes`,
Query: &qbtypes.Filter{
Expression: "attribute.key <= 10 and body not regexp '[0-9]++' and attribute.key not exists",
},
ExpectError: true,
},
{
Name: "attributes. is unsupported",
Query: &qbtypes.Filter{
Expression: "attributes.key = 'checkbody'",
},
ExpectError: true,
},
{
Name: "LIKE",
Query: &qbtypes.Filter{
Expression: "attribute.key LIKE 'foo%'",
},
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && like(string(attributes["key"]), "foo%")`,
},
{
Name: "NOT LIKE",
Query: &qbtypes.Filter{
Expression: "attribute.key NOT LIKE 'foo%'",
},
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && not like(string(attributes["key"]), "foo%")`,
},
{
Name: "ILIKE",
Query: &qbtypes.Filter{
Expression: "attribute.key ILIKE 'FOO%'",
},
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && ilike(string(attributes["key"]), "FOO%")`,
},
{
Name: "NOT ILIKE",
Query: &qbtypes.Filter{
Expression: "attribute.key NOT ILIKE 'FOO%'",
},
Expr: `attributes["key"] != nil && type(attributes["key"]) != "map" && not ilike(string(attributes["key"]), "FOO%")`,
},
{
Name: "body LIKE",
Query: &qbtypes.Filter{
Expression: "body LIKE 'Server%'",
},
Expr: `body != nil && type(body) != "map" && like(string(body), "Server%")`,
},
{
Name: "body ILIKE",
Query: &qbtypes.Filter{
Expression: "body ILIKE 'server%'",
},
Expr: `body != nil && type(body) != "map" && ilike(string(body), "server%")`,
},
}
for _, tt := range testCases {
@@ -267,250 +317,360 @@ func TestExpressionVSEntry(t *testing.T) {
var testCases = []struct {
Name string
Query *v3.FilterSet
Query *qbtypes.Filter
ExpectedMatches []int
}{
{
Name: "resource equal (env)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "resource.env = 'prod'",
},
ExpectedMatches: []int{0, 1, 4, 5, 9, 11, 13, 14, 17, 18},
},
{
Name: "NOT resource equal (unary NOT)",
Query: &qbtypes.Filter{
Expression: "NOT (resource.env = 'prod')",
},
ExpectedMatches: []int{2, 3, 6, 7, 8, 10, 12, 15, 16, 19, 20},
},
{
Name: "resource not equal (env)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "!="},
}},
Query: &qbtypes.Filter{
Expression: "resource.env != 'prod'",
},
ExpectedMatches: []int{2, 3, 6, 7, 8, 10, 12, 15, 16, 19, 20},
},
{
Name: "attribute less than (numeric)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 8, Operator: "<"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.count < 8",
},
ExpectedMatches: []int{4},
},
{
Name: "attribute greater than (numeric)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "count", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag}, Value: 8, Operator: ">"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.count > 8",
},
ExpectedMatches: []int{5},
},
{
Name: "body contains (case insensitive)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
}},
Query: &qbtypes.Filter{
Expression: "body contains 'checkbody'",
},
ExpectedMatches: []int{2, 9, 10, 16},
},
{
Name: "body ncontains",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "ncontains"},
}},
Name: "body NOT CONTAINS",
Query: &qbtypes.Filter{
Expression: "body NOT CONTAINS 'checkbody'",
},
ExpectedMatches: []int{0, 1, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 17},
},
{
Name: "body.msg (case insensitive)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body.msg", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: false}, Value: "checkbody", Operator: "contains"},
}},
Query: &qbtypes.Filter{
Expression: "body.msg contains 'checkbody'",
},
ExpectedMatches: []int{2, 9, 10, 18},
},
{
Name: "body regex",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex", Operator: "regex"},
}},
Name: "body REGEXP",
Query: &qbtypes.Filter{
Expression: "body REGEXP '[0-1]+regex'",
},
ExpectedMatches: []int{4},
},
{
Name: "body not regex",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex", Operator: "nregex"},
}},
Name: "body NOT REGEXP",
Query: &qbtypes.Filter{
Expression: "body NOT REGEXP '[0-1]+regex'",
},
ExpectedMatches: []int{0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17},
},
// body.log.message exists/nexists: expr checks "log.message" in fromJSON(body); nested key
// semantics depend on signoz stanza helper. Omitted here to avoid coupling to env shape.
{
Name: "body top-level key exists (body.msg)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body.msg", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "body.msg exists",
},
ExpectedMatches: []int{0, 1, 2, 3, 4, 5, 9, 10, 11, 12, 18},
},
{
Name: "body top-level key not exists (body.missing)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body.missing", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "body.missing not exists",
},
ExpectedMatches: []int{0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 18, 20},
},
{
Name: "attribute exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.service exists",
},
ExpectedMatches: []int{6, 7, 8, 15},
},
{
Name: "attribute not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.service not exists",
},
ExpectedMatches: []int{0, 1, 2, 3, 4, 5, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20},
},
{
Name: "trace_id exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "trace_id exists",
},
ExpectedMatches: []int{1, 2, 5, 7, 12, 15, 19},
},
{
Name: "trace_id not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "trace_id not exists",
},
ExpectedMatches: []int{0, 3, 4, 6, 8, 9, 10, 11, 13, 14, 16, 17, 18, 20},
},
{
Name: "span_id exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "span_id exists",
},
ExpectedMatches: []int{1, 3, 5, 12, 17, 20},
},
{
Name: "span_id not exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "span_id not exists",
},
ExpectedMatches: []int{0, 2, 4, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 18, 19},
},
{
Name: "in (attribute in list)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{"info", "error"}, Operator: "in"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.level in ['info', 'error']",
},
ExpectedMatches: []int{0, 1, 2, 14, 16, 20},
},
{
Name: "not in (attribute not in list)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: []any{"error", "warn"}, Operator: "nin"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.level not in ['error', 'warn']",
},
ExpectedMatches: []int{0, 2, 3, 16, 20},
},
{
Name: "multi filter AND",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "staging", Operator: "="},
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "info", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "resource.env = 'staging' and attribute.level = 'info'",
},
ExpectedMatches: []int{2, 16},
},
{
Name: "multi filter AND (two attributes)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "auth", Operator: "="},
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: "nexists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.service = 'auth' and attribute.level not exists",
},
ExpectedMatches: []int{6, 7},
},
// Multi-filter variations: body + attribute, three conditions, trace/span + attribute
{
Name: "multi filter AND body contains + attribute",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "Connection", Operator: "contains"},
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "body contains 'Connection' and resource.env = 'prod'",
},
ExpectedMatches: []int{14},
},
{
Name: "multi filter AND body contains + service",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "login", Operator: "contains"},
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "auth", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "body contains 'login' and attribute.service = 'auth'",
},
ExpectedMatches: []int{6, 15},
},
{
Name: "multi filter AND env + level (prod error)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "error", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "resource.env = 'prod' and attribute.level = 'error'",
},
ExpectedMatches: []int{1, 14},
},
{
Name: "multi filter AND three conditions (staging + checkbody + info)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "staging", Operator: "="},
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "info", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "resource.env = 'staging' and body contains 'checkbody' and attribute.level = 'info'",
},
ExpectedMatches: []int{2, 16},
},
{
Name: "multi filter AND trace_id exists + body contains",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "checkbody", Operator: "contains"},
}},
Query: &qbtypes.Filter{
Expression: "trace_id exists and body contains 'checkbody'",
},
ExpectedMatches: []int{2},
},
{
Name: "multi filter AND span_id nexists + service auth",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
{Key: v3.AttributeKey{Key: "service", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "auth", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "span_id not exists and attribute.service = 'auth'",
},
ExpectedMatches: []int{6, 7, 15},
},
{
Name: "multi filter AND body regex + attribute",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "[0-1]+regex", Operator: "regex"},
{Key: v3.AttributeKey{Key: "code", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "200", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "body regexp '[0-1]+regex' and attribute.code = '200'",
},
ExpectedMatches: []int{4},
},
{
Name: "multi filter AND no trace_id + no span_id + env prod",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
{Key: v3.AttributeKey{Key: "span_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "nexists"},
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "prod", Operator: "="},
}},
Query: &qbtypes.Filter{
Expression: "trace_id not exists and span_id not exists and resource.env = 'prod'",
},
ExpectedMatches: []int{0, 4, 9, 11, 13, 14, 18},
},
{
Name: "multi filter AND level warn + body contains",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "warn", Operator: "="},
{Key: v3.AttributeKey{Key: "body", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "disk", Operator: "contains"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.level = 'warn' and body contains 'disk'",
},
ExpectedMatches: []int{17},
},
{
Name: "no matches (attribute value not present)",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource}, Value: "never", Operator: "="},
}},
Name: "no matches (resource value not present)",
Query: &qbtypes.Filter{
Expression: "resource.env = 'never'",
},
ExpectedMatches: []int{},
},
{
Name: "attribute equal and trace_id exists",
Query: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{
{Key: v3.AttributeKey{Key: "code", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Value: "404", Operator: "="},
{Key: v3.AttributeKey{Key: "trace_id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeUnspecified, IsColumn: true}, Value: "", Operator: "exists"},
}},
Query: &qbtypes.Filter{
Expression: "attribute.code = '404' and trace_id exists",
},
ExpectedMatches: []int{5},
},
// LIKE / NOT LIKE / ILIKE / NOT ILIKE pattern and type-safety coverage
{
Name: "LIKE exact match (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.level LIKE 'info'",
},
ExpectedMatches: []int{0, 2, 16, 20},
},
{
Name: "LIKE prefix match (body)",
Query: &qbtypes.Filter{
Expression: "body LIKE 'Server%'",
},
ExpectedMatches: []int{13},
},
{
Name: "LIKE suffix match (body)",
Query: &qbtypes.Filter{
Expression: "body LIKE '%8080'",
},
ExpectedMatches: []int{13},
},
{
Name: "LIKE substring with % (body)",
Query: &qbtypes.Filter{
Expression: "body LIKE '%checkbody%'",
},
// map bodies excluded by type check; entries 18-20 have map body → no match
ExpectedMatches: []int{2, 9, 16},
},
{
Name: "LIKE single-char wildcard (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.level LIKE 'inf_'",
},
ExpectedMatches: []int{0, 2, 16, 20},
},
{
Name: "LIKE prefix on attribute",
Query: &qbtypes.Filter{
Expression: "attribute.service LIKE 'au%'",
},
ExpectedMatches: []int{6, 7, 15},
},
{
Name: "LIKE pattern on resource",
Query: &qbtypes.Filter{
Expression: "resource.host LIKE 'node-1%'",
},
ExpectedMatches: []int{1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19},
},
{
Name: "NOT LIKE exact match (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.level NOT LIKE 'info'",
},
ExpectedMatches: []int{1, 3, 14, 15, 17},
},
{
Name: "NOT LIKE suffix pattern (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.level NOT LIKE '%o'",
},
// level ending in 'o': info; not ending in 'o': error, debug, warn
ExpectedMatches: []int{1, 3, 14, 15, 17},
},
{
Name: "ILIKE case-insensitive exact (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.level ILIKE 'INFO'",
},
ExpectedMatches: []int{0, 2, 16, 20},
},
{
Name: "ILIKE case-insensitive prefix (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.status ILIKE 'ACT%'",
},
ExpectedMatches: []int{11},
},
{
Name: "ILIKE case-insensitive substring (body)",
Query: &qbtypes.Filter{
Expression: "body ILIKE '%checkbody%'",
},
// map bodies excluded by type check
ExpectedMatches: []int{2, 9, 10, 16},
},
{
Name: "NOT ILIKE case-insensitive exact (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.level NOT ILIKE 'INFO'",
},
ExpectedMatches: []int{1, 3, 14, 15, 17},
},
{
Name: "NOT ILIKE case-insensitive (attribute)",
Query: &qbtypes.Filter{
Expression: "attribute.service NOT ILIKE 'API'",
},
ExpectedMatches: []int{6, 7, 15},
},
{
Name: "LIKE numeric attribute converted via string()",
Query: &qbtypes.Filter{
Expression: "attribute.count LIKE '%5%'",
},
// count int64(5) -> string "5" matches; count int64(10) -> "10" does not
ExpectedMatches: []int{4},
},
{
Name: "LIKE with multi filter (attribute + resource)",
Query: &qbtypes.Filter{
Expression: "attribute.level LIKE 'info' and resource.env = 'prod'",
},
ExpectedMatches: []int{0},
},
}
for _, tt := range testCases {

View File

@@ -169,6 +169,7 @@ func NewSQLMigrationProviderFactories(
sqlmigration.NewAddAnonymousPublicDashboardTransactionFactory(sqlstore),
sqlmigration.NewAddRootUserFactory(sqlstore, sqlschema),
sqlmigration.NewAddUserEmailOrgIDIndexFactory(sqlstore, sqlschema),
sqlmigration.NewMigratePipelineFiltersV5Factory(sqlschema),
)
}

View File

@@ -0,0 +1,182 @@
package sqlmigration
import (
"context"
"encoding/json"
"log/slog"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/queryBuilderToExpr"
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/transition"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
)
type migratePipelineFiltersV5 struct {
sqlschema sqlschema.SQLSchema
logger *slog.Logger
}
func NewMigratePipelineFiltersV5Factory(
sqlschema sqlschema.SQLSchema,
) factory.ProviderFactory[SQLMigration, Config] {
return factory.NewProviderFactory(
factory.MustNewName("migrate_pipeline_filters_v5"),
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
return newMigratePipelineFiltersV5(ctx, c, sqlschema, ps.Logger)
},
)
}
func newMigratePipelineFiltersV5(
_ context.Context,
_ Config,
sqlschema sqlschema.SQLSchema,
logger *slog.Logger,
) (SQLMigration, error) {
if logger == nil {
logger = slog.Default()
}
return &migratePipelineFiltersV5{
sqlschema: sqlschema,
logger: logger,
}, nil
}
func (migration *migratePipelineFiltersV5) Register(migrations *migrate.Migrations) error {
if err := migrations.Register(migration.Up, migration.Down); err != nil {
return err
}
return nil
}
// pipelineFilterRow is used only during migration to read filter_deprecated and write to filter.
type pipelineFilterRow struct {
ID string `bun:"id,pk,type:text"`
FilterDeprecated string `bun:"filter_deprecated,type:text,notnull"`
}
func (migration *migratePipelineFiltersV5) Up(ctx context.Context, db *bun.DB) error {
table, uniqueConstraints, err := migration.sqlschema.GetTable(ctx, "pipelines")
if err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
// 1. Rename existing filter column to filter_deprecated
for _, sql := range migration.sqlschema.Operator().RenameColumn(table, &sqlschema.Column{Name: "filter"}, "filter_deprecated") {
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
return err
}
}
// 2. Add new filter column (v5); existing rows get default ''
for _, sql := range migration.sqlschema.Operator().AddColumn(table, uniqueConstraints, &sqlschema.Column{
Name: "filter",
DataType: sqlschema.DataTypeText,
Nullable: false,
}, "") {
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
return err
}
}
// 3. Copy v5 filter data: read from filter_deprecated, migrate to v5 expression, write to filter
var rows []pipelineFilterRow
if err := tx.NewSelect().
Table("pipelines").
Column("id", "filter_deprecated").
Scan(ctx, &rows); err != nil {
return err
}
for _, r := range rows {
raw := strings.TrimSpace(r.FilterDeprecated)
if raw == "" {
return errors.NewInternalf(errors.CodeInternal, "filter_deprecated is empty")
}
var filterSet v3.FilterSet
if err := json.Unmarshal([]byte(raw), &filterSet); err != nil {
return err
}
expr, migrated, err := transition.BuildFilterExpressionFromFilterSet(ctx, migration.logger, "logs", &filterSet)
if err != nil || !migrated || strings.TrimSpace(expr) == "" {
return err
}
filter := &qbtypes.Filter{Expression: expr}
if _, err = queryBuilderToExpr.Parse(filter); err != nil {
return err
}
out, err := json.Marshal(filter)
if err != nil {
return err
}
if _, err := tx.NewUpdate().
Table("pipelines").
Set("filter = ?", string(out)).
Where("id = ?", r.ID).
Exec(ctx); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
return nil
}
func (migration *migratePipelineFiltersV5) Down(ctx context.Context, db *bun.DB) error {
table, _, err := migration.sqlschema.GetTable(ctx, "pipelines")
if err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
_ = tx.Rollback()
}()
// 1. Drop the new filter column
for _, sql := range migration.sqlschema.Operator().DropColumn(table, &sqlschema.Column{Name: "filter"}) {
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
return err
}
}
// 2. Rename filter_deprecated back to filter
for _, sql := range migration.sqlschema.Operator().RenameColumn(table, &sqlschema.Column{Name: "filter_deprecated"}, "filter") {
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
return err
}
}
if err := tx.Commit(); err != nil {
return err
}
return nil
}

View File

@@ -118,6 +118,19 @@ func (column *Column) ToUpdateSQL(fmter SQLFormatter, tableName TableName, value
return sql
}
func (column *Column) ToRenameSQL(fmter SQLFormatter, tableName TableName, newName ColumnName) []byte {
sql := []byte{}
sql = append(sql, "ALTER TABLE "...)
sql = fmter.AppendIdent(sql, string(tableName))
sql = append(sql, " RENAME COLUMN "...)
sql = fmter.AppendIdent(sql, string(column.Name))
sql = append(sql, " TO "...)
sql = fmter.AppendIdent(sql, string(newName))
return sql
}
func (column *Column) ToSetNotNullSQL(fmter SQLFormatter, tableName TableName) []byte {
sql := []byte{}

View File

@@ -95,6 +95,19 @@ func (operator *Operator) DropColumn(table *Table, column *Column) [][]byte {
return [][]byte{column.ToDropSQL(operator.fmter, table.Name, operator.support.ColumnIfNotExistsExists)}
}
func (operator *Operator) RenameColumn(table *Table, column *Column, newName ColumnName) [][]byte {
index := operator.findColumnByName(table, column.Name)
// If the column does not exist, we do not need to rename it.
if index == -1 {
return [][]byte{}
}
sql := column.ToRenameSQL(operator.fmter, table.Name, newName)
table.Columns[index].Name = newName
return [][]byte{sql}
}
func (operator *Operator) DropConstraint(table *Table, uniqueConstraints []*UniqueConstraint, constraint Constraint) [][]byte {
// The name of the input constraint is not guaranteed to be the same as the name of the constraint in the database.
// So we need to find the constraint in the database and drop it.

View File

@@ -50,6 +50,9 @@ type SQLOperator interface {
// Returns a list of SQL statements to drop a column from a table.
DropColumn(*Table, *Column) [][]byte
// Returns a list of SQL statements to rename a column in a table.
RenameColumn(*Table, *Column, ColumnName) [][]byte
// Returns a list of SQL statements to drop a constraint from a table.
DropConstraint(*Table, []*UniqueConstraint, Constraint) [][]byte
}

View File

@@ -0,0 +1,79 @@
package transition
import (
"context"
"encoding/json"
"log/slog"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
)
// BuildFilterExpressionFromFilterSet converts a v3-style FilterSet JSON
// ({"op": "...", "items": [...]}) into a v5-style filter expression string
// (for example: "attribute.http.method = 'GET' AND resource.env = 'prod'").
//
// It reuses migrateCommon.createFilterExpression so that all the existing
// semantics around operators, variables, data types, and ambiguity handling
// are preserved.
//
// dataSource determines which ambiguity set to use ("logs", "traces", etc.).
// For log pipelines, pass "logs".
//
// Returns:
// - expression: the generated filter expression string
// - migrated: true if an expression was generated, false if there was
// nothing to migrate (e.g. empty filters)
// - err: non-nil only if the input JSON could not be parsed
func BuildFilterExpressionFromFilterSet(
ctx context.Context,
logger *slog.Logger,
dataSource string,
filterSet *v3.FilterSet,
) (expression string, migrated bool, err error) {
if filterSet == nil {
return "", false, nil
}
filterJSON, err := json.Marshal(filterSet)
if err != nil {
return "", false, err
}
var filters map[string]any
if err := json.Unmarshal([]byte(filterJSON), &filters); err != nil {
return "", false, err
}
mc := NewMigrateCommon(logger)
// add keys with type into ambiguity set to preserve context in generated expression
for _, item := range filterSet.Items {
if item.Key.Type == v3.AttributeKeyTypeUnspecified {
continue
}
mc.ambiguity[dataSource] = append(mc.ambiguity[dataSource], item.Key.Key)
}
// Shape expected by migrateCommon.createFilterExpression:
// queryData["dataSource"] == "logs" | "traces" | "metrics"
// queryData["filters"] == map[string]any{"op": "...", "items": [...]}
queryData := map[string]any{
"dataSource": dataSource,
"filters": filters,
}
if !mc.createFilterExpression(ctx, queryData) {
return "", false, nil
}
filterAny, ok := queryData["filter"].(map[string]any)
if !ok {
return "", false, nil
}
expr, ok := filterAny["expression"].(string)
if !ok || expr == "" {
return "", false, nil
}
return expr, true, nil
}

View File

@@ -0,0 +1,220 @@
package transition
import (
"context"
"log/slog"
"testing"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
)
func TestBuildFilterExpressionFromFilterSet(t *testing.T) {
ctx := context.Background()
logger := slog.Default()
tests := []struct {
name string
dataSource string
filterSet *v3.FilterSet
wantExpr string
wantMigrated bool
wantErr bool
}{
{
name: "empty filter JSON",
dataSource: "logs",
filterSet: &v3.FilterSet{},
wantExpr: "",
wantMigrated: false,
wantErr: false,
},
{
name: "empty items array",
dataSource: "logs",
filterSet: &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
wantExpr: "",
wantMigrated: false,
wantErr: false,
},
{
name: "multiple filter items with AND operator",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorEqual,
Value: "GET",
},
{
Key: v3.AttributeKey{Key: "environment", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
Operator: v3.FilterOperatorEqual,
Value: "prod",
},
},
},
wantExpr: `(attribute.http.method = 'GET' AND resource.environment = 'prod')`,
wantMigrated: true,
wantErr: false,
},
{
name: "multiple filter items with OR operator",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "OR",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorEqual,
Value: "error",
},
{
Key: v3.AttributeKey{Key: "level", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorEqual,
Value: "warn",
},
},
},
wantExpr: `(attribute.level = 'error' OR attribute.level = 'warn')`,
wantMigrated: true,
wantErr: false,
},
{
name: "in operator with array value",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorIn,
Value: []string{"api", "web", "worker"},
}},
},
wantExpr: `attribute.service.name IN ['api', 'web', 'worker']`,
wantMigrated: true,
wantErr: false,
},
{
name: "contains operator",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{Key: "message", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorContains,
Value: "error",
}},
},
wantExpr: `attribute.message CONTAINS 'error'`,
wantMigrated: true,
wantErr: false,
},
{
name: "not exists operator",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{Key: "trace.id", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorNotExists,
Value: nil,
}},
},
wantExpr: `attribute.trace.id NOT EXISTS`,
wantMigrated: true,
wantErr: false,
},
{
name: "regex operator",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{Key: "body"},
Operator: v3.FilterOperatorRegex,
Value: ".*",
}},
},
wantExpr: `body REGEXP '.*'`,
wantMigrated: true,
wantErr: false,
},
{
name: "has operator",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{Key: "tags", DataType: v3.AttributeKeyDataTypeArrayString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorHas,
Value: "production",
}},
},
wantExpr: `has(attribute.tags, 'production')`,
wantMigrated: true,
wantErr: false,
},
{
name: "complex filter with multiple operators",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{Key: "http.method", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorEqual,
Value: "POST",
},
{
Key: v3.AttributeKey{Key: "http.status_code", DataType: v3.AttributeKeyDataTypeInt64, Type: v3.AttributeKeyTypeTag},
Operator: v3.FilterOperatorGreaterThanOrEq,
Value: float64(400),
},
{
Key: v3.AttributeKey{Key: "resource.env", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
Operator: v3.FilterOperatorEqual,
Value: "prod",
},
},
},
wantExpr: `(attribute.http.method = 'POST' AND attribute.http.status_code >= 400 AND resource.resource.env = 'prod')`,
wantMigrated: true,
wantErr: false,
},
{
name: "filter with resource type (non-ambiguous key)",
dataSource: "logs",
filterSet: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{{
Key: v3.AttributeKey{Key: "service.name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeResource},
Operator: v3.FilterOperatorEqual,
Value: "frontend",
}},
},
wantExpr: `resource.service.name = 'frontend'`,
wantMigrated: true,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
expr, migrated, err := BuildFilterExpressionFromFilterSet(ctx, logger, tt.dataSource, tt.filterSet)
if (err != nil) != tt.wantErr {
t.Errorf("BuildFilterExpressionFromFilterSet() error = %v, wantErr %v", err, tt.wantErr)
return
}
if migrated != tt.wantMigrated {
t.Errorf("BuildFilterExpressionFromFilterSet() migrated = %v, want %v", migrated, tt.wantMigrated)
}
if expr != tt.wantExpr {
t.Errorf("BuildFilterExpressionFromFilterSet() expression = %v, want %v", expr, tt.wantExpr)
}
})
}
}

View File

@@ -19,7 +19,8 @@ type migrateCommon struct {
func NewMigrateCommon(logger *slog.Logger) *migrateCommon {
return &migrateCommon{
logger: logger,
logger: logger,
ambiguity: make(map[string][]string),
}
}

View File

@@ -7,9 +7,9 @@ import (
"strings"
"github.com/SigNoz/signoz/pkg/errors"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/queryBuilderToExpr"
"github.com/SigNoz/signoz/pkg/types"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/uptrace/bun"
)
@@ -57,7 +57,7 @@ type StoreablePipeline struct {
type GettablePipeline struct {
StoreablePipeline
Filter *v3.FilterSet `json:"filter"`
Filter *qbtypes.Filter `json:"filter"`
Config []PipelineOperator `json:"config"`
}
@@ -72,7 +72,7 @@ func (i *GettablePipeline) ParseRawConfig() error {
}
func (i *GettablePipeline) ParseFilter() error {
f := v3.FilterSet{}
f := qbtypes.Filter{}
err := json.Unmarshal([]byte(i.FilterString), &f)
if err != nil {
return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to parse filter")
@@ -200,7 +200,7 @@ type PostablePipeline struct {
Alias string `json:"alias"`
Description string `json:"description"`
Enabled bool `json:"enabled"`
Filter *v3.FilterSet `json:"filter"`
Filter *qbtypes.Filter `json:"filter"`
Config []PipelineOperator `json:"config"`
}
@@ -218,6 +218,14 @@ func (p *PostablePipeline) IsValid() error {
}
// check the filter
if p.Filter == nil || strings.TrimSpace(p.Filter.Expression) == "" {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "filter.expression is required")
}
// Validate that every field in the expression has an explicit context
// (attribute., resource., body., etc) so later pipeline processing does not
// need to guess. We do not validate that the field actually
// exists only that the context is specified.
_, err := queryBuilderToExpr.Parse(p.Filter)
if err != nil {
return err

View File

@@ -3,24 +3,13 @@ package pipelinetypes
import (
"testing"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
. "github.com/smartystreets/goconvey/convey"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/stretchr/testify/assert"
)
func TestIsValidPostablePipeline(t *testing.T) {
validPipelineFilterSet := &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeTag,
},
Operator: "=",
Value: "GET",
},
},
validPipelineFilter := &qbtypes.Filter{
Expression: `attribute.method = "GET"`,
}
var correctQueriesTest = []struct {
@@ -34,7 +23,7 @@ func TestIsValidPostablePipeline(t *testing.T) {
Name: "pipeline 1",
Alias: "pipeline1",
Enabled: true,
Filter: validPipelineFilterSet,
Filter: validPipelineFilter,
Config: []PipelineOperator{},
},
IsValid: false,
@@ -46,7 +35,7 @@ func TestIsValidPostablePipeline(t *testing.T) {
Name: "pipeline 1",
Alias: "pipeline1",
Enabled: true,
Filter: validPipelineFilterSet,
Filter: validPipelineFilter,
Config: []PipelineOperator{},
},
IsValid: false,
@@ -58,7 +47,7 @@ func TestIsValidPostablePipeline(t *testing.T) {
Name: "pipeline 1",
Alias: "pipeline1",
Enabled: true,
Filter: validPipelineFilterSet,
Filter: validPipelineFilter,
Config: []PipelineOperator{},
},
IsValid: true,
@@ -70,19 +59,21 @@ func TestIsValidPostablePipeline(t *testing.T) {
Name: "pipeline 1",
Alias: "pipeline1",
Enabled: true,
Filter: &v3.FilterSet{
Operator: "AND",
Items: []v3.FilterItem{
{
Key: v3.AttributeKey{
Key: "method",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeUnspecified,
},
Operator: "regex",
Value: "[0-9A-Z*",
},
},
Filter: &qbtypes.Filter{
Expression: "",
},
},
IsValid: false,
},
{
Name: "Filter without context prefix on field",
Pipeline: PostablePipeline{
OrderID: 1,
Name: "pipeline 1",
Alias: "pipeline1",
Enabled: true,
Filter: &qbtypes.Filter{
Expression: `method = "GET"`,
},
},
IsValid: false,
@@ -94,19 +85,19 @@ func TestIsValidPostablePipeline(t *testing.T) {
Name: "pipeline 1",
Alias: "pipeline1",
Enabled: true,
Filter: validPipelineFilterSet,
Filter: validPipelineFilter,
},
IsValid: true,
},
}
for _, test := range correctQueriesTest {
Convey(test.Name, t, func() {
t.Run(test.Name, func(t *testing.T) {
err := test.Pipeline.IsValid()
if test.IsValid {
So(err, ShouldBeNil)
assert.NoError(t, err)
} else {
So(err, ShouldBeError)
assert.Error(t, err)
}
})
}
@@ -365,12 +356,12 @@ var operatorTest = []struct {
func TestValidOperator(t *testing.T) {
for _, test := range operatorTest {
Convey(test.Name, t, func() {
t.Run(test.Name, func(t *testing.T) {
err := isValidOperator(test.Operator)
if test.IsValid {
So(err, ShouldBeNil)
assert.NoError(t, err)
} else {
So(err, ShouldBeError)
assert.Error(t, err)
}
})
}