mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-10 15:00:47 +01:00
Compare commits
1 Commits
feat/updat
...
feat/semco
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0c3223b23a |
17
pkg/querybuilder/clickhouse_quote.go
Normal file
17
pkg/querybuilder/clickhouse_quote.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package querybuilder
|
||||
|
||||
import "strings"
|
||||
|
||||
// ClickHouseStringLiteral quotes a value for a ClickHouse string literal.
|
||||
func ClickHouseStringLiteral(value string) string {
|
||||
escaped := strings.ReplaceAll(value, `\`, `\\`)
|
||||
escaped = strings.ReplaceAll(escaped, `'`, `\'`)
|
||||
return "'" + escaped + "'"
|
||||
}
|
||||
|
||||
// ClickHouseIdentifier quotes a value for a ClickHouse identifier.
|
||||
func ClickHouseIdentifier(value string) string {
|
||||
escaped := strings.ReplaceAll(value, `\`, `\\`)
|
||||
escaped = strings.ReplaceAll(escaped, "`", "\\`")
|
||||
return "`" + escaped + "`"
|
||||
}
|
||||
17
pkg/querybuilder/clickhouse_quote_test.go
Normal file
17
pkg/querybuilder/clickhouse_quote_test.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestClickHouseQuoting(t *testing.T) {
|
||||
t.Run("string literal", func(t *testing.T) {
|
||||
assert.Equal(t, `'name\'\\); SELECT 1 --'`, ClickHouseStringLiteral(`name'\); SELECT 1 --`))
|
||||
})
|
||||
|
||||
t.Run("identifier", func(t *testing.T) {
|
||||
assert.Equal(t, "`name\\`\\\\); SELECT 1 --`", ClickHouseIdentifier("name`\\); SELECT 1 --"))
|
||||
})
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package querybuilder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
@@ -9,6 +10,25 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
func physicalSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return []string{key.Name}
|
||||
}
|
||||
|
||||
func mapMemberExistenceExpression(column string, key *telemetrytypes.TelemetryFieldKey, member string) string {
|
||||
if materializedColumn, ok := key.SemconvMaterializedColumns[member]; ok {
|
||||
return ClickHouseIdentifier(materializedColumn + "_exists")
|
||||
}
|
||||
if key.Materialized && key.Name == member {
|
||||
physicalKey := key.Copy()
|
||||
physicalKey.Name = member
|
||||
return telemetrytypes.FieldKeyToMaterializedColumnNameForExists(physicalKey)
|
||||
}
|
||||
return fmt.Sprintf("mapContains(%s, %s)", column, ClickHouseStringLiteral(member))
|
||||
}
|
||||
|
||||
// ExistsExpression renders the existence predicate for a key resolved to the given
|
||||
// columns (negated when exists is false). Comparisons are against constants rendered
|
||||
// as literals, so the expression carries no bind args and can guard column expressions
|
||||
@@ -43,11 +63,26 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
|
||||
if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil {
|
||||
columnName = evolutionsEntries[0].ColumnName
|
||||
}
|
||||
rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name)
|
||||
if exists {
|
||||
return rawPath + " IS NOT NULL", nil
|
||||
members := physicalSemconvMembers(key)
|
||||
paths := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
paths = append(paths, fmt.Sprintf("%s.%s", columnName, ClickHouseIdentifier(member)))
|
||||
}
|
||||
return rawPath + " IS NULL", nil
|
||||
if len(paths) == 1 {
|
||||
if exists {
|
||||
return paths[0] + " IS NOT NULL", nil
|
||||
}
|
||||
return paths[0] + " IS NULL", nil
|
||||
}
|
||||
guards := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
guards = append(guards, path+" IS NOT NULL")
|
||||
}
|
||||
rawPath := "(" + strings.Join(guards, " OR ") + ")"
|
||||
if exists {
|
||||
return rawPath, nil
|
||||
}
|
||||
return "NOT " + rawPath, nil
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumFixedString:
|
||||
if exists {
|
||||
@@ -88,9 +123,14 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
if key.Materialized {
|
||||
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
members := physicalSemconvMembers(key)
|
||||
operands := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
operands = append(operands, mapMemberExistenceExpression(column.Name, key, member))
|
||||
}
|
||||
leftOperand := strings.Join(operands, " OR ")
|
||||
if len(operands) > 1 {
|
||||
leftOperand = "(" + leftOperand + ")"
|
||||
}
|
||||
if exists {
|
||||
return leftOperand, nil
|
||||
|
||||
41
pkg/querybuilder/exists_expr_test.go
Normal file
41
pkg/querybuilder/exists_expr_test.go
Normal file
@@ -0,0 +1,41 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExistsExpressionUsesEveryPhysicalSemconvMember(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{
|
||||
"deployment.environment.name",
|
||||
"deployment.environment",
|
||||
},
|
||||
SemconvMaterializedColumns: map[string]string{
|
||||
"deployment.environment": "attribute_string_deployment$$environment",
|
||||
},
|
||||
}
|
||||
columns := []*schema.Column{{
|
||||
Name: "attributes_string",
|
||||
Type: schema.MapColumnType{
|
||||
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
|
||||
ValueType: schema.ColumnTypeString,
|
||||
},
|
||||
}}
|
||||
|
||||
expression, err := ExistsExpression(columns, key, 0, 0, "unused", true)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
"(mapContains(attributes_string, 'deployment.environment.name') OR `attribute_string_deployment$$environment_exists`)",
|
||||
expression,
|
||||
)
|
||||
}
|
||||
49
pkg/querybuilder/semconv_materialization_test.go
Normal file
49
pkg/querybuilder/semconv_materialization_test.go
Normal file
@@ -0,0 +1,49 @@
|
||||
package querybuilder_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTraceFamilyUsesMaterializedHistoricalMember(t *testing.T) {
|
||||
current := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
historical := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
}
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
current.Name,
|
||||
telemetrytypes.FieldContextAttribute,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
|
||||
matches := querybuilder.MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {current},
|
||||
historical.Name: {historical},
|
||||
})
|
||||
require.Len(t, matches, 1, "family metadata should resolve to one logical field")
|
||||
expression, err := tracestelemetryschema.NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, 0, 0, matches[0])
|
||||
require.NoError(t, err, "resolved trace family should map to a value expression")
|
||||
|
||||
assert.Equal(
|
||||
t,
|
||||
"COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(`attribute_string_deployment$$environment`, ''), '')",
|
||||
expression,
|
||||
"family expression should retain the promoted historical member",
|
||||
)
|
||||
}
|
||||
@@ -4,12 +4,14 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -982,25 +984,105 @@ func assignIfEmpty(s *string, value string) {
|
||||
// MatchingFieldKeys returns the field keys from the map that match the given key,
|
||||
// honoring any context/data type the user specified.
|
||||
func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
fieldKeysForName := []*telemetrytypes.TelemetryFieldKey{}
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: field.FieldContext,
|
||||
}
|
||||
members := []string{field.Name}
|
||||
// Only trace field mappers understand semantic-convention families today.
|
||||
// Logs and metrics must keep using the requested spelling until theirs land.
|
||||
if field.Signal == telemetrytypes.SignalUnspecified || field.Signal == telemetrytypes.SignalTraces {
|
||||
members = semconv.Members(semconv.KindAttribute, selector)
|
||||
}
|
||||
isFamily := len(members) > 1
|
||||
fieldKeysForName := make([]*telemetrytypes.TelemetryFieldKey, 0)
|
||||
indexByIdentity := make(map[string]int)
|
||||
|
||||
// match by name; keep items whose context and data type match (unspecified matches any)
|
||||
for _, item := range fieldKeys[field.Name] {
|
||||
if (field.FieldContext == telemetrytypes.FieldContextUnspecified || field.FieldContext == item.FieldContext) &&
|
||||
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || field.FieldDataType == item.FieldDataType) {
|
||||
fieldKeysForName = append(fieldKeysForName, item)
|
||||
appendMatches := func(lookupName string, memberName string, contextAlreadyMatched bool) {
|
||||
for _, item := range fieldKeys[lookupName] {
|
||||
if !contextAlreadyMatched && field.FieldContext != telemetrytypes.FieldContextUnspecified && field.FieldContext != item.FieldContext {
|
||||
continue
|
||||
}
|
||||
if field.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && field.FieldDataType != item.FieldDataType {
|
||||
continue
|
||||
}
|
||||
|
||||
// A wildcard lookup may have found a same-named field in a scope where
|
||||
// this family does not apply. Keep exact names, but reject cross-member
|
||||
// matches outside the generated family scope.
|
||||
traceFamilyMatch := isFamily && item.Signal == telemetrytypes.SignalTraces
|
||||
if memberName != field.Name {
|
||||
if !traceFamilyMatch {
|
||||
continue
|
||||
}
|
||||
itemSelector := telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: item.FieldContext,
|
||||
}
|
||||
if !slices.Contains(semconv.Members(semconv.KindAttribute, itemSelector), memberName) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
physicalMembers := item.SemconvMembers
|
||||
if len(physicalMembers) == 0 {
|
||||
physicalMembers = []string{memberName}
|
||||
}
|
||||
materializedColumns := maps.Clone(item.SemconvMaterializedColumns)
|
||||
if item.Materialized {
|
||||
if materializedColumns == nil {
|
||||
materializedColumns = make(map[string]string)
|
||||
}
|
||||
physicalKey := item.Copy()
|
||||
physicalKey.Name = memberName
|
||||
materializedColumns[memberName] = strings.Trim(telemetrytypes.FieldKeyToMaterializedColumnName(physicalKey), "`")
|
||||
}
|
||||
identity := item.Signal.StringValue() + ";" + item.FieldContext.StringValue() + ";" + item.FieldDataType.StringValue()
|
||||
if traceFamilyMatch {
|
||||
if index, found := indexByIdentity[identity]; found {
|
||||
for _, physicalMember := range physicalMembers {
|
||||
if !slices.Contains(fieldKeysForName[index].SemconvMembers, physicalMember) {
|
||||
fieldKeysForName[index].SemconvMembers = append(fieldKeysForName[index].SemconvMembers, physicalMember)
|
||||
}
|
||||
}
|
||||
if len(materializedColumns) > 0 {
|
||||
if fieldKeysForName[index].SemconvMaterializedColumns == nil {
|
||||
fieldKeysForName[index].SemconvMaterializedColumns = make(map[string]string)
|
||||
}
|
||||
maps.Copy(fieldKeysForName[index].SemconvMaterializedColumns, materializedColumns)
|
||||
}
|
||||
continue
|
||||
}
|
||||
indexByIdentity[identity] = len(fieldKeysForName)
|
||||
}
|
||||
resolved := item.Copy()
|
||||
// The requested spelling is the response identity. Field mappers use
|
||||
// it to resolve the available family members current-first.
|
||||
if traceFamilyMatch {
|
||||
resolved.Name = field.Name
|
||||
resolved.SemconvMembers = slices.Clone(physicalMembers)
|
||||
resolved.SemconvMaterializedColumns = materializedColumns
|
||||
// Materialization is member-specific after family keys are merged.
|
||||
resolved.Materialized = false
|
||||
}
|
||||
fieldKeysForName = append(fieldKeysForName, resolved)
|
||||
}
|
||||
}
|
||||
|
||||
// A context may have been split off a name that legitimately contained it (e.g.
|
||||
// `attribute.key`); also look up the context-prefixed name so both readings resolve.
|
||||
// Members are current-first, so metadata from the current key wins when
|
||||
// both spellings describe the same signal/context/type.
|
||||
for _, member := range members {
|
||||
appendMatches(member, member, false)
|
||||
}
|
||||
|
||||
// A context may have been split off a name that legitimately contained it
|
||||
// (e.g. `attribute.key`); preserve that historical alternate reading for
|
||||
// every family member.
|
||||
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
contextPrefixedFieldName := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)
|
||||
for _, item := range fieldKeys[contextPrefixedFieldName] {
|
||||
// Context already matched via the lookup key; only data type needs checking.
|
||||
if field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || item.FieldDataType == field.FieldDataType {
|
||||
fieldKeysForName = append(fieldKeysForName, item)
|
||||
}
|
||||
for _, member := range members {
|
||||
appendMatches(fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), member), member, true)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestPrepareWhereClause_EmptyVariableList ensures PrepareWhereClause errors when a variable has an empty list value.
|
||||
@@ -685,6 +686,118 @@ func TestVisitKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchingFieldKeysResolvesCurrentTraceNameFromOldMetadata(t *testing.T) {
|
||||
old := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Description: "old metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
"deployment.environment.name",
|
||||
telemetrytypes.FieldContextResource,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
|
||||
matches := MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{old.Name: {old}})
|
||||
|
||||
require.Len(t, matches, 1, "trace family lookup must resolve before inspecting metadata")
|
||||
assert.Equal(t, "deployment.environment.name", matches[0].Name)
|
||||
assert.Equal(t, "old metadata", matches[0].Description)
|
||||
assert.Equal(t, []string{"deployment.environment"}, matches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
func TestMatchingFieldKeysUsesCurrentTraceMetadataForOldName(t *testing.T) {
|
||||
current := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Description: "current metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
old := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Description: "old metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
old.Name,
|
||||
telemetrytypes.FieldContextResource,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
|
||||
matches := MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {current},
|
||||
old.Name: {old},
|
||||
})
|
||||
|
||||
require.Len(t, matches, 1, "trace family lookup must resolve before inspecting metadata")
|
||||
assert.Equal(t, old.Name, matches[0].Name)
|
||||
assert.Equal(t, "current metadata", matches[0].Description)
|
||||
assert.Equal(t, []string{current.Name, old.Name}, matches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
func TestMatchingFieldKeysKeepsLogSemconvNamesLiteral(t *testing.T) {
|
||||
current := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
old := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
current.Name,
|
||||
telemetrytypes.FieldContextResource,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
|
||||
matches := MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {current},
|
||||
old.Name: {old},
|
||||
})
|
||||
|
||||
require.Len(t, matches, 1, "log lookup must keep the requested spelling literal")
|
||||
assert.Equal(t, current.Name, matches[0].Name)
|
||||
assert.Empty(t, matches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
func TestMatchingFieldKeysKeepsMetricSemconvNamesLiteral(t *testing.T) {
|
||||
current := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
old := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
current.Name,
|
||||
telemetrytypes.FieldContextResource,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
|
||||
matches := MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {current},
|
||||
old.Name: {old},
|
||||
})
|
||||
|
||||
require.Len(t, matches, 1, "metric lookup must keep the requested spelling literal")
|
||||
assert.Equal(t, current.Name, matches[0].Name)
|
||||
assert.Empty(t, matches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TestVisitComparison
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -44,6 +44,73 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
|
||||
return fmt.Sprintf(`%%%s%%`, key.Name)
|
||||
}
|
||||
|
||||
func memberKey(key *telemetrytypes.TelemetryFieldKey, name string) *telemetrytypes.TelemetryFieldKey {
|
||||
member := key.Copy()
|
||||
member.Name = name
|
||||
return member
|
||||
}
|
||||
|
||||
func keyIndexCondition(sb *sqlbuilder.SelectBuilder, column string, key *telemetrytypes.TelemetryFieldKey, members []string) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
conditions = append(conditions, sb.Like(column, keyIndexFilter(memberKey(key, member))))
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
|
||||
func valueIndexCondition(
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
column string,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
members []string,
|
||||
op qbtypes.FilterOperator,
|
||||
value any,
|
||||
caseInsensitive bool,
|
||||
) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
patterns := valueForIndexFilter(op, memberKey(key, member), value)
|
||||
switch values := patterns.(type) {
|
||||
case []string:
|
||||
for _, pattern := range values {
|
||||
conditions = append(conditions, sb.Like(column, pattern))
|
||||
}
|
||||
default:
|
||||
if caseInsensitive {
|
||||
conditions = append(conditions, sb.ILike(column, values))
|
||||
} else {
|
||||
conditions = append(conditions, sb.Like(column, values))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
|
||||
func memberPresenceCondition(sb *sqlbuilder.SelectBuilder, column string, members []string, exists bool) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
field := fmt.Sprintf("simpleJSONHas(%s, %s)", column, querybuilder.ClickHouseStringLiteral(member))
|
||||
if exists {
|
||||
conditions = append(conditions, sb.E(field, true))
|
||||
} else {
|
||||
conditions = append(conditions, sb.NE(field, true))
|
||||
}
|
||||
}
|
||||
if exists {
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
return sb.And(conditions...)
|
||||
}
|
||||
|
||||
// SkipResourceFilter is not applicable here: the fingerprint table only stores resource attributes.
|
||||
func (b *defaultConditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
@@ -115,8 +182,10 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
// as we have not changed the resource column in the resource fingerprint table.
|
||||
column := columns[0]
|
||||
|
||||
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
|
||||
valueForIndexFilter := valueForIndexFilter(op, key, value)
|
||||
members := resourceSemconvMembers(key)
|
||||
isFamily := len(members) > 1
|
||||
keyIdxFilter := keyIndexCondition(sb, column.Name, key, members)
|
||||
singleValueIndexFilter := valueForIndexFilter(op, memberKey(key, members[0]), value)
|
||||
|
||||
fieldName, err := b.fm.FieldFor(ctx, valuer.UUID{}, startNs, endNs, key)
|
||||
if err != nil {
|
||||
@@ -128,12 +197,15 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.E(fieldName, formattedValue),
|
||||
keyIdxFilter,
|
||||
sb.Like(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, false),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
if isFamily {
|
||||
return sb.NE(fieldName, formattedValue), nil
|
||||
}
|
||||
return sb.And(
|
||||
sb.NE(fieldName, formattedValue),
|
||||
sb.NotLike(column.Name, valueForIndexFilter),
|
||||
sb.NotLike(column.Name, singleValueIndexFilter),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.And(sb.GT(fieldName, formattedValue), keyIdxFilter), nil
|
||||
@@ -148,7 +220,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.ILike(fieldName, formattedValue),
|
||||
keyIdxFilter,
|
||||
sb.ILike(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, true),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotLike, qbtypes.FilterOperatorNotILike:
|
||||
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
|
||||
@@ -185,13 +257,11 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
mainCondition := sb.Or(inConditions...)
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.Like(column.Name, v))
|
||||
}
|
||||
}
|
||||
mainCondition = sb.And(mainCondition, keyIdxFilter, sb.Or(valConditions...))
|
||||
mainCondition = sb.And(
|
||||
mainCondition,
|
||||
keyIdxFilter,
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, false),
|
||||
)
|
||||
|
||||
return mainCondition, nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -204,8 +274,11 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
mainCondition := sb.And(notInConditions...)
|
||||
if isFamily {
|
||||
return mainCondition, nil
|
||||
}
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
|
||||
if valuesForIndexFilter, ok := singleValueIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.NotLike(column.Name, v))
|
||||
}
|
||||
@@ -215,13 +288,11 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
|
||||
case qbtypes.FilterOperatorExists:
|
||||
return sb.And(
|
||||
sb.E(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
|
||||
memberPresenceCondition(sb, column.Name, members, true),
|
||||
keyIdxFilter,
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotExists:
|
||||
return sb.And(
|
||||
sb.NE(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
|
||||
), nil
|
||||
return memberPresenceCondition(sb, column.Name, members, false), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
return sb.And(
|
||||
@@ -237,7 +308,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.ILike(fieldName, fmt.Sprintf(`%%%s%%`, formattedValue)),
|
||||
keyIdxFilter,
|
||||
sb.ILike(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, true),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
|
||||
|
||||
@@ -220,3 +220,216 @@ func TestConditionBuilder(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFamilyPositiveFilterExcludesKeylessRows(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') = ? AND (labels LIKE ? OR labels LIKE ?) AND (labels LIKE ? OR labels LIKE ?)")
|
||||
assert.Equal(t, []any{
|
||||
"production",
|
||||
"%deployment.environment.name%",
|
||||
"%deployment.environment%",
|
||||
`%deployment.environment.name":"production%`,
|
||||
`%deployment.environment":"production%`,
|
||||
}, args)
|
||||
}
|
||||
|
||||
func TestFamilyNotEqualIncludesKeylessRows(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotEqual, "staging", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') <> ?")
|
||||
assert.Equal(t, []any{"staging"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyNotInIncludesKeylessRows(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotIn, []any{"staging", "dev"}, sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "(COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') <> ? AND COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') <> ?)")
|
||||
assert.Equal(t, []any{"staging", "dev"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyNotLikeIncludesKeylessRows(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotLike, "%stag%", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "LOWER(COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '')) NOT LIKE LOWER(?)")
|
||||
assert.Equal(t, []any{"%stag%"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyNotContainsIncludesKeylessRows(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotContains, "stag", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "LOWER(COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '')) NOT LIKE LOWER(?)")
|
||||
assert.Equal(t, []any{"%stag%"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyNotRegexpIncludesKeylessRows(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotRegexp, "stag.*", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "NOT match(COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), ''), ?)")
|
||||
assert.Equal(t, []any{"stag.*"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyExistsChecksEveryMember(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorExists, nil, sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "(simpleJSONHas(labels, 'deployment.environment.name') = ? OR simpleJSONHas(labels, 'deployment.environment') = ?) AND (labels LIKE ? OR labels LIKE ?)")
|
||||
assert.Equal(t, []any{true, true, "%deployment.environment.name%", "%deployment.environment%"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyNotExistsChecksEveryMember(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotExists, nil, sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "simpleJSONHas(labels, 'deployment.environment.name') <> ? AND simpleJSONHas(labels, 'deployment.environment') <> ?")
|
||||
assert.Equal(t, []any{true, true}, args)
|
||||
}
|
||||
|
||||
func TestLogSemconvNameStaysLiteral(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, _, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Contains(t, sql, "simpleJSONExtractString(labels, 'deployment.environment.name') = ? AND labels LIKE ? AND labels LIKE ?")
|
||||
assert.NotContains(t, sql, "deployment.environment')")
|
||||
assert.Equal(t, []any{"production", "%deployment.environment.name%", `%deployment.environment.name":"production%`}, args)
|
||||
}
|
||||
|
||||
@@ -3,8 +3,11 @@ package resourcefilter
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -32,6 +35,20 @@ func NewFieldMapper() *defaultFieldMapper {
|
||||
return &defaultFieldMapper{}
|
||||
}
|
||||
|
||||
func resourceSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.Signal != telemetrytypes.SignalTraces || key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
@@ -66,7 +83,15 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
return "", err
|
||||
}
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
members := resourceSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(simpleJSONExtractString(%s, %s), '')", columns[0].Name, querybuilder.ClickHouseStringLiteral(member)))
|
||||
}
|
||||
return "COALESCE(" + strings.Join(values, ", ") + ", '')", nil
|
||||
}
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(members[0])), nil
|
||||
}
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
24
pkg/statementbuilder/resourcefilter/field_mapper_test.go
Normal file
24
pkg/statementbuilder/resourcefilter/field_mapper_test.go
Normal file
@@ -0,0 +1,24 @@
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFieldForQuotesRequestKeyName(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "name'\\); SELECT 1 --",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
expression, err := NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "simpleJSONExtractString(labels, "+querybuilder.ClickHouseStringLiteral(key.Name)+")", expression)
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
@@ -151,6 +152,14 @@ func (t *telemetryMetaStore) tracesTblStatementToFieldKeys(ctx context.Context)
|
||||
return materialisedKeys, nil
|
||||
}
|
||||
|
||||
func traceSemconvMembers(name string, fieldContext telemetrytypes.FieldContext) []string {
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: fieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
// getTracesKeys returns the keys from the spans that match the field selection criteria.
|
||||
func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
@@ -1320,6 +1329,17 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
// GetKeys backs key suggestions and remains literal. The internal multi-key
|
||||
// lookup expands only trace selectors so query builders see stored family members.
|
||||
expandedTraceSelectors := make([]*telemetrytypes.FieldKeySelector, 0, len(tracesSelectors))
|
||||
for _, selector := range tracesSelectors {
|
||||
for _, member := range traceSemconvMembers(selector.Name, selector.FieldContext) {
|
||||
memberSelector := selector.Copy()
|
||||
memberSelector.Name = member
|
||||
expandedTraceSelectors = append(expandedTraceSelectors, memberSelector)
|
||||
}
|
||||
}
|
||||
tracesSelectors = expandedTraceSelectors
|
||||
tracesKeys, tracesComplete, err := t.getTracesKeys(ctx, tracesSelectors)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
@@ -1542,7 +1562,16 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
members := traceSemconvMembers(fieldValueSelector.Name, fieldValueSelector.FieldContext)
|
||||
if len(members) == 1 {
|
||||
sb.Where(sb.E("tag_key", members[0]))
|
||||
} else {
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
sb.Where(sb.In("tag_key", memberValues...))
|
||||
}
|
||||
}
|
||||
|
||||
// now look at the field context
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -83,3 +84,38 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
t.Errorf("there were unfulfilled expectations: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAllValuesReturnsValuesFromEveryTraceSemconvFamilyMember(t *testing.T) {
|
||||
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, ®exMatcher{})
|
||||
mock := mockTelemetryStore.Mock()
|
||||
|
||||
metadata := NewTelemetryMetaStore(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockTelemetryStore,
|
||||
flaggertest.New(t),
|
||||
)
|
||||
|
||||
mock.ExpectQuery(`SELECT DISTINCT string_value, number_value FROM signoz_traces\.distributed_tag_attributes_v2 WHERE tag_key IN \(\?, \?\) AND tag_type = \? AND tag_data_type = \? LIMIT \?`).
|
||||
WithArgs("deployment.environment.name", "deployment.environment", "resource", "string", 51).
|
||||
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
|
||||
{Name: "string_value", Type: "String"},
|
||||
{Name: "number_value", Type: "Float64"},
|
||||
}, [][]any{
|
||||
{"production", float64(0)},
|
||||
{"staging", float64(0)},
|
||||
{"production", float64(0)},
|
||||
}))
|
||||
|
||||
values, complete, err := metadata.GetAllValues(context.Background(), valuer.UUID{}, &telemetrytypes.FieldValueSelector{
|
||||
FieldKeySelector: &telemetrytypes.FieldKeySelector{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Name: "deployment.environment",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, complete)
|
||||
assert.Equal(t, []string{"production", "staging"}, values.StringValues)
|
||||
assert.NoError(t, mock.ExpectationsWereMet(), "all expected metadata queries should be executed")
|
||||
}
|
||||
|
||||
@@ -308,6 +308,88 @@ func TestConditionFor(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionForSemconvFamilyPositiveFilterChecksPresence(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, warnings, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Empty(t, warnings)
|
||||
assert.Contains(t, sql, "(COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], ''), '') = ? AND (mapContains(attributes_string, 'deployment.environment.name') OR mapContains(attributes_string, 'deployment.environment')))")
|
||||
assert.Equal(t, []any{"production"}, args)
|
||||
}
|
||||
|
||||
func TestNewConditionBuilderAcceptsFieldMapperInterface(t *testing.T) {
|
||||
var mapper qbtypes.FieldMapper = NewFieldMapper()
|
||||
|
||||
require.NotNil(t, NewConditionBuilder(mapper))
|
||||
}
|
||||
|
||||
func TestConditionForSemconvFamilyPreservesMaterializedMemberExistsColumn(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
SemconvMaterializedColumns: map[string]string{
|
||||
"deployment.environment": "attribute_string_deployment$$environment",
|
||||
},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, warnings, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Empty(t, warnings)
|
||||
assert.Contains(t, sql, "`attribute_string_deployment$$environment_exists`")
|
||||
assert.NotContains(t, sql, "`attribute_string_deployment$environment_exists`")
|
||||
assert.Equal(t, []any{"production"}, args)
|
||||
}
|
||||
|
||||
func TestConditionForSemconvFamilyNotExistsChecksEveryMember(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
conditions, warnings, err := NewConditionBuilder(NewFieldMapper()).ConditionFor(
|
||||
context.Background(), valuer.UUID{}, 0, 0, key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {key}},
|
||||
qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotExists, nil, sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
assert.Empty(t, warnings)
|
||||
assert.Contains(t, sql, "NOT (mapContains(attributes_string, 'deployment.environment.name') OR mapContains(attributes_string, 'deployment.environment'))")
|
||||
assert.Empty(t, args)
|
||||
}
|
||||
|
||||
func TestConditionForResourceWithEvolution(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -167,6 +168,32 @@ func NewFieldMapper() *fieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func traceSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
func traceSemconvMapMemberExpressions(columnName string, key *telemetrytypes.TelemetryFieldKey, member string) (string, string) {
|
||||
if materializedColumn, ok := key.SemconvMaterializedColumns[member]; ok {
|
||||
return querybuilder.ClickHouseIdentifier(materializedColumn), querybuilder.ClickHouseIdentifier(materializedColumn + "_exists")
|
||||
}
|
||||
if key.Materialized && key.Name == member {
|
||||
physicalKey := key.Copy()
|
||||
physicalKey.Name = member
|
||||
return telemetrytypes.FieldKeyToMaterializedColumnName(physicalKey), telemetrytypes.FieldKeyToMaterializedColumnNameForExists(physicalKey)
|
||||
}
|
||||
return fmt.Sprintf("%s[%s]", columnName, querybuilder.ClickHouseStringLiteral(member)), fmt.Sprintf("mapContains(%s, %s)", columnName, querybuilder.ClickHouseStringLiteral(member))
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
@@ -291,10 +318,27 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
|
||||
}
|
||||
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
|
||||
// once clickHouse dependency is updated, we need to check if we can remove it.
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
|
||||
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
|
||||
members := traceSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
values := make([]string, 0, len(members))
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
// The String cast is required because ClickHouse does not allow
|
||||
// Variant/Dynamic values in GROUP BY.
|
||||
value := fmt.Sprintf("%s.%s::String", columnName, querybuilder.ClickHouseIdentifier(member))
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s, '')", value))
|
||||
guards = append(guards, fmt.Sprintf("%s.%s IS NOT NULL", columnName, querybuilder.ClickHouseIdentifier(member)))
|
||||
}
|
||||
// Missing Dynamic paths are NULL, so this family expression must
|
||||
// retain the same NULL result as a single JSON-path lookup.
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
|
||||
// once ClickHouse is updated, check whether this cast can be removed.
|
||||
exprs = append(exprs, fmt.Sprintf("%s.%s::String", columnName, querybuilder.ClickHouseIdentifier(members[0])))
|
||||
existExprs = append(existExprs, fmt.Sprintf("%s.%s IS NOT NULL", columnName, querybuilder.ClickHouseIdentifier(members[0])))
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumUInt64,
|
||||
schema.ColumnTypeEnumUInt32,
|
||||
@@ -319,13 +363,40 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
|
||||
members := traceSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
guards := make([]string, 0, len(members))
|
||||
memberValues := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
valueExpression, existsExpression := traceSemconvMapMemberExpressions(columnName, key, member)
|
||||
memberValues = append(memberValues, valueExpression)
|
||||
guards = append(guards, existsExpression)
|
||||
}
|
||||
if valueType.GetType() == schema.ColumnTypeEnumString {
|
||||
values := make([]string, 0, len(members))
|
||||
for _, memberValue := range memberValues {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s, '')", memberValue))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+", '')")
|
||||
} else {
|
||||
branches := make([]string, 0, len(members)*2)
|
||||
for i, memberValue := range memberValues {
|
||||
branches = append(branches, guards[i], memberValue)
|
||||
}
|
||||
// Numeric and boolean maps return zero for an absent key. If a
|
||||
// family of either type is enabled, this tail must become zero too.
|
||||
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
|
||||
}
|
||||
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
|
||||
} else if key.Materialized {
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
physicalKey := key.Copy()
|
||||
physicalKey.Name = members[0]
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(physicalKey))
|
||||
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(physicalKey))
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
|
||||
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
|
||||
exprs = append(exprs, fmt.Sprintf("%s[%s]", columnName, querybuilder.ClickHouseStringLiteral(members[0])))
|
||||
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, %s)", columnName, querybuilder.ClickHouseStringLiteral(members[0])))
|
||||
}
|
||||
default:
|
||||
return nil, nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -12,6 +13,20 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFieldForQuotesRequestKeyNames(t *testing.T) {
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "name'`\\); SELECT 1 --",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Contains(t, expression, "resource."+querybuilder.ClickHouseIdentifier(key.Name))
|
||||
assert.Contains(t, expression, "mapContains(resources_string, "+querybuilder.ClickHouseStringLiteral(key.Name)+")")
|
||||
}
|
||||
|
||||
func TestGetFieldKeyName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -80,7 +95,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
Materialized: true,
|
||||
Evolutions: mockEvolution,
|
||||
},
|
||||
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
|
||||
expectedResult: "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR `resource_string_deployment$$environment_exists`), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(`resource_string_deployment$$environment`, ''), ''), NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -120,6 +135,63 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldForResolvesCurrentTraceSemconvAttributeName(t *testing.T) {
|
||||
key := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, 0, 0, &key)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], ''), '')", expression)
|
||||
}
|
||||
|
||||
func TestFieldForResolvesOldTraceSemconvAttributeName(t *testing.T) {
|
||||
key := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, 0, 0, &key)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], ''), '')", expression)
|
||||
}
|
||||
|
||||
func TestFieldForPreservesResourceStorageDefaultsForSemconvFamily(t *testing.T) {
|
||||
key := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
Evolutions: MockEvolutionData(time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC)),
|
||||
}
|
||||
start := uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano())
|
||||
end := uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano())
|
||||
|
||||
expression, err := NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, start, end, &key)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (`resource_string_deployment$$environment$$name_exists` OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(`resource_string_deployment$$environment$$name`, ''), NULLIF(resources_string['deployment.environment'], ''), ''), NULL)", expression)
|
||||
}
|
||||
|
||||
func TestFieldForUsesAvailableTraceSemconvMember(t *testing.T) {
|
||||
key := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name"},
|
||||
}
|
||||
|
||||
expression, err := NewFieldMapper().FieldFor(context.Background(), valuer.UUID{}, 0, 0, &key)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "attributes_string['deployment.environment.name']", expression)
|
||||
}
|
||||
|
||||
func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
@@ -176,7 +248,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
},
|
||||
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedResult: "resource.`deployment.environment`::String",
|
||||
expectedResult: "COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, ''))",
|
||||
},
|
||||
{
|
||||
name: "Window straddles release - materialized resource",
|
||||
@@ -189,7 +261,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
|
||||
expectedResult: "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR `resource_string_deployment$$environment_exists`), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(`resource_string_deployment$$environment`, ''), ''), NULL)",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -155,6 +155,8 @@ var operatorInverseMapping = map[FilterOperator]FilterOperator{
|
||||
// doesn't have value "redis"
|
||||
// Since we don't know the intent, we don't add the exists filter. They are expected
|
||||
// to add exists filter themselves if exclusion is desired.
|
||||
// Negative predicates therefore include rows where the key is absent; value
|
||||
// expressions must preserve the storage column's absent-key default.
|
||||
//
|
||||
// For the positive predicates, the key existence is implied.
|
||||
func (f FilterOperator) AddDefaultExistsFilter() bool {
|
||||
|
||||
@@ -2,6 +2,8 @@ package telemetrytypes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -47,7 +49,81 @@ type TelemetryFieldKey struct {
|
||||
Indexes []TelemetryFieldKeySkipIndex `json:"-"`
|
||||
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
|
||||
|
||||
Evolutions []*EvolutionEntry `json:"-"`
|
||||
Evolutions []*EvolutionEntry `json:"-"`
|
||||
SemconvMembers []string `json:"-"`
|
||||
// SemconvMaterializedColumns maps a physical family spelling to its
|
||||
// materialized column name. It is populated only on resolved query keys.
|
||||
SemconvMaterializedColumns map[string]string `json:"-"`
|
||||
}
|
||||
|
||||
// Copy returns an independent copy of f.
|
||||
func (f *TelemetryFieldKey) Copy() *TelemetryFieldKey {
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
copied := *f
|
||||
copied.Indexes = slices.Clone(f.Indexes)
|
||||
if f.Evolutions != nil {
|
||||
copied.Evolutions = make([]*EvolutionEntry, len(f.Evolutions))
|
||||
for index, evolution := range f.Evolutions {
|
||||
if evolution != nil {
|
||||
copiedEvolution := *evolution
|
||||
copied.Evolutions[index] = &copiedEvolution
|
||||
}
|
||||
}
|
||||
}
|
||||
copied.SemconvMembers = slices.Clone(f.SemconvMembers)
|
||||
copied.SemconvMaterializedColumns = maps.Clone(f.SemconvMaterializedColumns)
|
||||
copied.JSONPlan = copyJSONAccessPlan(f.JSONPlan, f, &copied)
|
||||
|
||||
return &copied
|
||||
}
|
||||
|
||||
func copyJSONAccessPlan(plan JSONAccessPlan, sourceKey, copiedKey *TelemetryFieldKey) JSONAccessPlan {
|
||||
if plan == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
nodes := make(map[*JSONAccessNode]*JSONAccessNode)
|
||||
var copyNode func(*JSONAccessNode) *JSONAccessNode
|
||||
copyNode = func(node *JSONAccessNode) *JSONAccessNode {
|
||||
if node == nil {
|
||||
return nil
|
||||
}
|
||||
if copiedNode, ok := nodes[node]; ok {
|
||||
return copiedNode
|
||||
}
|
||||
|
||||
copiedNode := *node
|
||||
nodes[node] = &copiedNode
|
||||
copiedNode.Parent = copyNode(node.Parent)
|
||||
if node.Branches != nil {
|
||||
copiedNode.Branches = make(map[JSONAccessBranchType]*JSONAccessNode, len(node.Branches))
|
||||
for branchType, branch := range node.Branches {
|
||||
copiedNode.Branches[branchType] = copyNode(branch)
|
||||
}
|
||||
}
|
||||
if node.TerminalConfig != nil {
|
||||
copiedTerminal := *node.TerminalConfig
|
||||
switch node.TerminalConfig.Key {
|
||||
case nil:
|
||||
case sourceKey:
|
||||
copiedTerminal.Key = copiedKey
|
||||
default:
|
||||
copiedTerminal.Key = node.TerminalConfig.Key.Copy()
|
||||
}
|
||||
copiedNode.TerminalConfig = &copiedTerminal
|
||||
}
|
||||
|
||||
return &copiedNode
|
||||
}
|
||||
|
||||
copied := make(JSONAccessPlan, len(plan))
|
||||
for index, node := range plan {
|
||||
copied[index] = copyNode(node)
|
||||
}
|
||||
return copied
|
||||
}
|
||||
|
||||
func (f *TelemetryFieldKey) KeyNameContainsArray() bool {
|
||||
@@ -128,6 +204,8 @@ func (f *TelemetryFieldKey) OverrideMetadataFrom(src *TelemetryFieldKey) {
|
||||
f.Materialized = src.Materialized
|
||||
f.JSONPlan = src.JSONPlan
|
||||
f.Evolutions = src.Evolutions
|
||||
f.SemconvMembers = src.SemconvMembers
|
||||
f.SemconvMaterializedColumns = src.SemconvMaterializedColumns
|
||||
}
|
||||
|
||||
func (f *TelemetryFieldKey) Equal(key *TelemetryFieldKey) bool {
|
||||
@@ -233,6 +311,15 @@ type MetricContext struct {
|
||||
MetricNamespace string `json:"metricNamespace,omitempty"`
|
||||
}
|
||||
|
||||
// Copy returns an independent copy of m.
|
||||
func (m *MetricContext) Copy() *MetricContext {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
copied := *m
|
||||
return &copied
|
||||
}
|
||||
|
||||
type FieldKeySelector struct {
|
||||
StartUnixMilli int64 `json:"startUnixMilli"`
|
||||
EndUnixMilli int64 `json:"endUnixMilli"`
|
||||
@@ -246,6 +333,16 @@ type FieldKeySelector struct {
|
||||
MetricContext *MetricContext `json:"metricContext,omitempty"`
|
||||
}
|
||||
|
||||
// Copy returns an independent copy of s.
|
||||
func (s *FieldKeySelector) Copy() *FieldKeySelector {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
copied := *s
|
||||
copied.MetricContext = s.MetricContext.Copy()
|
||||
return &copied
|
||||
}
|
||||
|
||||
type FieldValueSelector struct {
|
||||
*FieldKeySelector
|
||||
ExistingQuery string `json:"existingQuery"`
|
||||
|
||||
83
pkg/types/telemetrytypes/field_copy_test.go
Normal file
83
pkg/types/telemetrytypes/field_copy_test.go
Normal file
@@ -0,0 +1,83 @@
|
||||
package telemetrytypes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestTelemetryFieldKeyCopyOwnsMutableState(t *testing.T) {
|
||||
original := &TelemetryFieldKey{
|
||||
Name: "items.name",
|
||||
FieldContext: FieldContextBody,
|
||||
FieldDataType: FieldDataTypeString,
|
||||
Indexes: []TelemetryFieldKeySkipIndex{
|
||||
{Name: "items.name"},
|
||||
},
|
||||
Evolutions: []*EvolutionEntry{
|
||||
{FieldName: "items.name"},
|
||||
},
|
||||
SemconvMembers: []string{"items.name", "item.name"},
|
||||
SemconvMaterializedColumns: map[string]string{
|
||||
"item.name": "body_string_item$$name",
|
||||
},
|
||||
}
|
||||
require.NoError(t, original.SetJSONAccessPlan(JSONColumnMetadata{BaseColumn: "body_v2"}, nil))
|
||||
require.Len(t, original.JSONPlan, 1)
|
||||
require.NotNil(t, original.JSONPlan[0].TerminalConfig)
|
||||
|
||||
copied := original.Copy()
|
||||
require.NotNil(t, copied)
|
||||
require.NotSame(t, original, copied)
|
||||
require.Len(t, copied.JSONPlan, 1)
|
||||
require.NotNil(t, copied.JSONPlan[0].TerminalConfig)
|
||||
|
||||
assert.NotSame(t, original.JSONPlan[0], copied.JSONPlan[0])
|
||||
assert.NotSame(t, original.JSONPlan[0].Parent, copied.JSONPlan[0].Parent)
|
||||
assert.Same(t, copied, copied.JSONPlan[0].TerminalConfig.Key)
|
||||
assert.Equal(t, original.JSONPlan[0].Alias(), copied.JSONPlan[0].Alias())
|
||||
|
||||
copied.Name = "changed"
|
||||
copied.Indexes[0].Name = "changed"
|
||||
copied.Evolutions[0].FieldName = "changed"
|
||||
copied.SemconvMembers[0] = "changed"
|
||||
copied.SemconvMaterializedColumns["item.name"] = "changed"
|
||||
copied.JSONPlan[0].Name = "changed"
|
||||
copied.JSONPlan[0].Parent.Name = "changed"
|
||||
|
||||
assert.Equal(t, "items.name", original.Name)
|
||||
assert.Equal(t, "items.name", original.Indexes[0].Name)
|
||||
assert.Equal(t, "items.name", original.Evolutions[0].FieldName)
|
||||
assert.Equal(t, "items.name", original.SemconvMembers[0])
|
||||
assert.Equal(t, "body_string_item$$name", original.SemconvMaterializedColumns["item.name"])
|
||||
assert.Equal(t, "items.name", original.JSONPlan[0].Name)
|
||||
assert.Equal(t, "body_v2", original.JSONPlan[0].Parent.Name)
|
||||
}
|
||||
|
||||
func TestFieldKeySelectorCopyOwnsMetricContext(t *testing.T) {
|
||||
original := &FieldKeySelector{
|
||||
Name: "state",
|
||||
MetricContext: &MetricContext{
|
||||
MetricName: "system.cpu.time",
|
||||
MetricNamespace: "system",
|
||||
},
|
||||
}
|
||||
|
||||
copied := original.Copy()
|
||||
require.NotNil(t, copied)
|
||||
require.NotNil(t, copied.MetricContext)
|
||||
assert.NotSame(t, original.MetricContext, copied.MetricContext)
|
||||
|
||||
copied.Name = "changed"
|
||||
copied.MetricContext.MetricName = "changed"
|
||||
|
||||
assert.Equal(t, "state", original.Name)
|
||||
assert.Equal(t, "system.cpu.time", original.MetricContext.MetricName)
|
||||
}
|
||||
|
||||
func TestNilFieldCopies(t *testing.T) {
|
||||
assert.Nil(t, (*TelemetryFieldKey)(nil).Copy())
|
||||
assert.Nil(t, (*FieldKeySelector)(nil).Copy())
|
||||
assert.Nil(t, (*MetricContext)(nil).Copy())
|
||||
}
|
||||
Reference in New Issue
Block a user