Compare commits

..

5 Commits

Author SHA1 Message Date
srikanthccv
6b4bb82efb feat: resolve semconv families across logs and metrics 2026-08-07 01:20:58 +05:30
srikanthccv
487d00b6c9 test: add semantic convention phase one closure gate 2026-08-07 01:20:55 +05:30
srikanthccv
59329d3aaa feat: support semconv evolution in services 2026-08-07 01:20:49 +05:30
srikanthccv
7c58b6af49 feat: resolve semantic convention names in trace queries 2026-08-07 01:20:08 +05:30
srikanthccv
e730192885 feat: generate semantic convention families 2026-08-07 01:20:01 +05:30
51 changed files with 4503 additions and 260 deletions

View File

@@ -53,6 +53,21 @@ jobs:
with:
PRIMUS_REF: main
GO_VERSION: 1.24
semconv-generated:
if: |
github.event_name == 'merge_group' ||
(github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))
runs-on: ubuntu-latest
steps:
- name: self-checkout
uses: actions/checkout@v4
- name: go-install
uses: actions/setup-go@v5
with:
go-version: "1.24"
- name: check-semconv-generated-files
run: go run ./scripts/semconv -check
build:
if: |
github.event_name == 'merge_group' ||

View File

@@ -220,6 +220,14 @@ py-test-teardown: ## Tear down the shared SigNoz backend
py-test: ## Runs integration tests
@cd tests && uv run pytest --basetemp=./tmp/ -vv --capture=no integration/tests/
.PHONY: py-test-semconv-phase1
py-test-semconv-phase1: py-test-setup ## Rebuild the shared stack and run the semantic-convention Phase 1 matrix
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py
.PHONY: py-test-semconv-phase2
py-test-semconv-phase2: py-test-setup ## Rebuild the shared stack and run the Phase 1-2 cross-signal matrices
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py integration/tests/queriersemconv/02_cross_signal.py
.PHONY: py-clean
py-clean: ## Clear all pycache and pytest cache from tests directory recursively
@echo ">> cleaning python cache files from tests directory"
@@ -233,6 +241,10 @@ py-clean: ## Clear all pycache and pytest cache from tests directory recursively
##############################################################
# generate commands
##############################################################
.PHONY: semconv-generate
semconv-generate: ## Regenerate semantic-convention families for Go and TypeScript
@go run ./scripts/semconv
.PHONY: gen-mocks
gen-mocks:
@echo ">> Generating mocks"

View File

@@ -0,0 +1,59 @@
// Code generated by scripts/semconv. DO NOT EDIT.
export type SemconvFamily = {
readonly current: string;
readonly old: readonly string[];
readonly kind: 'attribute' | 'metric';
readonly contexts: readonly string[];
readonly signals: readonly string[];
readonly applyToMetrics: readonly string[];
readonly valueMap: Readonly<Record<string, string>>;
};
export const SEMCONV_FAMILIES: readonly SemconvFamily[] = [
{
current: 'container.cpu.usage',
old: ['container.cpu.utilization'],
kind: 'metric',
contexts: ['metric'],
signals: ['metrics'],
applyToMetrics: [],
valueMap: {},
},
{
current: 'db.system.name',
old: ['db.system'],
kind: 'attribute',
contexts: ['attribute', 'resource'],
signals: ['logs', 'metrics', 'traces'],
applyToMetrics: [],
valueMap: {},
},
{
current: 'deployment.environment.name',
old: ['deployment.environment'],
kind: 'attribute',
contexts: ['attribute', 'resource'],
signals: ['logs', 'metrics', 'traces'],
applyToMetrics: [],
valueMap: {},
},
{
current: 'k8s.node.cpu.usage',
old: ['k8s.node.cpu.utilization'],
kind: 'metric',
contexts: ['metric'],
signals: ['metrics'],
applyToMetrics: [],
valueMap: {},
},
{
current: 'k8s.pod.cpu.usage',
old: ['k8s.pod.cpu.utilization'],
kind: 'metric',
contexts: ['metric'],
signals: ['metrics'],
applyToMetrics: [],
valueMap: {},
},
] as const;

View File

@@ -52,10 +52,10 @@ import (
"github.com/SigNoz/signoz/pkg/query-service/constants"
chErrors "github.com/SigNoz/signoz/pkg/query-service/errors"
"github.com/SigNoz/signoz/pkg/query-service/metrics"
"github.com/SigNoz/signoz/pkg/query-service/model"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/utils"
"github.com/SigNoz/signoz/pkg/semconv"
)
const (
@@ -3190,11 +3190,6 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID v
var rows driver.Rows
var attributeValues v3.FilterAttributeValueResponse
normalized := true
if constants.IsDotMetricsEnabled {
normalized = false
}
reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID))
if reductionEnabled {
@@ -3205,8 +3200,7 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID v
if req.Limit != 0 {
query = query + fmt.Sprintf(" LIMIT %d;", req.Limit)
}
names := []string{req.AggregateAttribute}
names = append(names, metrics.GetTransitionedMetric(req.AggregateAttribute, normalized))
names := semconv.MetricNames(req.AggregateAttribute)
rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, names, req.FilterAttributeKey, fmt.Sprintf("%%%s%%", req.SearchText), common.PastDayRoundOff())

View File

@@ -6,6 +6,8 @@ import (
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/utils"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
var resourceLogOperators = map[v3.FilterOperator]string{
@@ -29,13 +31,61 @@ var resourceLogOperators = map[v3.FilterOperator]string{
v3.FilterOperatorNotILike: "NOT ILIKE",
}
func resourceSemconvMembers(key string) []string {
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: key,
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
})
}
func resourceValueExpression(key string) string {
members := resourceSemconvMembers(key)
if len(members) == 1 {
return fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key)
}
values := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(simpleJSONExtractString(labels, '%s'), '')", member))
}
return "COALESCE(" + strings.Join(values, ", ") + ")"
}
func resourcePresenceExpression(key string, exists bool) string {
members := resourceSemconvMembers(key)
if len(members) == 1 {
if exists {
return fmt.Sprintf("simpleJSONHas(labels, '%s')", key)
}
return fmt.Sprintf("not simpleJSONHas(labels, '%s')", key)
}
conditions := make([]string, 0, len(members))
for _, member := range members {
if exists {
conditions = append(conditions, fmt.Sprintf("simpleJSONHas(labels, '%s')", member))
} else {
conditions = append(conditions, fmt.Sprintf("not simpleJSONHas(labels, '%s')", member))
}
}
separator := " OR "
if !exists {
separator = " AND "
}
return "(" + strings.Join(conditions, separator) + ")"
}
// buildResourceFilter builds a clickhouse filter string for resource labels
func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value interface{}) string {
// for all operators except contains and like
searchKey := fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key)
searchKey := resourceValueExpression(key)
// for contains and like it will be case insensitive
lowerSearchKey := fmt.Sprintf("simpleJSONExtractString(lower(labels), '%s')", key)
if len(resourceSemconvMembers(key)) > 1 {
lowerSearchKey = "lower(" + searchKey + ")"
}
chFmtVal := utils.ClickHouseFormattedValue(value)
@@ -43,9 +93,9 @@ func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value
switch op {
case v3.FilterOperatorExists:
return fmt.Sprintf("simpleJSONHas(labels, '%s')", key)
return resourcePresenceExpression(key, true)
case v3.FilterOperatorNotExists:
return fmt.Sprintf("not simpleJSONHas(labels, '%s')", key)
return resourcePresenceExpression(key, false)
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex:
return fmt.Sprintf(logsOp, searchKey, chFmtVal)
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
@@ -110,6 +160,38 @@ func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value inter
// we can use lower index for =, in etc but it's difficult to do it for !=, NIN etc
// if as x != "ABC" we cannot predict something like "not lower(labels) like '%%x%%abc%%'". It has it be "not lower(labels) like '%%x%%ABC%%'"
func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{}) string {
return buildResourceIndexFilterForKey(key, op, value, true)
}
func buildResourceIndexFilterForKey(key string, op v3.FilterOperator, value interface{}, resolveFamily bool) string {
members := []string{key}
if resolveFamily {
members = resourceSemconvMembers(key)
}
if len(members) > 1 {
switch op {
case v3.FilterOperatorNotEqual,
v3.FilterOperatorNotLike,
v3.FilterOperatorNotILike,
v3.FilterOperatorNotContains,
v3.FilterOperatorNotExists,
v3.FilterOperatorNotRegex,
v3.FilterOperatorNotIn:
return ""
}
conditions := make([]string, 0, len(members))
for _, member := range members {
if condition := buildResourceIndexFilterForKey(member, op, value, false); condition != "" {
conditions = append(conditions, condition)
}
}
if len(conditions) == 0 {
return ""
}
return "(" + strings.Join(conditions, " OR ") + ")"
}
// not using clickhouseFormattedValue as we don't wan't the quotes
strVal := fmt.Sprintf("%s", value)
fmtValEscapedForContains := utils.QuoteEscapedStringForContains(strVal, true)
@@ -206,14 +288,31 @@ func buildResourceFiltersFromGroupBy(groupBy []v3.AttributeKey) []string {
if attr.Type != v3.AttributeKeyTypeResource {
continue
}
conditions = append(conditions, fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", attr.Key, attr.Key))
members := resourceSemconvMembers(attr.Key)
if len(members) == 1 {
conditions = append(conditions, fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", attr.Key, attr.Key))
continue
}
indexConditions := make([]string, 0, len(members))
for _, member := range members {
indexConditions = append(indexConditions, fmt.Sprintf("labels like '%%%s%%'", member))
}
conditions = append(conditions, fmt.Sprintf("(%s AND (%s))", resourcePresenceExpression(attr.Key, true), strings.Join(indexConditions, " OR ")))
}
return conditions
}
func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeKey) string {
if aggregateAttribute.Key != "" && aggregateAttribute.Type == v3.AttributeKeyTypeResource {
return fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", aggregateAttribute.Key, aggregateAttribute.Key)
members := resourceSemconvMembers(aggregateAttribute.Key)
if len(members) == 1 {
return fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", aggregateAttribute.Key, aggregateAttribute.Key)
}
indexConditions := make([]string, 0, len(members))
for _, member := range members {
indexConditions = append(indexConditions, fmt.Sprintf("labels like '%%%s%%'", member))
}
return fmt.Sprintf("(%s AND (%s))", resourcePresenceExpression(aggregateAttribute.Key, true), strings.Join(indexConditions, " OR "))
}
return ""

View File

@@ -5,6 +5,8 @@ import (
"testing"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_buildResourceFilter(t *testing.T) {
@@ -552,3 +554,38 @@ func Test_buildResourceSubQuery(t *testing.T) {
})
}
}
func TestSemanticConventionResourceFamily(t *testing.T) {
const resolvedValue = "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''))"
for _, requestedName := range []string{"deployment.environment.name", "deployment.environment"} {
t.Run(requestedName, func(t *testing.T) {
assert.Equal(t, resolvedValue+" = 'production'", buildResourceFilter("=", requestedName, v3.FilterOperatorEqual, "production"))
assert.Equal(t, "(simpleJSONHas(labels, 'deployment.environment.name') OR simpleJSONHas(labels, 'deployment.environment'))", buildResourceFilter("", requestedName, v3.FilterOperatorExists, nil))
assert.Equal(t, "(not simpleJSONHas(labels, 'deployment.environment.name') AND not simpleJSONHas(labels, 'deployment.environment'))", buildResourceFilter("", requestedName, v3.FilterOperatorNotExists, nil))
assert.Equal(t, "(labels like '%deployment.environment.name\":\"production%' OR labels like '%deployment.environment\":\"production%')", buildResourceIndexFilter(requestedName, v3.FilterOperatorEqual, "production"))
assert.Empty(t, buildResourceIndexFilter(requestedName, v3.FilterOperatorNotEqual, "production"), "negative family filter must not use a rejecting index hint")
})
}
filters, err := buildResourceFiltersFromFilterItems(&v3.FilterSet{Items: []v3.FilterItem{{
Key: v3.AttributeKey{
Key: "deployment.environment.name",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeResource,
},
Operator: v3.FilterOperatorEqual,
Value: "production",
}}})
require.NoError(t, err, "family filter items must build before their output is inspected")
wantFilters := []string{
resolvedValue + " = 'production'",
"(labels like '%deployment.environment.name\":\"production%' OR labels like '%deployment.environment\":\"production%')",
}
assert.Equal(t, wantFilters, filters)
wantPresence := "((simpleJSONHas(labels, 'deployment.environment.name') OR simpleJSONHas(labels, 'deployment.environment')) AND (labels like '%deployment.environment.name%' OR labels like '%deployment.environment%'))"
groupBy := buildResourceFiltersFromGroupBy([]v3.AttributeKey{{Key: "deployment.environment", Type: v3.AttributeKeyTypeResource}})
assert.Equal(t, []string{wantPresence}, groupBy)
assert.Equal(t, wantPresence, buildResourceFiltersFromAggregateAttribute(v3.AttributeKey{Key: "deployment.environment.name", Type: v3.AttributeKeyTypeResource}))
}

View File

@@ -6,16 +6,32 @@ import (
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/SigNoz/signoz/pkg/query-service/model"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
var (
columns = map[string]struct{}{
"deployment_environment": {},
"k8s_cluster_name": {},
"k8s_namespace_name": {},
}
columns = serviceMapColumns()
)
func serviceMapColumns() map[string]string {
columns := map[string]string{
"k8s_cluster_name": "k8s_cluster_name",
"k8s_namespace_name": "k8s_namespace_name",
}
// Dependency-graph rows keep their historical physical column name. Both
// semantic-convention request spellings target that same derived column.
for _, member := range semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
}) {
columns[strings.ReplaceAll(member, ".", "_")] = "deployment_environment"
}
return columns
}
func BuildServiceMapQuery(tags []model.TagQuery) (string, []interface{}) {
var filterQuery string
var namedArgs []interface{}
@@ -24,39 +40,40 @@ func BuildServiceMapQuery(tags []model.TagQuery) (string, []interface{}) {
operator := tag.GetOperator()
value := tag.GetValues()
if _, ok := columns[key]; !ok {
column, ok := columns[key]
if !ok {
continue
}
switch operator {
case model.InOperator:
filterQuery += fmt.Sprintf(" AND %s IN @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s IN @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.NotInOperator:
filterQuery += fmt.Sprintf(" AND %s NOT IN @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s NOT IN @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.EqualOperator:
filterQuery += fmt.Sprintf(" AND %s = @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s = @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.NotEqualOperator:
filterQuery += fmt.Sprintf(" AND %s != @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s != @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.ContainsOperator:
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%%%s%%", value)))
case model.NotContainsOperator:
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%%%s%%", value)))
case model.StartsWithOperator:
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%s%%", value)))
case model.NotStartsWithOperator:
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%s%%", value)))
case model.ExistsOperator:
filterQuery += fmt.Sprintf(" AND %s IS NOT NULL", key)
filterQuery += fmt.Sprintf(" AND %s IS NOT NULL", column)
case model.NotExistsOperator:
filterQuery += fmt.Sprintf(" AND %s IS NULL", key)
filterQuery += fmt.Sprintf(" AND %s IS NULL", column)
}
}
return filterQuery, namedArgs

View File

@@ -0,0 +1,35 @@
package services
import (
"testing"
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/SigNoz/signoz/pkg/query-service/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestBuildServiceMapQueryAcceptsEnvironmentFamily(t *testing.T) {
for _, requestedName := range []string{"deployment.environment.name", "deployment.environment"} {
t.Run(requestedName, func(t *testing.T) {
tags := []model.TagQuery{model.NewTagQueryString(model.TagQueryParam{
Key: requestedName,
StringValues: []string{"production"},
Operator: model.InOperator,
TagType: model.ResourceAttributeTagType,
})}
query, args := BuildServiceMapQuery(tags)
argName := "deployment_environment"
if requestedName == "deployment.environment.name" {
argName = "deployment_environment_name"
}
assert.Equal(t, " AND deployment_environment IN @"+argName, query)
require.Len(t, args, 1)
named, ok := args[0].(driver.NamedValue)
require.True(t, ok)
assert.Equal(t, argName, named.Name)
assert.Equal(t, []interface{}{"production"}, named.Value)
})
}
}

View File

@@ -1,27 +0,0 @@
package metrics
var MetricsUnderTransition = map[string]string{
"k8s_pod_cpu_utilization": "k8s_pod_cpu_usage",
"k8s_node_cpu_utilization": "k8s_node_cpu_usage",
"container_cpu_utilization": "container_cpu_usage",
}
var DotMetricsUnderTransition = map[string]string{
"k8s.pod.cpu.utilization": "k8s.pod.cpu.usage",
"k8s.node.cpu.utilization": "k8s.node.cpu.usage",
"container.cpu.utilization": "container.cpu.usage",
}
func GetTransitionedMetric(metric string, normalized bool) string {
if normalized {
if _, ok := MetricsUnderTransition[metric]; ok {
return MetricsUnderTransition[metric]
}
return metric
} else {
if _, ok := DotMetricsUnderTransition[metric]; ok {
return DotMetricsUnderTransition[metric]
}
return metric
}
}

View File

@@ -10,8 +10,8 @@ import (
"log/slog"
"github.com/SigNoz/signoz/pkg/query-service/constants"
"github.com/SigNoz/signoz/pkg/query-service/metrics"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/semconv"
)
// ValidateAndCastValue validates and casts the value of a key to the corresponding data type of the key
@@ -234,12 +234,12 @@ func ClickHouseFormattedValue(v interface{}) string {
func ClickHouseFormattedMetricNames(v interface{}) string {
if name, ok := v.(string); ok {
transitionedMetrics := metrics.GetTransitionedMetric(name, !constants.IsDotMetricsEnabled)
if transitionedMetrics != name {
return ClickHouseFormattedValue([]interface{}{transitionedMetrics})
} else {
return ClickHouseFormattedValue([]interface{}{name})
members := semconv.MetricNames(name)
values := make([]interface{}, 0, len(members))
for _, member := range members {
values = append(values, member)
}
return ClickHouseFormattedValue(values)
}
return ClickHouseFormattedValue(v)

View File

@@ -2,13 +2,29 @@ package querybuilder
import (
"fmt"
"strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/semconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
func physicalSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
if len(key.SemconvMembers) > 0 {
return key.SemconvMembers
}
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
return []string{key.Name}
}
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: key.Signal,
FieldContext: key.FieldContext,
})
}
// ExistsExpression renders the existence predicate for a key resolved to the given
// columns (negated when exists is false). Comparisons are against constants rendered
// as literals, so the expression carries no bind args and can guard column expressions
@@ -43,11 +59,26 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil {
columnName = evolutionsEntries[0].ColumnName
}
rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name)
if exists {
return rawPath + " IS NOT NULL", nil
members := physicalSemconvMembers(key)
paths := make([]string, 0, len(members))
for _, member := range members {
paths = append(paths, fmt.Sprintf("%s.`%s`", columnName, member))
}
return rawPath + " IS NULL", nil
if len(paths) == 1 {
if exists {
return paths[0] + " IS NOT NULL", nil
}
return paths[0] + " IS NULL", nil
}
guards := make([]string, 0, len(paths))
for _, path := range paths {
guards = append(guards, path+" IS NOT NULL")
}
rawPath := "(" + strings.Join(guards, " OR ") + ")"
if exists {
return rawPath, nil
}
return "NOT " + rawPath, nil
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumFixedString:
if exists {
@@ -88,8 +119,16 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
if key.Materialized {
members := physicalSemconvMembers(key)
operands := make([]string, 0, len(members))
for _, member := range members {
operands = append(operands, fmt.Sprintf("mapContains(%s, '%s')", column.Name, member))
}
leftOperand := strings.Join(operands, " OR ")
if len(operands) > 1 {
leftOperand = "(" + leftOperand + ")"
}
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
}
if exists {

View File

@@ -10,6 +10,7 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
"github.com/SigNoz/signoz/pkg/semconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -38,6 +39,7 @@ type filterExpressionVisitor struct {
fullTextColumn *telemetrytypes.TelemetryFieldKey
skipResourceFilter bool
skipFullTextFilter bool
exactSemconv bool
variables map[string]qbtypes.VariableItem
keysWithWarnings map[string]bool
@@ -58,6 +60,7 @@ type FilterExprVisitorOpts struct {
FullTextColumn *telemetrytypes.TelemetryFieldKey
SkipResourceFilter bool
SkipFullTextFilter bool
ExactSemconv bool
Variables map[string]qbtypes.VariableItem
StartNs uint64
EndNs uint64
@@ -75,6 +78,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
fullTextColumn: opts.FullTextColumn,
skipResourceFilter: opts.SkipResourceFilter,
skipFullTextFilter: opts.SkipFullTextFilter,
exactSemconv: opts.ExactSemconv,
variables: opts.Variables,
keysWithWarnings: make(map[string]bool),
startNs: opts.StartNs,
@@ -379,7 +383,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
// VisitComparison handles all comparison operators.
func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
key := v.Visit(ctx.Key()).(*telemetrytypes.TelemetryFieldKey)
matching := MatchingFieldKeys(key, v.fieldKeys)
matching := v.matchingFieldKeys(key)
// Handle EXISTS specially
if ctx.EXISTS() != nil {
@@ -730,7 +734,7 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
return ErrorConditionLiteral
}
conds, ok := v.buildConditions(key, MatchingFieldKeys(key, v.fieldKeys), operator, value)
conds, ok := v.buildConditions(key, v.matchingFieldKeys(key), operator, value)
if !ok {
return ErrorConditionLiteral
}
@@ -923,7 +927,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
// buildConditions invokes the condition builder for a filter term, folding its
// warnings/errors into visitor state; returns false if an error was recorded.
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) ([]string, bool) {
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder)
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter, ExactSemconv: v.exactSemconv}, op, value, v.builder)
if err != nil {
_, _, _, _, errURL, _ := errors.Unwrapb(err)
assignIfEmpty(&v.mainErrorURL, errURL)
@@ -982,27 +986,116 @@ func assignIfEmpty(s *string, value string) {
// MatchingFieldKeys returns the field keys from the map that match the given key,
// honoring any context/data type the user specified.
func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
fieldKeysForName := []*telemetrytypes.TelemetryFieldKey{}
return matchingFieldKeys(field, fieldKeys, true)
}
// match by name; keep items whose context and data type match (unspecified matches any)
for _, item := range fieldKeys[field.Name] {
if (field.FieldContext == telemetrytypes.FieldContextUnspecified || field.FieldContext == item.FieldContext) &&
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || field.FieldDataType == item.FieldDataType) {
fieldKeysForName = append(fieldKeysForName, item)
// MatchingFieldKeysExact matches only the requested physical spelling.
func MatchingFieldKeysExact(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
return matchingFieldKeys(field, fieldKeys, false)
}
func matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, resolveSemconv bool) []*telemetrytypes.TelemetryFieldKey {
selector := telemetrytypes.FieldKeySelector{
Name: field.Name,
Signal: field.Signal,
FieldContext: field.FieldContext,
}
members := []string{field.Name}
if resolveSemconv {
members = semconv.AttributeMembers(selector)
}
isFamily := len(members) > 1
fieldKeysForName := make([]*telemetrytypes.TelemetryFieldKey, 0)
indexByIdentity := make(map[string]int)
appendMatches := func(lookupName string, memberName string, contextAlreadyMatched bool) {
for _, item := range fieldKeys[lookupName] {
if !contextAlreadyMatched && field.FieldContext != telemetrytypes.FieldContextUnspecified && field.FieldContext != item.FieldContext {
continue
}
if field.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && field.FieldDataType != item.FieldDataType {
continue
}
// A wildcard lookup may have found a same-named field in a scope where
// this family does not apply. Keep exact names, but reject cross-member
// matches outside the generated family scope.
if resolveSemconv && memberName != field.Name {
itemSelector := telemetrytypes.FieldKeySelector{
Name: field.Name,
Signal: item.Signal,
FieldContext: item.FieldContext,
}
if !slices.Contains(semconv.AttributeMembers(itemSelector), memberName) {
continue
}
}
physicalMembers := item.SemconvMembers
if len(physicalMembers) == 0 {
physicalMembers = []string{memberName}
}
identity := item.Signal.StringValue() + ";" + item.FieldContext.StringValue() + ";" + item.FieldDataType.StringValue()
if isFamily {
if index, found := indexByIdentity[identity]; found {
for _, physicalMember := range physicalMembers {
if !slices.Contains(fieldKeysForName[index].SemconvMembers, physicalMember) {
fieldKeysForName[index].SemconvMembers = append(fieldKeysForName[index].SemconvMembers, physicalMember)
}
}
continue
}
indexByIdentity[identity] = len(fieldKeysForName)
}
resolved := *item
// The requested spelling is the response identity. Field mappers use
// it to resolve the available family members current-first.
if isFamily {
resolved.Name = field.Name
resolved.SemconvMembers = slices.Clone(physicalMembers)
} else if !resolveSemconv {
resolved.SemconvMembers = []string{field.Name}
}
fieldKeysForName = append(fieldKeysForName, &resolved)
}
}
// A context may have been split off a name that legitimately contained it (e.g.
// `attribute.key`); also look up the context-prefixed name so both readings resolve.
// Members are current-first, so metadata from the current key wins when
// both spellings describe the same signal/context/type.
for _, member := range members {
appendMatches(member, member, false)
}
// A context may have been split off a name that legitimately contained it
// (e.g. `attribute.key`); preserve that historical alternate reading for
// every family member.
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
contextPrefixedFieldName := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)
for _, item := range fieldKeys[contextPrefixedFieldName] {
// Context already matched via the lookup key; only data type needs checking.
if field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || item.FieldDataType == field.FieldDataType {
fieldKeysForName = append(fieldKeysForName, item)
}
for _, member := range members {
appendMatches(fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), member), member, true)
}
}
return fieldKeysForName
}
func (v *filterExpressionVisitor) matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
if v.exactSemconv {
return MatchingFieldKeysExact(field, v.fieldKeys)
}
return MatchingFieldKeys(field, v.fieldKeys)
}
// ExactSemconvKeys returns copies pinned to their physical names, preventing a
// field mapper from expanding a synthesized or metadata-free key into a family.
func ExactSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
result := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
for _, key := range keys {
if key == nil {
continue
}
resolved := *key
resolved.SemconvMembers = []string{key.Name}
result = append(result, &resolved)
}
return result
}

View File

@@ -14,6 +14,7 @@ import (
"github.com/antlr4-go/antlr/v4"
sqlbuilder "github.com/huandu/go-sqlbuilder"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestPrepareWhereClause_EmptyVariableList ensures PrepareWhereClause errors when a variable has an empty list value.
@@ -685,6 +686,58 @@ func TestVisitKey(t *testing.T) {
}
}
func TestMatchingFieldKeysResolvesSemconvFamily(t *testing.T) {
current := &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment.name",
Description: "current metadata",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
old := &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
Description: "old metadata",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
current.Name: {current},
old.Name: {old},
}
for _, requestedName := range []string{current.Name, old.Name} {
requested := telemetrytypes.NewTelemetryFieldKey(
requestedName,
telemetrytypes.FieldContextResource,
telemetrytypes.FieldDataTypeString,
)
matches := MatchingFieldKeys(requested, fieldKeys)
require.Len(t, matches, 1, "family lookup must return one field before its metadata is inspected")
assert.Equal(t, requestedName, matches[0].Name)
assert.Equal(t, "current metadata", matches[0].Description)
assert.Equal(t, []string{current.Name, old.Name}, matches[0].SemconvMembers)
}
// A current-name query still resolves when metadata has seen only the old
// spelling. The returned name remains the request identity.
requested := telemetrytypes.NewTelemetryFieldKey(
current.Name,
telemetrytypes.FieldContextResource,
telemetrytypes.FieldDataTypeString,
)
matches := MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{old.Name: {old}})
require.Len(t, matches, 1, "family lookup must return one field before its metadata is inspected")
assert.Equal(t, current.Name, matches[0].Name)
assert.Equal(t, "old metadata", matches[0].Description)
assert.Equal(t, []string{old.Name}, matches[0].SemconvMembers)
exactMatches := MatchingFieldKeysExact(requested, fieldKeys)
require.Len(t, exactMatches, 1, "exact lookup must return one field before its metadata is inspected")
assert.Equal(t, current.Name, exactMatches[0].Name)
assert.Equal(t, []string{current.Name}, exactMatches[0].SemconvMembers)
}
// ---------------------------------------------------------------------------
// TestVisitComparison
// ---------------------------------------------------------------------------

3
pkg/semconv/doc.go Normal file
View File

@@ -0,0 +1,3 @@
// Package semconv resolves historical OpenTelemetry semantic-convention names
// into logical field families.
package semconv

View File

@@ -0,0 +1,48 @@
// Code generated by scripts/semconv. DO NOT EDIT.
package semconv
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
var families = []Family{
{
Current: "container.cpu.usage",
Old: []string{"container.cpu.utilization"},
Kind: KindMetric,
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
ApplyToMetrics: nil,
},
{
Current: "db.system.name",
Old: []string{"db.system"},
Kind: KindAttribute,
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource},
Signals: []telemetrytypes.Signal{telemetrytypes.SignalLogs, telemetrytypes.SignalMetrics, telemetrytypes.SignalTraces},
ApplyToMetrics: nil,
},
{
Current: "deployment.environment.name",
Old: []string{"deployment.environment"},
Kind: KindAttribute,
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource},
Signals: []telemetrytypes.Signal{telemetrytypes.SignalLogs, telemetrytypes.SignalMetrics, telemetrytypes.SignalTraces},
ApplyToMetrics: nil,
},
{
Current: "k8s.node.cpu.usage",
Old: []string{"k8s.node.cpu.utilization"},
Kind: KindMetric,
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
ApplyToMetrics: nil,
},
{
Current: "k8s.pod.cpu.usage",
Old: []string{"k8s.pod.cpu.utilization"},
Kind: KindMetric,
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
ApplyToMetrics: nil,
},
}

265
pkg/semconv/semconv.go Normal file
View File

@@ -0,0 +1,265 @@
package semconv
import (
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
//go:generate go run ../../scripts/semconv
// Kind identifies whether a family describes an attribute or a metric name.
type Kind struct {
valuer.String
}
// Family is one logical telemetry field. Old is ordered from the most recent
// predecessor to the oldest one and therefore also defines fallback order.
type Family struct {
Current string
Old []string
Kind Kind
Contexts []telemetrytypes.FieldContext
Signals []telemetrytypes.Signal
ApplyToMetrics []string
ValueMap map[string]string
}
type metricSpelling uint8
const (
metricSpellingDotted metricSpelling = iota
metricSpellingNormalized
)
var (
KindAttribute = Kind{String: valuer.NewString("attribute")}
KindMetric = Kind{String: valuer.NewString("metric")}
)
var memberToFamilies, familyMembers = buildIndexes()
// Enum returns the acceptable values for Kind.
func (Kind) Enum() []any {
return []any{KindAttribute, KindMetric}
}
// Lookup returns the enabled family containing selector.Name for kind. The
// slices and map in the returned value reference immutable generated data and
// must be treated as read-only. Use All when a mutable copy is required.
func Lookup(kind Kind, selector telemetrytypes.FieldKeySelector) (Family, bool) {
idx, ok := lookupIndex(kind, selector)
if !ok {
return Family{}, false
}
return families[idx], true
}
// Members returns the current name first, followed by historical names in
// fallback order. For enabled families the returned precomputed slice is
// read-only. A name outside an enabled family is returned unchanged.
func Members(kind Kind, selector telemetrytypes.FieldKeySelector) []string {
idx, ok := lookupIndex(kind, selector)
if !ok {
return []string{selector.Name}
}
return familyMembers[idx]
}
// AttributeMembers returns the physical attribute spellings that may represent
// selector.Name. Metrics have used both dotted and normalized label layouts;
// resource labels have additionally used a resource_ prefix. Keeping that
// storage detail here prevents metrics readers from maintaining local
// transition tables.
func AttributeMembers(selector telemetrytypes.FieldKeySelector) []string {
if selector.Signal != telemetrytypes.SignalMetrics {
return Members(KindAttribute, selector)
}
lookupSelector := selector
lookupSelector.Name = strings.TrimPrefix(selector.Name, "resource_")
family, style, ok := lookupMetricSpelling(KindAttribute, lookupSelector)
if !ok {
return []string{selector.Name}
}
logicalMembers := familyMembers[family]
result := make([]string, 0, len(logicalMembers)*4)
for _, member := range logicalMembers {
dotted := member
normalized := normalizeMetricSpelling(member)
variants := []string{dotted, normalized}
if style == metricSpellingNormalized {
variants[0], variants[1] = variants[1], variants[0]
}
if selector.FieldContext == telemetrytypes.FieldContextResource ||
selector.FieldContext == telemetrytypes.FieldContextUnspecified ||
strings.HasPrefix(selector.Name, "resource_") {
for _, variant := range variants {
result = appendUniqueString(result, "resource_"+variant)
}
}
for _, variant := range variants {
result = appendUniqueString(result, variant)
}
}
return result
}
// MetricNames returns the current and historical storage names for a metric.
// The input's dotted or normalized style is preserved because both layouts are
// valid metric identities and must not be mixed in one query.
func MetricNames(name string) []string {
selector := telemetrytypes.FieldKeySelector{
Name: name,
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextMetric,
}
family, style, ok := lookupMetricSpelling(KindMetric, selector)
if !ok {
return []string{name}
}
logicalMembers := familyMembers[family]
result := make([]string, 0, len(logicalMembers))
for _, member := range logicalMembers {
if style == metricSpellingNormalized {
member = normalizeMetricSpelling(member)
}
result = appendUniqueString(result, member)
}
return result
}
// CurrentAttribute returns the canonical dotted name for an attribute
// spelling, or selector.Name if no enabled family matches.
func CurrentAttribute(selector telemetrytypes.FieldKeySelector) string {
if selector.Signal != telemetrytypes.SignalMetrics {
return Current(KindAttribute, selector)
}
lookupSelector := selector
lookupSelector.Name = strings.TrimPrefix(selector.Name, "resource_")
family, _, ok := lookupMetricSpelling(KindAttribute, lookupSelector)
if !ok {
return selector.Name
}
return families[family].Current
}
// Current returns the current name for selector.Name, or the input name when
// it does not belong to an enabled family.
func Current(kind Kind, selector telemetrytypes.FieldKeySelector) string {
idx, ok := lookupIndex(kind, selector)
if !ok {
return selector.Name
}
return families[idx].Current
}
// All returns a defensive copy of every enabled family.
func All() []Family {
result := make([]Family, len(families))
for i := range families {
result[i] = cloneFamily(families[i])
}
return result
}
func buildIndexes() (map[string][]int, [][]string) {
index := make(map[string][]int)
members := make([][]string, len(families))
for i, family := range families {
members[i] = make([]string, 0, len(family.Old)+1)
members[i] = append(members[i], family.Current)
members[i] = append(members[i], family.Old...)
index[family.Current] = append(index[family.Current], i)
for _, old := range family.Old {
index[old] = append(index[old], i)
}
}
return index, members
}
func lookupIndex(kind Kind, selector telemetrytypes.FieldKeySelector) (int, bool) {
for _, idx := range memberToFamilies[selector.Name] {
if matchesSelector(families[idx], kind, selector) {
return idx, true
}
}
return 0, false
}
func lookupMetricSpelling(kind Kind, selector telemetrytypes.FieldKeySelector) (int, metricSpelling, bool) {
if idx, ok := lookupIndex(kind, selector); ok {
return idx, metricSpellingDotted, true
}
for idx, family := range families {
if !matchesSelector(family, kind, selector) {
continue
}
for _, member := range familyMembers[idx] {
if normalizeMetricSpelling(member) == selector.Name {
return idx, metricSpellingNormalized, true
}
}
}
return 0, metricSpellingDotted, false
}
func normalizeMetricSpelling(name string) string {
return strings.ReplaceAll(name, ".", "_")
}
func appendUniqueString(values []string, value string) []string {
if value == "" || slices.Contains(values, value) {
return values
}
return append(values, value)
}
func matchesSelector(family Family, kind Kind, selector telemetrytypes.FieldKeySelector) bool {
if family.Kind != kind {
return false
}
if selector.Signal != telemetrytypes.SignalUnspecified && len(family.Signals) > 0 {
if !slices.Contains(family.Signals, selector.Signal) {
return false
}
}
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && len(family.Contexts) > 0 {
if !slices.Contains(family.Contexts, selector.FieldContext) {
return false
}
}
if selector.Signal == telemetrytypes.SignalMetrics && len(family.ApplyToMetrics) > 0 {
if selector.MetricContext == nil {
return false
}
return slices.Contains(family.ApplyToMetrics, selector.MetricContext.MetricName)
}
return true
}
func cloneFamily(family Family) Family {
valueMap := family.ValueMap
family.Old = slices.Clone(family.Old)
family.Contexts = slices.Clone(family.Contexts)
family.Signals = slices.Clone(family.Signals)
family.ApplyToMetrics = slices.Clone(family.ApplyToMetrics)
if valueMap != nil {
family.ValueMap = make(map[string]string, len(valueMap))
for old, current := range valueMap {
family.ValueMap[old] = current
}
}
return family
}

114
pkg/semconv/semconv_test.go Normal file
View File

@@ -0,0 +1,114 @@
package semconv
import (
"testing"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMembersReturnsCurrentBeforeHistoricalName(t *testing.T) {
selector := telemetrytypes.FieldKeySelector{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
}
assert.Equal(t,
[]string{"deployment.environment.name", "deployment.environment"},
Members(KindAttribute, selector),
"members should use current-first fallback order",
)
}
func TestCurrentReturnsCanonicalName(t *testing.T) {
selector := telemetrytypes.FieldKeySelector{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
}
assert.Equal(t,
"deployment.environment.name",
Current(KindAttribute, selector),
"historical name should resolve to the current family name",
)
}
func TestMembersReturnsInputWhenKindDoesNotMatch(t *testing.T) {
selector := telemetrytypes.FieldKeySelector{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
}
assert.Equal(t,
[]string{"deployment.environment"},
Members(KindMetric, selector),
"an attribute family must not match a metric-name lookup",
)
}
func TestAllReturnsDefensiveCopies(t *testing.T) {
first := All()
require.NotEmpty(t, first, "generated families must not be empty")
first[0].Old[0] = "mutated"
second := All()
assert.NotEqual(t, "mutated", second[0].Old[0], "All must not expose mutable generated data")
}
func TestAttributeMembersIncludesMetricResourceStorageSpellings(t *testing.T) {
selector := telemetrytypes.FieldKeySelector{
Name: "db.system.name",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextResource,
}
assert.Equal(t, []string{
"resource_db.system.name", "resource_db_system_name", "db.system.name", "db_system_name",
"resource_db.system", "resource_db_system", "db.system", "db_system",
}, AttributeMembers(selector), "resource metric attributes should cover every historical storage layout")
}
func TestCurrentAttributeResolvesNormalizedMetricResourceSpelling(t *testing.T) {
selector := telemetrytypes.FieldKeySelector{
Name: "resource_db_system",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextResource,
}
assert.Equal(t, "db.system.name", CurrentAttribute(selector), "normalized resource spelling should resolve to the dotted current name")
}
func TestAttributeMembersPreservesNormalizedMetricPointStyle(t *testing.T) {
selector := telemetrytypes.FieldKeySelector{
Name: "db_system",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextAttribute,
}
assert.Equal(t, []string{
"db_system_name", "db.system.name", "db_system", "db.system",
}, AttributeMembers(selector), "normalized point attribute should remain the preferred storage spelling")
}
func TestMetricNamesPreservesDottedStyle(t *testing.T) {
assert.Equal(t,
[]string{"k8s.pod.cpu.usage", "k8s.pod.cpu.utilization"},
MetricNames("k8s.pod.cpu.usage"),
"dotted metric input should produce dotted family names",
)
}
func TestMetricNamesPreservesNormalizedStyle(t *testing.T) {
assert.Equal(t,
[]string{"k8s_pod_cpu_usage", "k8s_pod_cpu_utilization"},
MetricNames("k8s_pod_cpu_utilization"),
"normalized metric input should produce normalized family names",
)
}
func TestMetricNamesReturnsUnknownNameUnchanged(t *testing.T) {
assert.Equal(t, []string{"custom_metric"}, MetricNames("custom_metric"), "unknown metrics should not be expanded")
}

View File

@@ -235,6 +235,7 @@ func NewSQLMigrationProviderFactories(
sqlmigration.NewFillDashboardSpecCollectionsFactory(sqlstore, dashboardStore),
sqlmigration.NewScrubEmailChannelTransportFactory(sqlstore),
sqlmigration.NewAddDashboardTuplesFactory(sqlstore),
sqlmigration.NewMigrateDeploymentEnvironmentQuickFilterFactory(),
)
}

View File

@@ -0,0 +1,127 @@
package sqlmigration
import (
"context"
"encoding/json"
"log/slog"
"time"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
)
const deploymentEnvironmentCurrent = "deployment.environment.name"
type migrateDeploymentEnvironmentQuickFilter struct {
logger *slog.Logger
}
type semconvQuickFilterRow struct {
bun.BaseModel `bun:"table:quick_filter"`
ID string `bun:"id"`
Filter string `bun:"filter"`
}
func NewMigrateDeploymentEnvironmentQuickFilterFactory() factory.ProviderFactory[SQLMigration, Config] {
return factory.NewProviderFactory(
factory.MustNewName("migrate_semconv_quick_filter"),
func(_ context.Context, settings factory.ProviderSettings, _ Config) (SQLMigration, error) {
return &migrateDeploymentEnvironmentQuickFilter{logger: settings.Logger}, nil
},
)
}
func (migration *migrateDeploymentEnvironmentQuickFilter) Register(migrations *migrate.Migrations) error {
return migrations.Register(migration.Up, migration.Down)
}
func deploymentEnvironmentOld() string {
members := semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: deploymentEnvironmentCurrent,
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
})
if len(members) < 2 {
return deploymentEnvironmentCurrent
}
return members[1]
}
func rewriteQuickFilterSemconv(filterJSON, from, to string) (string, bool, error) {
var filters []map[string]any
if err := json.Unmarshal([]byte(filterJSON), &filters); err != nil {
return "", false, err
}
changed := false
for _, filter := range filters {
if key, ok := filter["key"].(string); ok && key == from {
filter["key"] = to
changed = true
}
}
if !changed {
return filterJSON, false, nil
}
rewritten, err := json.Marshal(filters)
if err != nil {
return "", false, err
}
return string(rewritten), true, nil
}
func (migration *migrateDeploymentEnvironmentQuickFilter) migrate(ctx context.Context, db *bun.DB, from, to string) error {
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
rows := make([]*semconvQuickFilterRow, 0)
if err := tx.NewSelect().
Model(&rows).
Where("signal IN (?)", bun.In([]string{"traces", "api_monitoring", "exceptions"})).
Scan(ctx); err != nil {
return err
}
for _, row := range rows {
rewritten, changed, err := rewriteQuickFilterSemconv(row.Filter, from, to)
if err != nil {
// Quick filters are user-editable. One malformed legacy row must not
// prevent the application from starting or block every other org's
// migration.
if migration.logger != nil {
migration.logger.WarnContext(ctx, "skipping quick filter with unreadable filter JSON",
slog.String("quick_filter_id", row.ID), slog.Any("error", err))
}
continue
}
if !changed {
continue
}
if _, err := tx.NewUpdate().
Model((*semconvQuickFilterRow)(nil)).
Set("filter = ?", rewritten).
Set("updated_at = ?", time.Now()).
Where("id = ?", row.ID).
Exec(ctx); err != nil {
return err
}
}
return tx.Commit()
}
func (migration *migrateDeploymentEnvironmentQuickFilter) Up(ctx context.Context, db *bun.DB) error {
return migration.migrate(ctx, db, deploymentEnvironmentOld(), deploymentEnvironmentCurrent)
}
func (migration *migrateDeploymentEnvironmentQuickFilter) Down(ctx context.Context, db *bun.DB) error {
return migration.migrate(ctx, db, deploymentEnvironmentCurrent, deploymentEnvironmentOld())
}

View File

@@ -0,0 +1,38 @@
package sqlmigration
import (
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRewriteQuickFilterSemconv(t *testing.T) {
oldName := deploymentEnvironmentOld()
input := `[{"key":"service.name","dataType":"string","type":"resource"},{"key":"` + oldName + `","dataType":"string","type":"resource","custom":true}]`
rewritten, changed, err := rewriteQuickFilterSemconv(input, oldName, deploymentEnvironmentCurrent)
require.NoError(t, err)
assert.True(t, changed)
var filters []map[string]any
require.NoError(t, json.Unmarshal([]byte(rewritten), &filters))
assert.Equal(t, "service.name", filters[0]["key"])
assert.Equal(t, deploymentEnvironmentCurrent, filters[1]["key"])
assert.Equal(t, true, filters[1]["custom"], "unknown filter properties must be preserved")
restored, changed, err := rewriteQuickFilterSemconv(rewritten, deploymentEnvironmentCurrent, oldName)
require.NoError(t, err)
assert.True(t, changed)
require.NoError(t, json.Unmarshal([]byte(restored), &filters))
assert.Equal(t, oldName, filters[1]["key"])
}
func TestRewriteQuickFilterSemconvNoop(t *testing.T) {
input := `[{"key":"service.name","dataType":"string","type":"resource"}]`
rewritten, changed, err := rewriteQuickFilterSemconv(input, deploymentEnvironmentOld(), deploymentEnvironmentCurrent)
require.NoError(t, err)
assert.False(t, changed)
assert.Equal(t, input, rewritten)
}

View File

@@ -10,6 +10,7 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/statementbuilder"
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
@@ -31,6 +32,15 @@ const (
OthersMultiTemporality = `IF(LOWER(temporality) LIKE LOWER('delta'), %s, %s) AS per_series_value`
)
func metricNameValues(name string) []any {
members := semconv.MetricNames(name)
values := make([]any, 0, len(members))
for _, member := range members {
values = append(values, member)
}
return values
}
type StatementBuilder struct {
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
@@ -341,7 +351,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
sb.SelectMore(col)
}
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
sb.GTE("unix_milli", start),
sb.LTE("unix_milli", end),
)
@@ -385,7 +395,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", agg.MetricName),
sb.In("metric_name", metricNameValues(agg.MetricName)...),
sb.GTE("unix_milli", start),
sb.LT("unix_milli", end),
)
@@ -427,7 +437,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", agg.MetricName),
sb.In("metric_name", metricNameValues(agg.MetricName)...),
sb.GTE("unix_milli", start),
sb.LT("unix_milli", end),
)
@@ -505,7 +515,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
sb.GTE("unix_milli", start),
sb.LT("unix_milli", end),
)
@@ -560,7 +570,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
}
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
sb.GTE("unix_milli", start),
sb.LTE("unix_milli", end),
)
@@ -638,7 +648,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
sb.GTE("unix_milli", start),
sb.LT("unix_milli", end),
)
@@ -679,7 +689,7 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
baseSb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
baseSb.Where(
baseSb.In("metric_name", query.Aggregations[0].MetricName),
baseSb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
baseSb.GTE("unix_milli", start),
baseSb.LT("unix_milli", end),
)
@@ -770,7 +780,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
sb.Where(
sb.In("metric_name", query.Aggregations[0].MetricName),
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
sb.GTE("unix_milli", start),
sb.LT("unix_milli", end),
)

View File

@@ -13,9 +13,21 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMetricNameValuesResolveFamily(t *testing.T) {
assert.Equal(t,
[]any{"k8s.pod.cpu.usage", "k8s.pod.cpu.utilization"},
metricNameValues("k8s.pod.cpu.usage"),
)
assert.Equal(t,
[]any{"container_cpu_usage", "container_cpu_utilization"},
metricNameValues("container_cpu_utilization"),
)
}
func TestStatementBuilder(t *testing.T) {
cases := []struct {
name string

View File

@@ -44,6 +44,73 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
return fmt.Sprintf(`%%%s%%`, key.Name)
}
func memberKey(key *telemetrytypes.TelemetryFieldKey, name string) *telemetrytypes.TelemetryFieldKey {
member := *key
member.Name = name
return &member
}
func keyIndexCondition(sb *sqlbuilder.SelectBuilder, column string, key *telemetrytypes.TelemetryFieldKey, members []string) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
conditions = append(conditions, sb.Like(column, keyIndexFilter(memberKey(key, member))))
}
if len(conditions) == 1 {
return conditions[0]
}
return sb.Or(conditions...)
}
func valueIndexCondition(
sb *sqlbuilder.SelectBuilder,
column string,
key *telemetrytypes.TelemetryFieldKey,
members []string,
op qbtypes.FilterOperator,
value any,
caseInsensitive bool,
) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
patterns := valueForIndexFilter(op, memberKey(key, member), value)
switch values := patterns.(type) {
case []string:
for _, pattern := range values {
conditions = append(conditions, sb.Like(column, pattern))
}
default:
if caseInsensitive {
conditions = append(conditions, sb.ILike(column, values))
} else {
conditions = append(conditions, sb.Like(column, values))
}
}
}
if len(conditions) == 1 {
return conditions[0]
}
return sb.Or(conditions...)
}
func memberPresenceCondition(sb *sqlbuilder.SelectBuilder, column string, members []string, exists bool) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
field := fmt.Sprintf("simpleJSONHas(%s, '%s')", column, member)
if exists {
conditions = append(conditions, sb.E(field, true))
} else {
conditions = append(conditions, sb.NE(field, true))
}
}
if exists {
if len(conditions) == 1 {
return conditions[0]
}
return sb.Or(conditions...)
}
return sb.And(conditions...)
}
// SkipResourceFilter is not applicable here: the fingerprint table only stores resource attributes.
func (b *defaultConditionBuilder) ConditionFor(
ctx context.Context,
@@ -115,8 +182,10 @@ func (b *defaultConditionBuilder) conditionForKey(
// as we have not changed the resource column in the resource fingerprint table.
column := columns[0]
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
valueForIndexFilter := valueForIndexFilter(op, key, value)
members := resourceSemconvMembers(key)
isFamily := len(members) > 1
keyIdxFilter := keyIndexCondition(sb, column.Name, key, members)
singleValueIndexFilter := valueForIndexFilter(op, memberKey(key, members[0]), value)
fieldName, err := b.fm.FieldFor(ctx, valuer.UUID{}, startNs, endNs, key)
if err != nil {
@@ -128,12 +197,15 @@ func (b *defaultConditionBuilder) conditionForKey(
return sb.And(
sb.E(fieldName, formattedValue),
keyIdxFilter,
sb.Like(column.Name, valueForIndexFilter),
valueIndexCondition(sb, column.Name, key, members, op, value, false),
), nil
case qbtypes.FilterOperatorNotEqual:
if isFamily {
return sb.NE(fieldName, formattedValue), nil
}
return sb.And(
sb.NE(fieldName, formattedValue),
sb.NotLike(column.Name, valueForIndexFilter),
sb.NotLike(column.Name, singleValueIndexFilter),
), nil
case qbtypes.FilterOperatorGreaterThan:
return sb.And(sb.GT(fieldName, formattedValue), keyIdxFilter), nil
@@ -148,7 +220,7 @@ func (b *defaultConditionBuilder) conditionForKey(
return sb.And(
sb.ILike(fieldName, formattedValue),
keyIdxFilter,
sb.ILike(column.Name, valueForIndexFilter),
valueIndexCondition(sb, column.Name, key, members, op, value, true),
), nil
case qbtypes.FilterOperatorNotLike, qbtypes.FilterOperatorNotILike:
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
@@ -185,13 +257,11 @@ func (b *defaultConditionBuilder) conditionForKey(
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
}
mainCondition := sb.Or(inConditions...)
valConditions := make([]string, 0, len(values))
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
for _, v := range valuesForIndexFilter {
valConditions = append(valConditions, sb.Like(column.Name, v))
}
}
mainCondition = sb.And(mainCondition, keyIdxFilter, sb.Or(valConditions...))
mainCondition = sb.And(
mainCondition,
keyIdxFilter,
valueIndexCondition(sb, column.Name, key, members, op, value, false),
)
return mainCondition, nil
case qbtypes.FilterOperatorNotIn:
@@ -204,8 +274,11 @@ func (b *defaultConditionBuilder) conditionForKey(
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
}
mainCondition := sb.And(notInConditions...)
if isFamily {
return mainCondition, nil
}
valConditions := make([]string, 0, len(values))
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
if valuesForIndexFilter, ok := singleValueIndexFilter.([]string); ok {
for _, v := range valuesForIndexFilter {
valConditions = append(valConditions, sb.NotLike(column.Name, v))
}
@@ -215,13 +288,11 @@ func (b *defaultConditionBuilder) conditionForKey(
case qbtypes.FilterOperatorExists:
return sb.And(
sb.E(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
memberPresenceCondition(sb, column.Name, members, true),
keyIdxFilter,
), nil
case qbtypes.FilterOperatorNotExists:
return sb.And(
sb.NE(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
), nil
return memberPresenceCondition(sb, column.Name, members, false), nil
case qbtypes.FilterOperatorRegexp:
return sb.And(
@@ -237,7 +308,7 @@ func (b *defaultConditionBuilder) conditionForKey(
return sb.And(
sb.ILike(fieldName, fmt.Sprintf(`%%%s%%`, formattedValue)),
keyIdxFilter,
sb.ILike(column.Name, valueForIndexFilter),
valueIndexCondition(sb, column.Name, key, members, op, value, true),
), nil
case qbtypes.FilterOperatorNotContains:
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else

View File

@@ -198,6 +198,75 @@ func TestConditionBuilder(t *testing.T) {
expected: "match(simpleJSONExtractString(labels, 'k8s.namespace.name'), ?) AND labels LIKE ?",
expectedArgs: []any{"ban.*", "%k8s.namespace.name%"},
},
{
name: "semantic convention family equality uses current-first fallback",
key: &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
},
op: qbtypes.FilterOperatorEqual,
value: "production",
expected: "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), '')) = ? AND (labels LIKE ? OR labels LIKE ?) AND (labels LIKE ? OR labels LIKE ?)",
expectedArgs: []any{
"production",
"%deployment.environment.name%",
"%deployment.environment%",
`%deployment.environment.name":"production%`,
`%deployment.environment":"production%`,
},
},
{
name: "old semantic convention request uses only current metadata member",
key: &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name"},
},
op: qbtypes.FilterOperatorEqual,
value: "production",
expected: "simpleJSONExtractString(labels, 'deployment.environment.name') = ? AND labels LIKE ? AND labels LIKE ?",
expectedArgs: []any{"production", "%deployment.environment.name%", `%deployment.environment.name":"production%`},
},
{
name: "semantic convention family negative filter does not reject fallback rows",
key: &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
},
op: qbtypes.FilterOperatorNotEqual,
value: "staging",
expected: "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), '')) <> ?",
expectedArgs: []any{"staging"},
},
{
name: "semantic convention family exists checks every member",
key: &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
},
op: qbtypes.FilterOperatorExists,
expected: "(simpleJSONHas(labels, 'deployment.environment.name') = ? OR simpleJSONHas(labels, 'deployment.environment') = ?) AND (labels LIKE ? OR labels LIKE ?)",
expectedArgs: []any{true, true, "%deployment.environment.name%", "%deployment.environment%"},
},
{
name: "semantic convention family not exists checks every member",
key: &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
},
op: qbtypes.FilterOperatorNotExists,
expected: "(simpleJSONHas(labels, 'deployment.environment.name') <> ? AND simpleJSONHas(labels, 'deployment.environment') <> ?)",
expectedArgs: []any{true, true},
},
}
fm := NewFieldMapper()

View File

@@ -3,8 +3,10 @@ package resourcefilter
import (
"context"
"fmt"
"strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/semconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -32,6 +34,20 @@ func NewFieldMapper() *defaultFieldMapper {
return &defaultFieldMapper{}
}
func resourceSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
if key.FieldContext != telemetrytypes.FieldContextResource {
return []string{key.Name}
}
if len(key.SemconvMembers) > 0 {
return key.SemconvMembers
}
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
})
}
func (m *defaultFieldMapper) getColumn(
_ context.Context,
_, _ uint64,
@@ -66,7 +82,15 @@ func (m *defaultFieldMapper) FieldFor(
return "", err
}
if key.FieldContext == telemetrytypes.FieldContextResource {
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
members := resourceSemconvMembers(key)
if len(members) > 1 {
values := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(simpleJSONExtractString(%s, '%s'), '')", columns[0].Name, member))
}
return "COALESCE(" + strings.Join(values, ", ") + ")", nil
}
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, members[0]), nil
}
return columns[0].Name, nil
}

View File

@@ -1,35 +0,0 @@
package telemetrymetadata
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
type BackwardCompatibleKeyMap map[string]string
var (
TracesBackwardCompatKeys = BackwardCompatibleKeyMap{
"net.peer.name": "server.address",
"server.address": "net.peer.name",
"http.url": "url.full",
"url.full": "http.url",
}
// LogsBackwardCompatKeys contains bidirectional mappings for logs.
// Currently empty, can be extended in the future.
LogsBackwardCompatKeys = BackwardCompatibleKeyMap{}
// MetricsBackwardCompatKeys contains bidirectional mappings for metrics.
// Currently empty, can be extended in the future.
MetricsBackwardCompatKeys = BackwardCompatibleKeyMap{}
)
func GetBackwardCompatKeysForSignal(signal telemetrytypes.Signal) BackwardCompatibleKeyMap {
switch signal {
case telemetrytypes.SignalTraces:
return TracesBackwardCompatKeys
case telemetrytypes.SignalLogs:
return LogsBackwardCompatKeys
case telemetrytypes.SignalMetrics:
return MetricsBackwardCompatKeys
default:
return BackwardCompatibleKeyMap{}
}
}

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
"slices"
"strings"
"time"
@@ -14,6 +15,7 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
@@ -151,6 +153,129 @@ func (t *telemetryMetaStore) tracesTblStatementToFieldKeys(ctx context.Context)
return materialisedKeys, nil
}
func attributeSemconvMembers(name string, signal telemetrytypes.Signal, fieldContext telemetrytypes.FieldContext) []string {
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
Name: name,
Signal: signal,
FieldContext: fieldContext,
})
}
func traceSemconvMembers(name string, fieldContext telemetrytypes.FieldContext) []string {
return attributeSemconvMembers(name, telemetrytypes.SignalTraces, fieldContext)
}
func semconvDuplicateFactor(signal telemetrytypes.Signal) int {
factor := 1
for _, family := range semconv.All() {
if family.Kind != semconv.KindAttribute {
continue
}
if _, ok := semconv.Lookup(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: family.Current,
Signal: signal,
}); ok {
factor = max(factor, len(attributeSemconvMembers(family.Current, signal, telemetrytypes.FieldContextUnspecified)))
}
}
return factor
}
func traceSemconvDuplicateFactor() int {
return semconvDuplicateFactor(telemetrytypes.SignalTraces)
}
func inStrings(sb *sqlbuilder.SelectBuilder, column string, values []string) string {
args := make([]any, 0, len(values))
for _, value := range values {
args = append(args, value)
}
return sb.In(column, args...)
}
// canonicalizeTraceSemconvKeys presents one current-name key for each family.
// If metadata contains both spellings, metadata attached to the current name
// wins; otherwise the old entry is copied under the current response name.
func canonicalizeTraceSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
return canonicalizeSemconvKeys(keys, telemetrytypes.SignalTraces)
}
// canonicalizeSemconvKeys presents one current-name key for each enabled
// attribute family and records only the physical spellings actually present in
// metadata. That lets field mappers retain their optimized single-key SQL for
// homogeneous ranges while mixed ranges coalesce current-first.
func canonicalizeSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey, signal telemetrytypes.Signal) []*telemetrytypes.TelemetryFieldKey {
result := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
indexByIdentity := make(map[string]int)
currentSourceByIdentity := make(map[string]bool)
for _, key := range keys {
if key.Signal != signal {
result = append(result, key)
continue
}
selector := telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: signal,
FieldContext: key.FieldContext,
}
members := semconv.AttributeMembers(selector)
current := semconv.CurrentAttribute(selector)
if len(members) == 1 && current == key.Name {
result = append(result, key)
continue
}
resolved := *key
resolved.Name = current
resolved.SemconvMembers = []string{key.Name}
identity := resolved.Name + ";" + resolved.Signal.StringValue() + ";" + resolved.FieldContext.StringValue() + ";" + resolved.FieldDataType.StringValue()
physicalName := strings.TrimPrefix(key.Name, "resource_")
fromCurrent := physicalName == current || physicalName == strings.ReplaceAll(current, ".", "_")
if index, found := indexByIdentity[identity]; found {
physicalMembers := result[index].SemconvMembers
if fromCurrent && !currentSourceByIdentity[identity] {
result[index] = &resolved
currentSourceByIdentity[identity] = true
}
for _, member := range physicalMembers {
if !slices.Contains(result[index].SemconvMembers, member) {
result[index].SemconvMembers = append(result[index].SemconvMembers, member)
}
}
if !slices.Contains(result[index].SemconvMembers, key.Name) {
result[index].SemconvMembers = append(result[index].SemconvMembers, key.Name)
}
continue
}
indexByIdentity[identity] = len(result)
currentSourceByIdentity[identity] = fromCurrent
result = append(result, &resolved)
}
for _, key := range result {
if len(key.SemconvMembers) < 2 {
continue
}
present := make(map[string]bool, len(key.SemconvMembers))
for _, member := range key.SemconvMembers {
present[member] = true
}
ordered := make([]string, 0, len(key.SemconvMembers))
for _, member := range attributeSemconvMembers(key.Name, signal, key.FieldContext) {
if present[member] {
ordered = append(ordered, member)
}
}
key.SemconvMembers = ordered
}
return result
}
// getTracesKeys returns the keys from the spans that match the field selection criteria.
func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
@@ -202,10 +327,23 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
// key part of the selector
fieldKeyConds := []string{}
members := traceSemconvMembers(fieldKeySelector.Name, fieldKeySelector.FieldContext)
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
fieldKeyConds = append(fieldKeyConds, sb.E("tagKey", fieldKeySelector.Name))
if len(members) == 1 {
fieldKeyConds = append(fieldKeyConds, sb.E("tagKey", members[0]))
} else {
memberValues := make([]any, 0, len(members))
for _, member := range members {
memberValues = append(memberValues, member)
}
fieldKeyConds = append(fieldKeyConds, sb.In("tagKey", memberValues...))
}
} else {
fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
memberConditions := make([]string, 0, len(members))
for _, member := range members {
memberConditions = append(memberConditions, sb.ILike("tagKey", "%"+escapeForLike(member)+"%"))
}
fieldKeyConds = append(fieldKeyConds, sb.Or(memberConditions...))
}
searchTexts = append(searchTexts, fieldKeySelector.Name)
@@ -238,8 +376,10 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
mainSb.From(mainSb.BuilderAs(sb, "sub_query"))
mainSb.GroupBy("tag_key", "tag_type", "tag_data_type")
mainSb.OrderBy("priority")
// query one extra to check if we hit the limit
mainSb.Limit(limit + 1)
// Family members collapse after the database query. In the worst case each
// logical key occupies one row per family member, so fetch enough physical
// rows to return the requested logical page, plus one to detect truncation.
mainSb.Limit(limit*traceSemconvDuplicateFactor() + 1)
query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -249,14 +389,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
}
defer rows.Close()
keys := []*telemetrytypes.TelemetryFieldKey{}
rowCount := 0
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
@@ -285,8 +418,11 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTracesKeys.Error())
}
// hit the limit? (only counting DB results)
complete := rowCount <= limit
keys = canonicalizeTraceSemconvKeys(keys)
complete := len(keys) <= limit
if !complete {
keys = keys[:limit]
}
staticKeys := []string{"isRoot", "isEntryPoint"}
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
@@ -460,10 +596,19 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
continue
}
fieldKeyConds := []string{}
members := attributeSemconvMembers(sel.Name, telemetrytypes.SignalLogs, fieldContext)
if sel.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
fieldKeyConds = append(fieldKeyConds, sb.E("name", sel.Name))
memberValues := make([]any, 0, len(members))
for _, member := range members {
memberValues = append(memberValues, member)
}
fieldKeyConds = append(fieldKeyConds, sb.In("name", memberValues...))
} else {
fieldKeyConds = append(fieldKeyConds, sb.ILike("name", "%"+escapeForLike(sel.Name)+"%"))
memberConds := make([]string, 0, len(members))
for _, member := range members {
memberConds = append(memberConds, sb.ILike("name", "%"+escapeForLike(member)+"%"))
}
fieldKeyConds = append(fieldKeyConds, sb.Or(memberConds...))
}
if sel.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
fieldKeyConds = append(fieldKeyConds, sb.E("datatype", sel.FieldDataType.TagDataType()))
@@ -556,6 +701,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
limit = 1000
}
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalLogs)
mainQuery := fmt.Sprintf(`
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
FROM (
@@ -564,7 +710,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
GROUP BY tag_key, tag_type, tag_data_type
ORDER BY priority
LIMIT %d
`, strings.Join(queries, " UNION ALL "), limit+1)
`, strings.Join(queries, " UNION ALL "), dbLimit+1)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
@@ -585,7 +731,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
if rowCount > dbLimit {
break
}
@@ -633,7 +779,16 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
}
// hit the limit? (only counting DB results)
complete := rowCount <= limit
complete := rowCount <= dbLimit
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalLogs)
if len(keys) > limit {
keys = keys[:limit]
complete = false
}
mapOfKeys = make(map[string]*telemetrytypes.TelemetryFieldKey, len(keys))
for _, key := range keys {
mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
}
staticKeys := []string{}
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
@@ -941,10 +1096,19 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
conds := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
fieldConds := []string{}
members := attributeSemconvMembers(fieldKeySelector.Name, telemetrytypes.SignalMetrics, fieldKeySelector.FieldContext)
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
memberValues := make([]any, 0, len(members))
for _, member := range members {
memberValues = append(memberValues, member)
}
fieldConds = append(fieldConds, sb.In("attr_name", memberValues...))
} else {
fieldConds = append(fieldConds, sb.ILike("attr_name", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
memberConds := make([]string, 0, len(members))
for _, member := range members {
memberConds = append(memberConds, sb.ILike("attr_name", "%"+escapeForLike(member)+"%"))
}
fieldConds = append(fieldConds, sb.Or(memberConds...))
}
fieldConds = append(fieldConds, sb.NotLike("attr_name", "\\_\\_%"))
@@ -960,7 +1124,12 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
if fieldKeySelector.MetricContext != nil {
if fieldKeySelector.MetricContext.MetricName != "" {
fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
metricNames := semconv.MetricNames(fieldKeySelector.MetricContext.MetricName)
metricValues := make([]any, 0, len(metricNames))
for _, metricName := range metricNames {
metricValues = append(metricValues, metricName)
}
fieldConds = append(fieldConds, sb.In("metric_name", metricValues...))
}
if fieldKeySelector.MetricContext.MetricNamespace != "" {
fieldConds = append(fieldConds, sb.Like("metric_name", escapeForLike(fieldKeySelector.MetricContext.MetricNamespace)+"%"))
@@ -982,7 +1151,8 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
mainSb.GroupBy("name", "field_context", "field_data_type")
mainSb.OrderBy("priority")
// query one extra to check if we hit the limit
mainSb.Limit(limit + 1)
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
mainSb.Limit(dbLimit + 1)
query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -997,7 +1167,7 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
if rowCount > dbLimit {
break
}
@@ -1022,7 +1192,12 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
}
// hit the limit?
complete := rowCount <= limit
complete := rowCount <= dbLimit
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalMetrics)
if len(keys) > limit {
keys = keys[:limit]
complete = false
}
return keys, complete, nil
}
@@ -1044,16 +1219,30 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
var limit int
for _, fieldKeySelector := range fieldKeySelectors {
fieldConds := []string{}
members := attributeSemconvMembers(fieldKeySelector.Name, telemetrytypes.SignalMetrics, fieldKeySelector.FieldContext)
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
memberValues := make([]any, 0, len(members))
for _, member := range members {
memberValues = append(memberValues, member)
}
fieldConds = append(fieldConds, sb.In("attr_name", memberValues...))
} else {
fieldConds = append(fieldConds, sb.Like("attr_name", "%"+fieldKeySelector.Name+"%"))
memberConds := make([]string, 0, len(members))
for _, member := range members {
memberConds = append(memberConds, sb.Like("attr_name", "%"+member+"%"))
}
fieldConds = append(fieldConds, sb.Or(memberConds...))
}
fieldConds = append(fieldConds, sb.NotLike("attr_name", "\\_\\_%"))
if fieldKeySelector.MetricContext != nil {
if fieldKeySelector.MetricContext.MetricName != "" {
fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
metricNames := semconv.MetricNames(fieldKeySelector.MetricContext.MetricName)
metricValues := make([]any, 0, len(metricNames))
for _, metricName := range metricNames {
metricValues = append(metricValues, metricName)
}
fieldConds = append(fieldConds, sb.In("metric_name", metricValues...))
}
if fieldKeySelector.MetricContext.MetricNamespace != "" {
fieldConds = append(fieldConds, sb.Like("metric_name", escapeForLike(fieldKeySelector.MetricContext.MetricNamespace)+"%"))
@@ -1068,7 +1257,8 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
limit = 1000
}
sb.Limit(limit)
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
sb.Limit(dbLimit + 1)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
@@ -1082,7 +1272,7 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
if rowCount > dbLimit {
break
}
@@ -1091,9 +1281,14 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
}
fieldContext := telemetrytypes.FieldContextAttribute
if strings.HasPrefix(name, "resource_") {
fieldContext = telemetrytypes.FieldContextResource
}
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalMetrics,
Name: name,
Signal: telemetrytypes.SignalMetrics,
FieldContext: fieldContext,
})
}
@@ -1102,46 +1297,17 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
}
// hit the limit?
complete := rowCount <= limit
complete := rowCount <= dbLimit
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalMetrics)
if len(keys) > limit {
keys = keys[:limit]
complete = false
}
return keys, complete, nil
}
// applyBackwardCompatibleKeys adds backward compatible key aliases to the map.
func applyBackwardCompatibleKeys(mapOfKeys map[string][]*telemetrytypes.TelemetryFieldKey) {
// Get backward compatible keys for all signals
backwardCompatKeysBySignal := map[telemetrytypes.Signal]BackwardCompatibleKeyMap{
telemetrytypes.SignalTraces: GetBackwardCompatKeysForSignal(telemetrytypes.SignalTraces),
telemetrytypes.SignalLogs: GetBackwardCompatKeysForSignal(telemetrytypes.SignalLogs),
telemetrytypes.SignalMetrics: GetBackwardCompatKeysForSignal(telemetrytypes.SignalMetrics),
}
// Iterate over existing keys and add aliases if they exist in backward compat mapping
for srcKey, srcKeys := range mapOfKeys {
for _, srcKeyEntry := range srcKeys {
backwardCompatKeys := backwardCompatKeysBySignal[srcKeyEntry.Signal]
if backwardCompatKeys == nil {
continue
}
if aliasKey, ok := backwardCompatKeys[srcKey]; ok {
if _, aliasExists := mapOfKeys[aliasKey]; !aliasExists {
aliasKeyEntry := &telemetrytypes.TelemetryFieldKey{
Name: aliasKey,
Signal: srcKeyEntry.Signal,
FieldContext: srcKeyEntry.FieldContext,
FieldDataType: srcKeyEntry.FieldDataType,
}
mapOfKeys[aliasKey] = []*telemetrytypes.TelemetryFieldKey{aliasKeyEntry}
}
// Found the alias for this signal, no need to check other entries
break
}
}
}
}
func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
if len(selectors) == 0 {
return keys
@@ -1272,7 +1438,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
applyBackwardCompatibleKeys(mapOfKeys)
mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
@@ -1353,7 +1518,6 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
applyBackwardCompatibleKeys(mapOfKeys)
mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
@@ -1367,7 +1531,18 @@ func (t *telemetryMetaStore) GetKey(ctx context.Context, orgID valuer.UUID, fiel
if err != nil {
return nil, err
}
return keys[fieldKeySelector.Name], nil
members := semconv.Members(semconv.KindAttribute, *fieldKeySelector)
resolved := make([]*telemetrytypes.TelemetryFieldKey, 0)
seen := make(map[*telemetrytypes.TelemetryFieldKey]bool)
for _, member := range members {
for _, key := range keys[member] {
if !seen[key] {
resolved = append(resolved, key)
seen[key] = true
}
}
}
return resolved, nil
}
func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *telemetrytypes.FieldValueSelector) ([]string, bool, error) {
@@ -1486,7 +1661,6 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, orgID valuer.
if limit == 0 {
limit = 50
}
// query one extra to check if we hit the limit
sb.Limit(limit + 1)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
@@ -1542,7 +1716,16 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
members := traceSemconvMembers(fieldValueSelector.Name, fieldValueSelector.FieldContext)
if len(members) == 1 {
sb.Where(sb.E("tag_key", members[0]))
} else {
memberValues := make([]any, 0, len(members))
for _, member := range members {
memberValues = append(memberValues, member)
}
sb.Where(sb.In("tag_key", memberValues...))
}
}
// now look at the field context
@@ -1632,7 +1815,12 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
members := attributeSemconvMembers(fieldValueSelector.Name, telemetrytypes.SignalLogs, fieldValueSelector.FieldContext)
memberValues := make([]any, 0, len(members))
for _, member := range members {
memberValues = append(memberValues, member)
}
sb.Where(sb.In("tag_key", memberValues...))
}
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
@@ -1830,7 +2018,9 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID val
From(t.metricsDBName + "." + t.metricsFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("attr_name", fieldValueSelector.Name))
sb.Where(inStrings(sb, "attr_name", attributeSemconvMembers(
fieldValueSelector.Name, telemetrytypes.SignalMetrics, fieldValueSelector.FieldContext,
)))
}
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
@@ -1842,7 +2032,7 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID val
}
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
}
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
@@ -1976,7 +2166,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValuesForTable(ctx context.C
From(t.metricsDBName + "." + tableName)
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
}
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
@@ -2039,12 +2229,14 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
From(t.meterDBName + "." + t.meterFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("attr.1", fieldValueSelector.Name))
sb.Where(inStrings(sb, "attr.1", attributeSemconvMembers(
fieldValueSelector.Name, telemetrytypes.SignalMetrics, fieldValueSelector.FieldContext,
)))
}
sb.Where(sb.NotLike("attr.1", "\\_\\_%"))
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
}
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
@@ -2063,8 +2255,10 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
if limit == 0 {
limit = 50
}
// query one extra to check if we hit the limit
sb.Limit(limit + 1)
// A value can be present under several physical spellings; over-fetch and
// de-duplicate after scanning so one family cannot consume the result limit.
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
sb.Limit(dbLimit + 1)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
@@ -2074,11 +2268,13 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
defer rows.Close()
values := &telemetrytypes.TelemetryFieldValues{}
seen := make(map[string]bool)
rowCount := 0
uniqueCount := 0
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
if rowCount > dbLimit {
break
}
@@ -2087,12 +2283,16 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterValues.Error())
}
if len(attribute) > 1 {
values.StringValues = append(values.StringValues, attribute[1])
if !seen[attribute[1]] && uniqueCount < limit {
values.StringValues = append(values.StringValues, attribute[1])
seen[attribute[1]] = true
uniqueCount++
}
}
}
// hit the limit?
complete := rowCount <= limit
complete := rowCount <= dbLimit && uniqueCount < limit
return values, complete, nil
}

View File

@@ -12,6 +12,7 @@ import (
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -83,3 +84,121 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
t.Errorf("there were unfulfilled expectations: %s", err)
}
}
func TestCanonicalizeTraceSemconvKeys(t *testing.T) {
oldResource := &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
Description: "old resource metadata",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
currentResource := &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment.name",
Description: "current resource metadata",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
oldAttribute := &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
oldLogResource := &telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
result := canonicalizeTraceSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
oldResource,
currentResource,
oldAttribute,
oldLogResource,
})
require.Len(t, result, 3)
assert.Equal(t, "deployment.environment.name", result[0].Name)
assert.Equal(t, "current resource metadata", result[0].Description)
assert.Equal(t, []string{"deployment.environment.name", "deployment.environment"}, result[0].SemconvMembers)
assert.Equal(t, "deployment.environment.name", result[1].Name)
assert.Equal(t, telemetrytypes.FieldContextAttribute, result[1].FieldContext)
assert.Equal(t, []string{"deployment.environment"}, result[1].SemconvMembers)
assert.Equal(t, "deployment.environment", result[2].Name, "phase 1 must not rewrite raw log metadata")
}
func TestCanonicalizeLogAndMetricSemconvKeys(t *testing.T) {
logKeys := canonicalizeSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
{
Name: "db.system",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
{
Name: "db.system.name",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}, telemetrytypes.SignalLogs)
require.Len(t, logKeys, 1)
assert.Equal(t, "db.system.name", logKeys[0].Name)
assert.Equal(t, []string{"db.system.name", "db.system"}, logKeys[0].SemconvMembers)
metricKeys := canonicalizeSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
{
Name: "resource_db_system",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
{
Name: "db.system.name",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}, telemetrytypes.SignalMetrics)
require.Len(t, metricKeys, 1)
assert.Equal(t, "db.system.name", metricKeys[0].Name)
assert.Equal(t, []string{"db.system.name", "resource_db_system"}, metricKeys[0].SemconvMembers)
}
func TestGetSpanFieldValuesMergesSemconvFamily(t *testing.T) {
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
mock := mockTelemetryStore.Mock()
metadata := NewTelemetryMetaStore(
instrumentationtest.New().ToProviderSettings(),
mockTelemetryStore,
flaggertest.New(t),
)
mock.ExpectQuery(`SELECT DISTINCT string_value, number_value FROM signoz_traces\.distributed_tag_attributes_v2 WHERE tag_key IN \(\?, \?\) AND tag_type = \? AND tag_data_type = \? LIMIT \?`).
WithArgs("deployment.environment.name", "deployment.environment", "resource", "string", 51).
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
{Name: "string_value", Type: "String"},
{Name: "number_value", Type: "Float64"},
}, [][]any{
{"production", float64(0)},
{"staging", float64(0)},
{"production", float64(0)},
}))
values, complete, err := metadata.GetAllValues(context.Background(), valuer.UUID{}, &telemetrytypes.FieldValueSelector{
FieldKeySelector: &telemetrytypes.FieldKeySelector{
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Name: "deployment.environment",
},
})
require.NoError(t, err)
assert.True(t, complete)
assert.Equal(t, []string{"production", "staging"}, values.StringValues)
assert.NoError(t, mock.ExpectationsWereMet(), "all expected metadata queries should be executed")
}

View File

@@ -89,10 +89,12 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
// Create and store the TelemetryFieldKey
field := &telemetrytypes.TelemetryFieldKey{
Name: fieldName,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
Materialized: true,
Name: fieldName,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
Materialized: true,
MaterializedColumnName: columnName,
MaterializedSemconv: strings.Count(defaultExprStr, "['") > 1,
}
v.Fields = append(v.Fields, field)

View File

@@ -5,8 +5,25 @@ import (
"testing"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestExtractFieldKeyPreservesHistoricalMaterializedColumnName(t *testing.T) {
statement := `CREATE TABLE signoz_traces.signoz_index_v3
(
attributes_string Map(LowCardinality(String), String),
` + "`attribute_string_db$$system`" + ` LowCardinality(String)
DEFAULT if(mapContains(attributes_string, 'db.system.name'), attributes_string['db.system.name'], attributes_string['db.system'])
) ENGINE = MergeTree ORDER BY tuple()`
keys, err := ExtractFieldKeysFromTblStatement(statement)
require.NoError(t, err, "table statement should parse")
require.Len(t, keys, 1, "table statement should contain one materialized key")
assert.Equal(t, "db.system.name", keys[0].Name)
assert.Equal(t, "attribute_string_db$$system", keys[0].MaterializedColumnName)
assert.True(t, keys[0].MaterializedSemconv)
}
func TestExtractFieldKeysFromTblStatement(t *testing.T) {
var statement = `CREATE TABLE signoz_logs.logs_v2

View File

@@ -453,6 +453,9 @@ func (c *conditionBuilder) ConditionFor(
sb *sqlbuilder.SelectBuilder,
) ([]string, []string, error) {
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
if options.ExactSemconv {
matches = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
}
skipResourceFilter := options.SkipResourceFilter
// search() resolves its own (optional) scope; handle it before key resolution.
@@ -499,6 +502,9 @@ func (c *conditionBuilder) ConditionFor(
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
}
}
if options.ExactSemconv {
keys = querybuilder.ExactSemconvKeys(keys)
}
if skipResourceFilter && !synthesized {
filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))

View File

@@ -10,6 +10,7 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -67,6 +68,20 @@ type fieldMapper struct {
fl flagger.Flagger
}
func logSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
return []string{key.Name}
}
if len(key.SemconvMembers) > 0 {
return key.SemconvMembers
}
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: telemetrytypes.SignalLogs,
FieldContext: key.FieldContext,
})
}
func NewFieldMapper(fl flagger.Flagger) qbtypes.FieldMapper {
return &fieldMapper{fl: fl}
}
@@ -141,8 +156,20 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
case schema.ColumnTypeEnumJSON:
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
members := logSemconvMembers(key)
if len(members) > 1 {
values := make([]string, 0, len(members))
guards := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(%s.`%s`::String, '')", columnName, member))
guards = append(guards, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, member))
}
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
existExpr = append(existExpr, "("+strings.Join(guards, " OR ")+")")
} else {
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, members[0]))
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, members[0]))
}
case telemetrytypes.FieldContextBody:
if key.Name == messageSubField {
exprs = append(exprs, messageSubColumn)
@@ -181,13 +208,32 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
// a key could have been materialized, if so return the materialized column name
if key.Materialized {
members := logSemconvMembers(key)
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
existExpr = append(existExpr, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
} else if len(members) > 1 {
guards := make([]string, 0, len(members))
for _, member := range members {
guards = append(guards, fmt.Sprintf("mapContains(%s, '%s')", columnName, member))
}
if valueType.GetType() == schema.ColumnTypeEnumString {
values := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(%s['%s'], '')", columnName, member))
}
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
} else {
branches := make([]string, 0, len(members)*2+1)
for i, member := range members {
branches = append(branches, guards[i], fmt.Sprintf("%s['%s']", columnName, member))
}
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
}
existExpr = append(existExpr, "("+strings.Join(guards, " OR ")+")")
} else {
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, members[0]))
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, members[0]))
}
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)

View File

@@ -2,6 +2,7 @@ package logstelemetryschema
import (
"context"
"strings"
"testing"
"time"
@@ -14,6 +15,32 @@ import (
"github.com/stretchr/testify/require"
)
func TestFieldForSemconvFamily(t *testing.T) {
ctx := context.Background()
fm := NewFieldMapper(flaggertest.New(t)).(*fieldMapper)
key := &telemetrytypes.TelemetryFieldKey{
Name: "db.system.name",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
expression, err := fm.FieldFor(ctx, valuer.UUID{}, 0, 0, key)
require.NoError(t, err)
assert.Equal(t, "COALESCE(NULLIF(attributes_string['db.system.name'], ''), NULLIF(attributes_string['db.system'], ''))", expression)
assert.Less(t, strings.Index(expression, "db.system.name"), strings.Index(expression, "db.system']"), "current spelling must win")
exists, err := fm.existsExpressionFor(ctx, valuer.UUID{}, 0, 0, key, true)
require.NoError(t, err)
assert.Equal(t, "(mapContains(attributes_string, 'db.system.name') OR mapContains(attributes_string, 'db.system'))", exists)
exact := *key
exact.SemconvMembers = []string{"db.system"}
expression, err = fm.FieldFor(ctx, valuer.UUID{}, 0, 0, &exact)
require.NoError(t, err)
assert.Equal(t, "attributes_string['db.system']", expression)
}
func TestGetColumn(t *testing.T) {
ctx := context.Background()

View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/querybuilder"
@@ -135,10 +136,22 @@ func (c *conditionBuilder) conditionFor(
return "true", nil
}
if operator == qbtypes.FilterOperatorExists {
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
members := metricAttributeMembers(key)
guards := make([]string, 0, len(members))
for _, member := range members {
guards = append(guards, fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", member))
}
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
guard := strings.Join(guards, " OR ")
if len(guards) > 1 {
guard = "(" + guard + ")"
}
if operator == qbtypes.FilterOperatorExists {
return guard, nil
}
if len(guards) == 1 {
return "not " + guard, nil
}
return "NOT " + guard, nil
}
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
}
@@ -151,7 +164,7 @@ func (c *conditionBuilder) ConditionFor(
endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
_ qbtypes.ConditionBuilderOptions,
options qbtypes.ConditionBuilderOptions,
operator qbtypes.FilterOperator,
value any,
sb *sqlbuilder.SelectBuilder,
@@ -162,7 +175,15 @@ func (c *conditionBuilder) ConditionFor(
return nil, nil, err
}
requestedKey := *key
if requestedKey.Signal == telemetrytypes.SignalUnspecified {
requestedKey.Signal = telemetrytypes.SignalMetrics
}
key = &requestedKey
keys := querybuilder.MatchingFieldKeys(key, fieldKeys)
if options.ExactSemconv {
keys = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
}
var warnings []string
if len(keys) == 0 {
if _, isColumn := timeSeriesV4Columns[key.Name]; isColumn {
@@ -180,6 +201,9 @@ func (c *conditionBuilder) ConditionFor(
}
}
}
if options.ExactSemconv {
keys = querybuilder.ExactSemconvKeys(keys)
}
conds := make([]string, 0, len(keys))
for _, k := range keys {

View File

@@ -390,3 +390,45 @@ func TestConditionForKeyNotInMetadata(t *testing.T) {
})
}
}
func TestConditionForSemconvMetricLabels(t *testing.T) {
ctx := context.Background()
fm := NewFieldMapper()
conditionBuilder := NewConditionBuilder(fm)
requested := telemetrytypes.TelemetryFieldKey{
Name: "db.system.name",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
current := requested
legacyNormalized := requested
legacyNormalized.Name = "resource_db_system"
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
current.Name: {&current},
legacyNormalized.Name: {&legacyNormalized},
}
sb := sqlbuilder.NewSelectBuilder()
conditions, warnings, err := conditionBuilder.ConditionFor(
ctx, valuer.UUID{}, 0, 0, &requested, fieldKeys, qbtypes.ConditionBuilderOptions{},
qbtypes.FilterOperatorEqual, "postgresql", sb,
)
require.NoError(t, err)
assert.Empty(t, warnings)
sb.Where(conditions...)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, query, "COALESCE(NULLIF(JSONExtractString(labels, 'db.system.name'), ''), NULLIF(JSONExtractString(labels, 'resource_db_system'), '')) = ?")
assert.Equal(t, []any{"postgresql"}, args)
sb = sqlbuilder.NewSelectBuilder()
conditions, _, err = conditionBuilder.ConditionFor(
ctx, valuer.UUID{}, 0, 0, &requested, fieldKeys, qbtypes.ConditionBuilderOptions{ExactSemconv: true},
qbtypes.FilterOperatorExists, nil, sb,
)
require.NoError(t, err)
sb.Where(conditions...)
query, _ = sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, query, "has(JSONExtractKeys(labels), 'db.system.name')")
assert.NotContains(t, query, "resource_db_system")
}

View File

@@ -4,8 +4,10 @@ import (
"context"
"fmt"
"slices"
"strings"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/semconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -38,6 +40,23 @@ var (
type fieldMapper struct{}
func metricAttributeMembers(key *telemetrytypes.TelemetryFieldKey) []string {
if key.FieldContext != telemetrytypes.FieldContextResource &&
key.FieldContext != telemetrytypes.FieldContextScope &&
key.FieldContext != telemetrytypes.FieldContextAttribute &&
key.FieldContext != telemetrytypes.FieldContextUnspecified {
return []string{key.Name}
}
if len(key.SemconvMembers) > 0 {
return key.SemconvMembers
}
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: telemetrytypes.SignalMetrics,
FieldContext: key.FieldContext,
})
}
// CandidateKeys returns nil: metrics has no attribute-map fallback, so a context-missing
// key stays unresolved and the caller errors.
func (m *fieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, _ *telemetrytypes.TelemetryFieldKey, _ any, _ map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
@@ -80,19 +99,30 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endN
switch key.FieldContext {
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
return metricLabelExpression(columns[0].Name, metricAttributeMembers(key)), nil
case telemetrytypes.FieldContextMetric:
return columns[0].Name, nil
case telemetrytypes.FieldContextUnspecified:
if slices.Contains(IntrinsicFields, key.Name) {
return columns[0].Name, nil
}
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
return metricLabelExpression(columns[0].Name, metricAttributeMembers(key)), nil
}
return columns[0].Name, nil
}
func metricLabelExpression(columnName string, members []string) string {
if len(members) == 1 {
return fmt.Sprintf("JSONExtractString(%s, '%s')", columnName, members[0])
}
values := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(JSONExtractString(%s, '%s'), '')", columnName, member))
}
return "COALESCE(" + strings.Join(values, ", ") + ")"
}
func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
return m.getColumn(ctx, tsStart, tsEnd, key)
}

View File

@@ -2,6 +2,7 @@ package metricstelemetryschema
import (
"context"
"strings"
"testing"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
@@ -12,6 +13,32 @@ import (
"github.com/stretchr/testify/require"
)
func TestMetricLabelSemconvSpellings(t *testing.T) {
fm := NewFieldMapper()
key := &telemetrytypes.TelemetryFieldKey{
Name: "db.system.name",
Signal: telemetrytypes.SignalMetrics,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
expression, err := fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
require.NoError(t, err)
for _, member := range []string{
"resource_db.system.name", "resource_db_system_name", "db.system.name", "db_system_name",
"resource_db.system", "resource_db_system", "db.system", "db_system",
} {
assert.Contains(t, expression, "'"+member+"'")
}
assert.Less(t, strings.Index(expression, "resource_db.system.name"), strings.Index(expression, "resource_db.system'"))
exact := *key
exact.SemconvMembers = []string{"resource_db_system"}
expression, err = fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, &exact)
require.NoError(t, err)
assert.Equal(t, "JSONExtractString(labels, 'resource_db_system')", expression)
}
func TestGetColumn(t *testing.T) {
ctx := context.Background()

View File

@@ -154,6 +154,15 @@ func (c *conditionBuilder) conditionFor(
// in the query builder, `exists` and `not exists` are used for
// key membership checks, so depending on the column type, the condition changes
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
// A semantic-convention family is represented by one current-first value
// expression, but presence still has to inspect every physical member. In
// particular, using ExistsExpression below with the requested key would add
// a mapContains check for only that spelling and reject fallback-only rows.
if isTraceSemconvFamily(key) {
if fm, ok := c.fm.(*fieldMapper); ok {
return fm.existsExpressionFor(ctx, orgID, startNs, endNs, key, operator == qbtypes.FilterOperatorExists)
}
}
columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key)
if err != nil {
return "", err
@@ -211,6 +220,9 @@ func (c *conditionBuilder) ConditionFor(
}
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
if options.ExactSemconv {
matches = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
}
skipResourceFilter := options.SkipResourceFilter
keys, warning := querybuilder.ResolveKeys(key, matches)
@@ -256,6 +268,9 @@ func (c *conditionBuilder) ConditionFor(
synthesized = true
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
}
if options.ExactSemconv {
keys = querybuilder.ExactSemconvKeys(keys)
}
// When a resource sub-query already covers the term, drop resource keys from the main
// query. Synthesized keys are exempt: the sub-query skips keys absent from metadata.

View File

@@ -210,6 +210,32 @@ func TestConditionFor(t *testing.T) {
expectedSQL: "NOT mapContains(attributes_string, 'user.id')",
expectedError: nil,
},
{
name: "Equal operator - semantic convention family",
key: telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment.name",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
},
operator: qbtypes.FilterOperatorEqual,
value: "production",
expectedSQL: "(COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], '')) = ? AND ((mapContains(attributes_string, 'deployment.environment.name') OR mapContains(attributes_string, 'deployment.environment'))))",
expectedArgs: []any{"production"},
expectedError: nil,
},
{
name: "Not Exists operator - semantic convention family",
key: telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
},
operator: qbtypes.FilterOperatorNotExists,
expectedSQL: "NOT (((mapContains(attributes_string, 'deployment.environment.name') OR mapContains(attributes_string, 'deployment.environment'))))",
expectedError: nil,
},
{
name: "Exists operator - json field",
key: telemetrytypes.TelemetryFieldKey{

View File

@@ -8,6 +8,7 @@ import (
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/semconv"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -167,6 +168,32 @@ func NewFieldMapper() *fieldMapper {
return &fieldMapper{}
}
func traceSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
return []string{key.Name}
}
if len(key.SemconvMembers) > 0 {
return key.SemconvMembers
}
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: key.FieldContext,
})
}
func isTraceSemconvFamily(key *telemetrytypes.TelemetryFieldKey) bool {
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
return false
}
_, ok := semconv.Lookup(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: key.Name,
Signal: telemetrytypes.SignalTraces,
FieldContext: key.FieldContext,
})
return ok
}
func (m *fieldMapper) getColumn(
_ context.Context,
_, _ uint64,
@@ -291,10 +318,25 @@ func (m *fieldMapper) resolveColumnExprs(
if key.FieldContext != telemetrytypes.FieldContextResource {
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
}
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
members := traceSemconvMembers(key)
if len(members) > 1 {
values := make([]string, 0, len(members))
guards := make([]string, 0, len(members))
for _, member := range members {
// The String cast is required because ClickHouse does not allow
// Variant/Dynamic values in GROUP BY.
value := fmt.Sprintf("%s.`%s`::String", columnName, member)
values = append(values, fmt.Sprintf("NULLIF(%s, '')", value))
guards = append(guards, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, member))
}
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
} else {
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once ClickHouse is updated, check whether this cast can be removed.
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, members[0]))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, members[0]))
}
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,
@@ -319,13 +361,32 @@ func (m *fieldMapper) resolveColumnExprs(
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
// a key could have been materialized, if so return the materialized column name
if key.Materialized {
members := traceSemconvMembers(key)
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
} else if len(members) > 1 {
guards := make([]string, 0, len(members))
for _, member := range members {
guards = append(guards, fmt.Sprintf("mapContains(%s, '%s')", columnName, member))
}
if valueType.GetType() == schema.ColumnTypeEnumString {
values := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(%s['%s'], '')", columnName, member))
}
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
} else {
branches := make([]string, 0, len(members)*2)
for i, member := range members {
branches = append(branches, guards[i], fmt.Sprintf("%s['%s']", columnName, member))
}
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
}
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
} else {
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, members[0]))
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, members[0]))
}
default:
return nil, nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
@@ -529,6 +590,25 @@ func (m *fieldMapper) existsExpressionFor(
key *telemetrytypes.TelemetryFieldKey,
exists bool,
) (string, error) {
if isTraceSemconvFamily(key) {
_, existExprs, _, err := m.resolveColumnExprs(ctx, tsStart, tsEnd, key)
if err != nil {
return "", err
}
if len(existExprs) == 0 {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no existence expression found for field %s", key.Name)
}
parts := make([]string, 0, len(existExprs))
for _, expression := range existExprs {
parts = append(parts, "("+expression+")")
}
combined := strings.Join(parts, " OR ")
if exists {
return combined, nil
}
return "NOT (" + combined + ")", nil
}
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
if err != nil {
return "", err

View File

@@ -5,6 +5,8 @@ import (
"testing"
"time"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -80,7 +82,7 @@ func TestGetFieldKeyName(t *testing.T) {
Materialized: true,
Evolutions: mockEvolution,
},
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
expectedResult: "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(resources_string['deployment.environment'], '')), NULL)",
expectedError: nil,
},
{
@@ -120,6 +122,51 @@ func TestGetFieldKeyName(t *testing.T) {
}
}
func TestFieldForSemconvFamily(t *testing.T) {
ctx := context.Background()
fm := NewFieldMapper()
start := uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano())
end := uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano())
for _, requestedName := range []string{"deployment.environment.name", "deployment.environment"} {
attributeKey := telemetrytypes.TelemetryFieldKey{
Name: requestedName,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
attributeExpression, err := fm.FieldFor(ctx, valuer.UUID{}, start, end, &attributeKey)
require.NoError(t, err)
assert.Equal(t,
"COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], ''))",
attributeExpression,
)
resourceKey := telemetrytypes.TelemetryFieldKey{
Name: requestedName,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
Materialized: true,
Evolutions: MockEvolutionData(time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC)),
}
resourceExpression, err := fm.FieldFor(ctx, valuer.UUID{}, start, end, &resourceKey)
require.NoError(t, err)
assert.Equal(t,
"multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(resources_string['deployment.environment'], '')), NULL)",
resourceExpression,
)
}
oldRequestWithCurrentOnlyMetadata := telemetrytypes.TelemetryFieldKey{
Name: "deployment.environment",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
SemconvMembers: []string{"deployment.environment.name"},
}
expression, err := fm.FieldFor(ctx, valuer.UUID{}, start, end, &oldRequestWithCurrentOnlyMetadata)
require.NoError(t, err)
assert.Equal(t, "attributes_string['deployment.environment.name']", expression)
}
func TestFieldForResourceWithEvolution(t *testing.T) {
ctx := context.Background()
releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
@@ -176,7 +223,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
},
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "resource.`deployment.environment`::String",
expectedResult: "COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, ''))",
},
{
name: "Window straddles release - materialized resource",
@@ -189,7 +236,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
},
tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
expectedResult: "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(resources_string['deployment.environment'], '')), NULL)",
},
}
@@ -303,3 +350,27 @@ func TestColumnExpressionForTimestampAttributeCollision(t *testing.T) {
assert.Contains(t, result, "attributes_number['timestamp']")
})
}
func TestDBSystemFamilyUsesSemconvAwareMaterializedColumn(t *testing.T) {
fm := NewFieldMapper()
key := &telemetrytypes.TelemetryFieldKey{
Name: "db.system.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
Materialized: true,
MaterializedColumnName: "attribute_string_db$$system",
MaterializedSemconv: true,
SemconvMembers: []string{"db.system.name", "db.system"},
}
expression, err := fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
require.NoError(t, err)
assert.Equal(t, "`attribute_string_db$$system`", expression)
exists, err := querybuilder.ExistsExpression(
[]*schema.Column{indexV3Columns["attributes_string"]}, key, 0, 0, expression, true,
)
require.NoError(t, err)
assert.Equal(t, "`attribute_string_db$$system_exists`", exists)
}

View File

@@ -45,6 +45,10 @@ type ConditionBuilder interface {
type ConditionBuilderOptions struct {
// SkipResourceFilter drops the resource context from the candidate set.
SkipResourceFilter bool
// ExactSemconv disables semantic-convention family expansion. It is an
// internal escape hatch for diagnostics and migrations that must address one
// physical spelling only; public query APIs continue to resolve families.
ExactSemconv bool
}
type AggExprRewriter interface {
// Rewrite rewrites the aggregation expression to be used in the query.

View File

@@ -141,7 +141,7 @@ func NewSignalFilterFromStorableQuickFilter(storableQuickFilter *StorableQuickFi
func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
tracesFilters := []map[string]interface{}{
{"key": "duration_nano", "dataType": "float64", "type": "tag"},
{"key": "deployment.environment", "dataType": "string", "type": "resource"},
{"key": "deployment.environment.name", "dataType": "string", "type": "resource"},
{"key": "hasError", "dataType": "bool", "type": "tag"},
{"key": "service.name", "dataType": "string", "type": "resource"},
{"key": "name", "dataType": "string", "type": "tag"},
@@ -166,13 +166,13 @@ func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
}
apiMonitoringFilters := []map[string]interface{}{
{"key": "deployment.environment", "dataType": "string", "type": "resource"},
{"key": "deployment.environment.name", "dataType": "string", "type": "resource"},
{"key": "service.name", "dataType": "string", "type": "resource"},
{"key": "rpc.method", "dataType": "string", "type": "tag"},
}
exceptionsFilters := []map[string]interface{}{
{"key": "deployment.environment", "dataType": "string", "type": "resource"},
{"key": "deployment.environment.name", "dataType": "string", "type": "resource"},
{"key": "service.name", "dataType": "string", "type": "resource"},
{"key": "host.name", "dataType": "string", "type": "resource"},
{"key": "k8s.cluster.name", "dataType": "string", "type": "resource"},

View File

@@ -0,0 +1,37 @@
package quickfiltertypes
import (
"encoding/json"
"testing"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDefaultTraceQuickFiltersUseCurrentEnvironmentName(t *testing.T) {
filters, err := NewDefaultQuickFilter(valuer.GenerateUUID())
require.NoError(t, err)
traceSignals := map[string]bool{
SignalTraces.StringValue(): true,
SignalApiMonitoring.StringValue(): true,
SignalExceptions.StringValue(): true,
}
for _, filter := range filters {
if !traceSignals[filter.Signal.StringValue()] {
continue
}
var keys []v3.AttributeKey
require.NoError(t, json.Unmarshal([]byte(filter.Filter), &keys))
found := false
for _, key := range keys {
if key.Key == "deployment.environment.name" {
found = true
}
assert.NotEqual(t, "deployment.environment", key.Key)
}
assert.True(t, found, "missing environment quick filter for %s", filter.Signal.StringValue())
}
}

View File

@@ -46,8 +46,16 @@ type TelemetryFieldKey struct {
JSONPlan JSONAccessPlan `json:"-"`
Indexes []TelemetryFieldKeySkipIndex `json:"-"`
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
// MaterializedColumnName preserves the physical column when its DEFAULT
// expression resolves a newer semantic-convention key than the historical
// column identifier (for example db.system.name in db$$system).
MaterializedColumnName string `json:"-"`
// MaterializedSemconv is true when the column's DEFAULT expression already
// coalesces every enabled family member and is therefore safe for a family query.
MaterializedSemconv bool `json:"-"`
Evolutions []*EvolutionEntry `json:"-"`
Evolutions []*EvolutionEntry `json:"-"`
SemconvMembers []string `json:"-"`
}
func (f *TelemetryFieldKey) KeyNameContainsArray() bool {
@@ -126,8 +134,11 @@ func (f *TelemetryFieldKey) OverrideMetadataFrom(src *TelemetryFieldKey) {
f.FieldDataType = src.FieldDataType
f.Indexes = src.Indexes
f.Materialized = src.Materialized
f.MaterializedColumnName = src.MaterializedColumnName
f.MaterializedSemconv = src.MaterializedSemconv
f.JSONPlan = src.JSONPlan
f.Evolutions = src.Evolutions
f.SemconvMembers = src.SemconvMembers
}
func (f *TelemetryFieldKey) Equal(key *TelemetryFieldKey) bool {
@@ -202,6 +213,9 @@ func TelemetryFieldKeyToText(key *TelemetryFieldKey) string {
}
func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
if key.MaterializedColumnName != "" {
return fmt.Sprintf("`%s`", key.MaterializedColumnName)
}
return fmt.Sprintf("`%s_%s_%s`",
key.FieldContext.String,
fieldDataTypes[key.FieldDataType.StringValue()].StringValue(),
@@ -210,6 +224,9 @@ func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
}
func FieldKeyToMaterializedColumnNameForExists(key *TelemetryFieldKey) string {
if key.MaterializedColumnName != "" {
return fmt.Sprintf("`%s_exists`", key.MaterializedColumnName)
}
return fmt.Sprintf("`%s_%s_%s_exists`",
key.FieldContext.String,
fieldDataTypes[key.FieldDataType.StringValue()].StringValue(),

721
scripts/semconv/generate.go Normal file
View File

@@ -0,0 +1,721 @@
package main
import (
"bytes"
"errors"
"flag"
"fmt"
"go/format"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
const (
kindAttribute = "attribute"
kindMetric = "metric"
)
type stringListFlag []string
func (f *stringListFlag) String() string { return strings.Join(*f, ",") }
func (f *stringListFlag) Set(value string) error {
*f = append(*f, value)
return nil
}
type schemaFile struct {
FileFormat string `yaml:"file_format"`
SchemaURL string `yaml:"schema_url"`
Versions map[string]schemaVersion `yaml:"versions"`
}
type schemaVersion struct {
All changeSection `yaml:"all"`
Resources changeSection `yaml:"resources"`
Spans changeSection `yaml:"spans"`
Logs changeSection `yaml:"logs"`
Metrics changeSection `yaml:"metrics"`
}
type changeSection struct {
Changes []schemaChange `yaml:"changes"`
}
type schemaChange struct {
RenameAttributes *attributeRename `yaml:"rename_attributes"`
RenameMetrics map[string]string `yaml:"rename_metrics"`
}
type attributeRename struct {
AttributeMap map[string]string `yaml:"attribute_map"`
ApplyToMetrics []string `yaml:"apply_to_metrics"`
}
type overlayFile struct {
DefaultEnabled bool `yaml:"default_enabled"`
// Families is keyed only by current name. One name cannot carry separate
// policies for attribute and metric families; set kind explicitly whenever
// a metric-name family is configured.
Families map[string]overlayFamily `yaml:"families"`
}
type overlayFamily struct {
Enabled *bool `yaml:"enabled"`
Kind string `yaml:"kind"`
Old []string `yaml:"old"`
AddOld []string `yaml:"add_old"`
ExcludeOld []string `yaml:"exclude_old"`
Contexts []string `yaml:"contexts"`
Signals []string `yaml:"signals"`
AddContexts []string `yaml:"add_contexts"`
AddSignals []string `yaml:"add_signals"`
ApplyToMetrics []string `yaml:"apply_to_metrics"`
AddApplyToMetrics []string `yaml:"add_apply_to_metrics"`
ValueMap map[string]string `yaml:"value_map"`
}
type edge struct {
old string
current string
kind string
contexts []string
signals []string
allContexts bool
allSignals bool
applyToMetrics []string
}
type graphKey struct{ kind, name string }
type generatedFamily struct {
Current string
Old []string
Kind string
Contexts []string
Signals []string
ApplyToMetrics []string
ValueMap map[string]string
}
func main() {
root, err := findRepoRoot()
if err != nil {
fatal(err)
}
var schemaPaths stringListFlag
flag.Var(&schemaPaths, "schema", "schema source (repeatable)")
overlayPath := flag.String("overlay", filepath.Join(root, "scripts/semconv/overlay.yaml"), "SigNoz overlay")
goOutput := flag.String("go-out", filepath.Join(root, "pkg/semconv/families_gen.go"), "generated Go output")
tsOutput := flag.String("ts-out", filepath.Join(root, "frontend/src/constants/generated/semconvFamilies.gen.ts"), "generated TypeScript output")
check := flag.Bool("check", false, "fail if generated files are stale")
flag.Parse()
if len(schemaPaths) == 0 {
schemaPaths = append(schemaPaths, filepath.Join(root, "scripts/semconv/schema-1.42.0.yaml"))
}
families, err := generate(schemaPaths, *overlayPath)
if err != nil {
fatal(err)
}
goBytes, err := renderGo(families)
if err != nil {
fatal(err)
}
tsBytes := renderTypeScript(families)
if *check {
if err := checkFile(*goOutput, goBytes); err != nil {
fatal(err)
}
if err := checkFile(*tsOutput, tsBytes); err != nil {
fatal(err)
}
return
}
if err := os.WriteFile(*goOutput, goBytes, 0o644); err != nil {
fatal(err)
}
if err := os.WriteFile(*tsOutput, tsBytes, 0o644); err != nil {
fatal(err)
}
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
func findRepoRoot() (string, error) {
dir, err := os.Getwd()
if err != nil {
return "", err
}
for {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return dir, nil
}
parent := filepath.Dir(dir)
if parent == dir {
return "", errors.New("could not find repository root")
}
dir = parent
}
}
func generate(schemaPaths []string, overlayPath string) ([]generatedFamily, error) {
var schemas []schemaFile
for _, path := range schemaPaths {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("read schema %s: %w", path, err)
}
var schema schemaFile
if err := decodeKnownFields(data, &schema); err != nil {
return nil, fmt.Errorf("parse schema %s: %w", path, err)
}
schemas = append(schemas, schema)
}
overlayData, err := os.ReadFile(overlayPath)
if err != nil {
return nil, fmt.Errorf("read overlay: %w", err)
}
var overlay overlayFile
if err := decodeKnownFields(overlayData, &overlay); err != nil {
return nil, fmt.Errorf("parse overlay: %w", err)
}
return buildFamilies(schemas, overlay)
}
func decodeKnownFields(data []byte, target any) error {
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
return decoder.Decode(target)
}
func collectEdges(schemas []schemaFile) ([]edge, error) {
var edges []edge
for _, schema := range schemas {
versions := make([]string, 0, len(schema.Versions))
versionParts := make(map[string][3]int, len(schema.Versions))
for version := range schema.Versions {
parts, err := parseSchemaVersion(version)
if err != nil {
return nil, err
}
versions = append(versions, version)
versionParts[version] = parts
}
sort.Slice(versions, func(i, j int) bool {
return compareVersionParts(versionParts[versions[i]], versionParts[versions[j]]) < 0
})
for _, versionName := range versions {
version := schema.Versions[versionName]
var versionEdges []edge
sections := []struct {
name string
section changeSection
}{
{name: "all", section: version.All},
{name: "resources", section: version.Resources},
{name: "spans", section: version.Spans},
{name: "logs", section: version.Logs},
{name: "metrics", section: version.Metrics},
}
for _, scoped := range sections {
contexts, signals, allContexts, allSignals, err := scopeForSection(scoped.name)
if err != nil {
return nil, err
}
for _, change := range scoped.section.Changes {
if change.RenameAttributes != nil {
for _, old := range sortedMapKeys(change.RenameAttributes.AttributeMap) {
versionEdges = append(versionEdges, edge{
old: old, current: change.RenameAttributes.AttributeMap[old], kind: kindAttribute,
contexts: contexts, signals: signals,
allContexts: allContexts, allSignals: allSignals,
applyToMetrics: change.RenameAttributes.ApplyToMetrics,
})
}
}
for _, old := range sortedMapKeys(change.RenameMetrics) {
versionEdges = append(versionEdges, edge{
old: old, current: change.RenameMetrics[old], kind: kindMetric,
contexts: []string{"metric"}, signals: []string{"metrics"},
})
}
}
}
if err := rejectSameVersionChains(versionName, versionEdges); err != nil {
return nil, err
}
edges = append(edges, versionEdges...)
}
}
return edges, nil
}
func rejectSameVersionChains(version string, edges []edge) error {
oldNames := make(map[graphKey]struct{}, len(edges))
for _, item := range edges {
oldNames[graphKey{kind: item.kind, name: item.old}] = struct{}{}
}
for _, item := range edges {
if _, ok := oldNames[graphKey{kind: item.kind, name: item.current}]; ok {
return fmt.Errorf(
"schema version %q contains a same-version %s rename chain through %q",
version,
item.kind,
item.current,
)
}
}
return nil
}
func parseSchemaVersion(version string) ([3]int, error) {
parts := strings.Split(version, ".")
if len(parts) != 3 {
return [3]int{}, fmt.Errorf("schema version %q must contain major, minor, and patch numbers", version)
}
var parsed [3]int
for i, part := range parts {
value, err := strconv.Atoi(part)
if err != nil || value < 0 {
return [3]int{}, fmt.Errorf("schema version %q contains invalid numeric component %q", version, part)
}
parsed[i] = value
}
return parsed, nil
}
func compareVersionParts(left, right [3]int) int {
for i := range left {
if left[i] < right[i] {
return -1
}
if left[i] > right[i] {
return 1
}
}
return 0
}
func scopeForSection(section string) (contexts, signals []string, allContexts, allSignals bool, err error) {
switch section {
case "all":
return nil, nil, true, true, nil
case "resources":
return []string{"resource"}, nil, false, true, nil
case "spans":
return []string{"attribute"}, []string{"traces"}, false, false, nil
case "logs":
return []string{"attribute"}, []string{"logs"}, false, false, nil
case "metrics":
return []string{"attribute"}, []string{"metrics"}, false, false, nil
default:
return nil, nil, false, false, fmt.Errorf("unsupported schema section %q", section)
}
}
func buildFamilies(schemas []schemaFile, overlay overlayFile) ([]generatedFamily, error) {
edges, err := collectEdges(schemas)
if err != nil {
return nil, err
}
next := make(map[graphKey]string)
for _, item := range edges {
key := graphKey{kind: item.kind, name: item.old}
if existing, ok := next[key]; ok && existing == item.current {
// Repeated entries are common in chained schema histories. Treat an
// identical edge as a no-op so it cannot sever a later edge in the
// same chain (A -> B, B -> C, then a repeated A -> B).
continue
}
// Schema history occasionally repeats an old name with a newer direct
// destination or rolls a rename back. Edges are collected
// oldest-to-newest, so the latest published current name must be a root.
delete(next, graphKey{kind: item.kind, name: item.current})
next[key] = item.current
}
type familyState struct {
family generatedFamily
distance map[string]int
allContexts bool
allSignals bool
}
states := map[graphKey]*familyState{}
for _, item := range edges {
root, distance, err := rootFor(next, item.kind, item.old)
if err != nil {
return nil, err
}
key := graphKey{kind: item.kind, name: root}
state := states[key]
if state == nil {
state = &familyState{
family: generatedFamily{Current: root, Kind: item.kind},
distance: map[string]int{},
}
states[key] = state
}
if prior, ok := state.distance[item.old]; !ok || distance < prior {
state.distance[item.old] = distance
}
state.allContexts = state.allContexts || item.allContexts
state.allSignals = state.allSignals || item.allSignals
state.family.Contexts = appendUnique(state.family.Contexts, item.contexts...)
state.family.Signals = appendUnique(state.family.Signals, item.signals...)
state.family.ApplyToMetrics = appendUnique(state.family.ApplyToMetrics, item.applyToMetrics...)
}
for _, state := range states {
for old := range state.distance {
if old != state.family.Current {
state.family.Old = append(state.family.Old, old)
}
}
sort.Slice(state.family.Old, func(i, j int) bool {
left, right := state.family.Old[i], state.family.Old[j]
if state.distance[left] != state.distance[right] {
return state.distance[left] < state.distance[right]
}
return left < right
})
if state.allContexts {
state.family.Contexts = nil
} else {
sort.Strings(state.family.Contexts)
}
if state.allSignals {
state.family.Signals = nil
} else {
sort.Strings(state.family.Signals)
}
sort.Strings(state.family.ApplyToMetrics)
}
for _, current := range sortedMapKeys(overlay.Families) {
policy := overlay.Families[current]
kind, err := normalizedOverlayKind(current, policy)
if err != nil {
return nil, err
}
policy.Kind = kind
overlay.Families[current] = policy
key := graphKey{kind: kind, name: current}
state := states[key]
if state == nil {
if len(policy.Old) == 0 {
return nil, fmt.Errorf(
"overlay family %q with kind %q is absent from schemas and has no old members",
current,
kind,
)
}
state = &familyState{
family: generatedFamily{Current: current, Kind: kind, Old: append([]string(nil), policy.Old...)},
distance: map[string]int{},
}
states[key] = state
}
applyOverlay(&state.family, policy)
}
var result []generatedFamily
for key, state := range states {
policy, hasPolicy := overlay.Families[key.name]
enabled := overlay.DefaultEnabled
if hasPolicy && policy.Kind != key.kind {
hasPolicy = false
}
if hasPolicy && policy.Enabled != nil {
enabled = *policy.Enabled
}
if !enabled {
continue
}
if len(state.family.Old) == 0 {
return nil, fmt.Errorf(
"enabled family %q with kind %q has no old members",
state.family.Current,
state.family.Kind,
)
}
sort.Strings(state.family.Contexts)
sort.Strings(state.family.Signals)
sort.Strings(state.family.ApplyToMetrics)
result = append(result, state.family)
}
sort.Slice(result, func(i, j int) bool {
if result[i].Current != result[j].Current {
return result[i].Current < result[j].Current
}
return result[i].Kind < result[j].Kind
})
return result, nil
}
func rootFor(next map[graphKey]string, kind, name string) (string, int, error) {
seen := map[string]bool{}
distance := 0
for {
if seen[name] {
return "", 0, fmt.Errorf("rename cycle for %s %q", kind, name)
}
seen[name] = true
current, ok := next[graphKey{kind: kind, name: name}]
if !ok {
return name, distance, nil
}
name = current
distance++
}
}
func normalizedOverlayKind(current string, policy overlayFamily) (string, error) {
kind := policy.Kind
if kind == "" {
kind = kindAttribute
}
if kind != kindAttribute && kind != kindMetric {
return "", fmt.Errorf("overlay family %q has unsupported kind %q", current, kind)
}
return kind, nil
}
func applyOverlay(family *generatedFamily, policy overlayFamily) {
if policy.Kind != "" {
family.Kind = policy.Kind
}
if policy.Old != nil {
family.Old = append([]string(nil), policy.Old...)
}
family.Old = appendUnique(family.Old, policy.AddOld...)
if len(policy.ExcludeOld) > 0 {
excluded := make(map[string]bool, len(policy.ExcludeOld))
for _, old := range policy.ExcludeOld {
excluded[old] = true
}
family.Old = deleteMatching(family.Old, excluded)
}
if policy.Contexts != nil {
family.Contexts = append([]string(nil), policy.Contexts...)
}
if policy.Signals != nil {
family.Signals = append([]string(nil), policy.Signals...)
}
family.Contexts = appendUnique(family.Contexts, policy.AddContexts...)
family.Signals = appendUnique(family.Signals, policy.AddSignals...)
if policy.ApplyToMetrics != nil {
family.ApplyToMetrics = append([]string(nil), policy.ApplyToMetrics...)
}
family.ApplyToMetrics = appendUnique(family.ApplyToMetrics, policy.AddApplyToMetrics...)
if policy.ValueMap != nil {
family.ValueMap = make(map[string]string, len(policy.ValueMap))
for old, current := range policy.ValueMap {
family.ValueMap[old] = current
}
}
}
func appendUnique(values []string, additions ...string) []string {
seen := make(map[string]bool, len(values)+len(additions))
for _, value := range values {
seen[value] = true
}
for _, value := range additions {
if value == "" || seen[value] {
continue
}
seen[value] = true
values = append(values, value)
}
return values
}
func deleteMatching(values []string, excluded map[string]bool) []string {
result := values[:0]
for _, value := range values {
if !excluded[value] {
result = append(result, value)
}
}
return result
}
func renderGo(families []generatedFamily) ([]byte, error) {
var out bytes.Buffer
out.WriteString("// Code generated by scripts/semconv. DO NOT EDIT.\n\n")
out.WriteString("package semconv\n\n")
needsTelemetryTypes := false
for _, family := range families {
if len(family.Contexts) > 0 || len(family.Signals) > 0 {
needsTelemetryTypes = true
break
}
}
if needsTelemetryTypes {
out.WriteString("import \"github.com/SigNoz/signoz/pkg/types/telemetrytypes\"\n\n")
}
out.WriteString("var families = []Family{\n")
for _, family := range families {
contexts, err := goFieldContextSlice(family.Contexts)
if err != nil {
return nil, fmt.Errorf("render family %q: %w", family.Current, err)
}
signals, err := goSignalSlice(family.Signals)
if err != nil {
return nil, fmt.Errorf("render family %q: %w", family.Current, err)
}
out.WriteString("\t{\n")
fmt.Fprintf(&out, "\t\tCurrent: %s,\n", strconv.Quote(family.Current))
fmt.Fprintf(&out, "\t\tOld: %s,\n", goStringSlice(family.Old))
if family.Kind == kindMetric {
out.WriteString("\t\tKind: KindMetric,\n")
} else {
out.WriteString("\t\tKind: KindAttribute,\n")
}
fmt.Fprintf(&out, "\t\tContexts: %s,\n", contexts)
fmt.Fprintf(&out, "\t\tSignals: %s,\n", signals)
fmt.Fprintf(&out, "\t\tApplyToMetrics: %s,\n", goStringSlice(family.ApplyToMetrics))
if len(family.ValueMap) > 0 {
out.WriteString("\t\tValueMap: map[string]string{\n")
keys := sortedMapKeys(family.ValueMap)
for _, key := range keys {
fmt.Fprintf(&out, "\t\t\t%s: %s,\n", strconv.Quote(key), strconv.Quote(family.ValueMap[key]))
}
out.WriteString("\t\t},\n")
}
out.WriteString("\t},\n")
}
out.WriteString("}\n")
return format.Source(out.Bytes())
}
func goStringSlice(values []string) string {
if len(values) == 0 {
return "nil"
}
quoted := make([]string, len(values))
for i, value := range values {
quoted[i] = strconv.Quote(value)
}
return "[]string{" + strings.Join(quoted, ", ") + "}"
}
func goFieldContextSlice(values []string) (string, error) {
if len(values) == 0 {
return "nil", nil
}
constants := make([]string, len(values))
for i, value := range values {
switch value {
case "metric":
constants[i] = "telemetrytypes.FieldContextMetric"
case "resource":
constants[i] = "telemetrytypes.FieldContextResource"
case "attribute":
constants[i] = "telemetrytypes.FieldContextAttribute"
default:
return "", fmt.Errorf("unsupported field context %q", value)
}
}
return "[]telemetrytypes.FieldContext{" + strings.Join(constants, ", ") + "}", nil
}
func goSignalSlice(values []string) (string, error) {
if len(values) == 0 {
return "nil", nil
}
constants := make([]string, len(values))
for i, value := range values {
switch value {
case "traces":
constants[i] = "telemetrytypes.SignalTraces"
case "logs":
constants[i] = "telemetrytypes.SignalLogs"
case "metrics":
constants[i] = "telemetrytypes.SignalMetrics"
default:
return "", fmt.Errorf("unsupported signal %q", value)
}
}
return "[]telemetrytypes.Signal{" + strings.Join(constants, ", ") + "}", nil
}
func renderTypeScript(families []generatedFamily) []byte {
var out bytes.Buffer
out.WriteString("// Code generated by scripts/semconv. DO NOT EDIT.\n\n")
out.WriteString("export type SemconvFamily = {\n")
out.WriteString("\treadonly current: string;\n\treadonly old: readonly string[];\n")
out.WriteString("\treadonly kind: 'attribute' | 'metric';\n")
out.WriteString("\treadonly contexts: readonly string[];\n\treadonly signals: readonly string[];\n")
out.WriteString("\treadonly applyToMetrics: readonly string[];\n")
out.WriteString("\treadonly valueMap: Readonly<Record<string, string>>;\n};\n\n")
out.WriteString("export const SEMCONV_FAMILIES: readonly SemconvFamily[] = [\n")
for _, family := range families {
out.WriteString("\t{\n")
fmt.Fprintf(&out, "\t\tcurrent: %s,\n", tsString(family.Current))
fmt.Fprintf(&out, "\t\told: %s,\n", tsStringSlice(family.Old))
fmt.Fprintf(&out, "\t\tkind: %s,\n", tsString(family.Kind))
fmt.Fprintf(&out, "\t\tcontexts: %s,\n", tsStringSlice(family.Contexts))
fmt.Fprintf(&out, "\t\tsignals: %s,\n", tsStringSlice(family.Signals))
fmt.Fprintf(&out, "\t\tapplyToMetrics: %s,\n", tsStringSlice(family.ApplyToMetrics))
out.WriteString("\t\tvalueMap: {")
keys := sortedMapKeys(family.ValueMap)
for i, key := range keys {
if i > 0 {
out.WriteString(", ")
}
fmt.Fprintf(&out, "%s: %s", tsString(key), tsString(family.ValueMap[key]))
}
out.WriteString("},\n\t},\n")
}
out.WriteString("] as const;\n")
return out.Bytes()
}
func tsString(value string) string {
quoted := strconv.Quote(value)
return "'" + strings.ReplaceAll(quoted[1:len(quoted)-1], "'", `\'`) + "'"
}
func tsStringSlice(values []string) string {
quoted := make([]string, len(values))
for i, value := range values {
quoted[i] = tsString(value)
}
return "[" + strings.Join(quoted, ", ") + "]"
}
func sortedMapKeys[T any](values map[string]T) []string {
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func checkFile(path string, expected []byte) error {
actual, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("generated file %s is missing: run go run ./scripts/semconv", path)
}
if !bytes.Equal(actual, expected) {
return fmt.Errorf("generated file %s is stale: run go run ./scripts/semconv", path)
}
return nil
}

View File

@@ -0,0 +1,366 @@
package main
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSchemaDecoderRejectsUnsupportedSection(t *testing.T) {
var schema schemaFile
err := decodeKnownFields([]byte(`
versions:
1.0.0:
span_events:
changes:
- rename_events:
event_map:
old: current
`), &schema)
assert.ErrorContains(t, err, "field span_events not found", "unsupported schema sections must fail generation")
}
func TestBuildFamiliesRejectsMalformedSchemaVersion(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
latest:
spans:
changes: []
`), &schema), "test schema must decode")
_, err := buildFamilies([]schemaFile{schema}, overlayFile{})
assert.ErrorContains(t, err, `schema version "latest"`, "malformed versions must not be silently reordered")
}
func TestBuildFamiliesResolvesRenameChain(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
4.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
a: b
3.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
b: c
x: c
2.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
a: b
- rename_attributes:
attribute_map:
a: b
`), &schema), "test schema must decode")
enabled := true
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
"c": {Enabled: &enabled},
}})
require.NoError(t, err)
assert.Equal(t, []generatedFamily{{
Current: "c",
Old: []string{"b", "x", "a"},
Kind: kindAttribute,
Contexts: []string{"attribute"},
Signals: []string{"traces"},
}}, families, "predecessors should be ordered by distance and then name")
}
func TestBuildFamiliesMapsSchemaSectionsToScopes(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
1.0.0:
resources:
changes:
- rename_attributes:
attribute_map:
resource.old: resource.current
logs:
changes:
- rename_attributes:
attribute_map:
log.old: log.current
metrics:
changes:
- rename_attributes:
attribute_map:
state: cpu.mode
apply_to_metrics: [system.cpu.time]
- rename_metrics:
old.metric: current.metric
`), &schema), "test schema must decode")
enabled := true
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
"resource.current": {Enabled: &enabled},
"log.current": {Enabled: &enabled},
"cpu.mode": {Enabled: &enabled},
"current.metric": {Enabled: &enabled, Kind: kindMetric},
}})
require.NoError(t, err)
assert.Equal(t, []generatedFamily{
{
Current: "cpu.mode", Old: []string{"state"}, Kind: kindAttribute,
Contexts: []string{"attribute"}, Signals: []string{"metrics"},
ApplyToMetrics: []string{"system.cpu.time"},
},
{
Current: "current.metric", Old: []string{"old.metric"}, Kind: kindMetric,
Contexts: []string{"metric"}, Signals: []string{"metrics"},
},
{
Current: "log.current", Old: []string{"log.old"}, Kind: kindAttribute,
Contexts: []string{"attribute"}, Signals: []string{"logs"},
},
{
Current: "resource.current", Old: []string{"resource.old"}, Kind: kindAttribute,
Contexts: []string{"resource"},
},
}, families, "schema sections should produce their documented signal and context scopes")
}
func TestOverlayAddsFamilyWithoutSchemaHistory(t *testing.T) {
enabled := true
families, err := buildFamilies(nil, overlayFile{Families: map[string]overlayFamily{
"added.current": {
Enabled: &enabled,
Old: []string{"added.old"},
Contexts: []string{"resource"},
Signals: []string{"traces"},
},
}})
require.NoError(t, err)
assert.Equal(t, []generatedFamily{{
Current: "added.current",
Old: []string{"added.old"},
Kind: kindAttribute,
Contexts: []string{"resource"},
Signals: []string{"traces"},
}}, families, "an explicit overlay family should not require schema history")
}
func TestOverlayOverridesGeneratedFamily(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
1.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
old: current
`), &schema), "test schema must decode")
enabled := true
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
"current": {
Enabled: &enabled,
AddOld: []string{"older"},
ExcludeOld: []string{"old"},
AddContexts: []string{"resource"},
AddSignals: []string{"logs"},
ValueMap: map[string]string{"legacy": "current"},
},
}})
require.NoError(t, err)
assert.Equal(t, []generatedFamily{{
Current: "current",
Old: []string{"older"},
Kind: kindAttribute,
Contexts: []string{"attribute", "resource"},
Signals: []string{"logs", "traces"},
ValueMap: map[string]string{"legacy": "current"},
}}, families, "overlay additions and exclusions should be applied to the generated family")
}
func TestOverlayDisablesFamilyWhenDefaultIsEnabled(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
1.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
old: current
`), &schema), "test schema must decode")
disabled := false
families, err := buildFamilies([]schemaFile{schema}, overlayFile{
DefaultEnabled: true,
Families: map[string]overlayFamily{
"current": {Enabled: &disabled},
},
})
require.NoError(t, err)
assert.Empty(t, families, "an explicitly disabled family must override default_enabled")
}
func TestRenderGoIsDeterministic(t *testing.T) {
families := []generatedFamily{{
Current: "current", Old: []string{"old"}, Kind: kindAttribute,
ValueMap: map[string]string{"b": "2", "a": "1"},
}}
first, err := renderGo(families)
require.NoError(t, err)
second, err := renderGo(families)
require.NoError(t, err)
assert.Equal(t, first, second, "Go generation must not depend on map iteration order")
}
func TestRenderGoUsesCanonicalTelemetryTypes(t *testing.T) {
families := []generatedFamily{{
Current: "current", Old: []string{"old"}, Kind: kindAttribute,
Contexts: []string{"resource"}, Signals: []string{"traces"},
}}
output, err := renderGo(families)
require.NoError(t, err)
assert.Contains(t, string(output), "telemetrytypes.FieldContextResource", "generated contexts should use telemetrytypes")
assert.Contains(t, string(output), "telemetrytypes.SignalTraces", "generated signals should use telemetrytypes")
}
func TestRenderTypeScriptIsDeterministic(t *testing.T) {
families := []generatedFamily{{
Current: "current", Old: []string{"old"}, Kind: kindAttribute,
ValueMap: map[string]string{"b": "2", "a": "1"},
}}
assert.Equal(t, renderTypeScript(families), renderTypeScript(families), "TypeScript generation must not depend on map iteration order")
}
func TestBuildFamiliesHandlesRenameRollback(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
2.0.0:
metrics:
changes:
- rename_metrics:
temporary: original
1.0.0:
metrics:
changes:
- rename_metrics:
original: temporary
`), &schema), "test schema must decode")
enabled := true
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
"original": {Enabled: &enabled, Kind: kindMetric},
}})
require.NoError(t, err)
assert.Equal(t, []generatedFamily{{
Current: "original",
Old: []string{"temporary"},
Kind: kindMetric,
Contexts: []string{"metric"},
Signals: []string{"metrics"},
}}, families, "the latest rollback destination should remain the family root")
}
func TestBuildFamiliesRejectsSameVersionRenameChain(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
1.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
x: y
y: z
`), &schema), "test schema must decode")
_, err := buildFamilies([]schemaFile{schema}, overlayFile{})
assert.ErrorContains(t, err, `same-version attribute rename chain through "y"`, "order-sensitive same-version chains must be rejected")
}
func TestBuildFamiliesRejectsOverlayFamilyWithoutHistory(t *testing.T) {
enabled := true
_, err := buildFamilies(nil, overlayFile{Families: map[string]overlayFamily{
"missing": {Enabled: &enabled},
}})
assert.ErrorContains(t, err, `overlay family "missing" with kind "attribute" is absent`, "an overlay cannot invent a family without old members")
}
func TestBuildFamiliesRejectsEnabledFamilyWithoutOldMembers(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
1.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
old: current
`), &schema), "test schema must decode")
enabled := true
_, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
"current": {Enabled: &enabled, ExcludeOld: []string{"old"}},
}})
assert.ErrorContains(t, err, `enabled family "current" with kind "attribute" has no old members`, "exclude_old cannot empty an enabled family")
}
func TestOverlayKindDefaultsToAttribute(t *testing.T) {
var schema schemaFile
require.NoError(t, decodeKnownFields([]byte(`
versions:
1.0.0:
spans:
changes:
- rename_attributes:
attribute_map:
attribute.old: shared.current
metrics:
changes:
- rename_metrics:
metric.old: shared.current
`), &schema), "test schema must decode")
enabled := true
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
"shared.current": {Enabled: &enabled},
}})
require.NoError(t, err)
assert.Equal(t, []generatedFamily{{
Current: "shared.current",
Old: []string{"attribute.old"},
Kind: kindAttribute,
Contexts: []string{"attribute"},
Signals: []string{"traces"},
}}, families, "a kind-less overlay policy should affect only the attribute family")
}
func TestCheckFileReportsStaleOutput(t *testing.T) {
path := filepath.Join(t.TempDir(), "generated.go")
require.NoError(t, os.WriteFile(path, []byte("old"), 0o600), "test output must be writable")
assert.ErrorContains(t, checkFile(path, []byte("new")), "is stale", "check mode must reject stale generated output")
}
func TestTypeScriptStringEscapesControlCharacters(t *testing.T) {
assert.Equal(t, `'line\n\t\x01\'\\end'`, tsString("line\n\t\x01'\\end"), "generated TypeScript strings must remain valid literals")
}

View File

@@ -0,0 +1,37 @@
# SigNoz semantic-convention rollout policy.
#
# Families are keyed by their current OpenTelemetry name. Schema-derived
# families are disabled by default so rollout remains explicit and reversible.
default_enabled: false
families:
deployment.environment.name:
enabled: true
contexts: [resource, attribute]
signals: [traces, logs, metrics]
db.system.name:
enabled: true
contexts: [resource, attribute]
signals: [traces, logs, metrics]
# These metric renames predate the schema history vendored above. Keep them
# in the same generated registry so every v5 metric query uses one source of
# truth instead of the legacy hand-written transition table.
k8s.pod.cpu.usage:
enabled: true
kind: metric
old: [k8s.pod.cpu.utilization]
contexts: [metric]
signals: [metrics]
k8s.node.cpu.usage:
enabled: true
kind: metric
old: [k8s.node.cpu.utilization]
contexts: [metric]
signals: [metrics]
container.cpu.usage:
enabled: true
kind: metric
old: [container.cpu.utilization]
contexts: [metric]
signals: [metrics]

View File

@@ -0,0 +1,760 @@
file_format: 1.1.0
schema_url: https://opentelemetry.io/schemas/1.42.0
versions:
1.42.0:
metrics:
changes:
- rename_metrics:
v8js.memory.heap.limit: v8js.memory.heap.space.size
1.41.1:
1.41.0:
metrics:
changes:
- rename_metrics:
k8s.container.cpu.limit: k8s.container.cpu.limit.desired
k8s.container.cpu.limit_utilization: k8s.container.cpu.limit.utilization
k8s.container.cpu.request: k8s.container.cpu.request.desired
k8s.container.cpu.request_utilization: k8s.container.cpu.request.utilization
k8s.container.memory.limit: k8s.container.memory.limit.desired
k8s.container.memory.request: k8s.container.memory.request.desired
1.40.0:
all:
changes:
- rename_attributes:
attribute_map:
feature_flag.evaluation.error.message: feature_flag.error.message
metrics:
changes:
- rename_metrics:
system.memory.shared: system.memory.linux.shared
1.39.0:
all:
changes:
- rename_attributes:
attribute_map:
linux.memory.slab.state: system.memory.linux.slab.state
peer.service: service.peer.name
rpc.connect_rpc.error_code: rpc.response.status_code
rpc.connect_rpc.request.metadata: rpc.request.metadata
rpc.connect_rpc.response.metadata: rpc.response.metadata
rpc.grpc.request.metadata: rpc.request.metadata
rpc.grpc.response.metadata: rpc.response.metadata
rpc.jsonrpc.request_id: jsonrpc.request.id
rpc.jsonrpc.version: jsonrpc.protocol.version
rpc.system: rpc.system.name
metrics:
changes:
- rename_metrics:
process.open_file_descriptor.count: process.unix.file_descriptor.count
system.linux.memory.available: system.memory.linux.available
system.linux.memory.slab.usage: system.memory.linux.slab.usage
1.38.0:
all:
changes:
- rename_attributes:
attribute_map:
process.context_switch_type: process.context_switch.type
process.paging.fault_type: system.paging.fault.type
system.cpu.logical_number: cpu.logical_number
system.paging.type: system.paging.fault.type
system.process.status: process.state
system.processes.status: process.state
metrics:
changes:
- rename_metrics:
k8s.cronjob.active_jobs: k8s.cronjob.job.active
k8s.daemonset.current_scheduled_nodes: k8s.daemonset.node.current_scheduled
k8s.daemonset.desired_scheduled_nodes: k8s.daemonset.node.desired_scheduled
k8s.daemonset.misscheduled_nodes: k8s.daemonset.node.misscheduled
k8s.daemonset.ready_nodes: k8s.daemonset.node.ready
k8s.deployment.available_pods: k8s.deployment.pod.available
k8s.deployment.desired_pods: k8s.deployment.pod.desired
k8s.hpa.current_pods: k8s.hpa.pod.current
k8s.hpa.desired_pods: k8s.hpa.pod.desired
k8s.hpa.max_pods: k8s.hpa.pod.max
k8s.hpa.min_pods: k8s.hpa.pod.min
k8s.job.active_pods: k8s.job.pod.active
k8s.job.desired_successful_pods: k8s.job.pod.desired_successful
k8s.job.failed_pods: k8s.job.pod.failed
k8s.job.max_parallel_pods: k8s.job.pod.max_parallel
k8s.job.successful_pods: k8s.job.pod.successful
k8s.node.allocatable.cpu: k8s.node.cpu.allocatable
k8s.node.allocatable.ephemeral_storage: k8s.node.ephemeral_storage.allocatable
k8s.node.allocatable.memory: k8s.node.memory.allocatable
k8s.node.allocatable.pods: k8s.node.pod.allocatable
k8s.replicaset.available_pods: k8s.replicaset.pod.available
k8s.replicaset.desired_pods: k8s.replicaset.pod.desired
k8s.replication_controller.available_pods: k8s.replicationcontroller.pod.available
k8s.replication_controller.desired_pods: k8s.replicationcontroller.pod.desired
k8s.replicationcontroller.available_pods: k8s.replicationcontroller.pod.available
k8s.replicationcontroller.desired_pods: k8s.replicationcontroller.pod.desired
k8s.statefulset.current_pods: k8s.statefulset.pod.current
k8s.statefulset.desired_pods: k8s.statefulset.pod.desired
k8s.statefulset.ready_pods: k8s.statefulset.pod.ready
k8s.statefulset.updated_pods: k8s.statefulset.pod.updated
v8js.heap.space.available_size: v8js.memory.heap.space.available_size
v8js.heap.space.physical_size: v8js.memory.heap.space.physical_size
1.37.0:
all:
changes:
- rename_attributes:
attribute_map:
android.state: android.app.state
container.runtime: container.runtime.name
enduser.role: user.roles
gen_ai.openai.request.service_tier: openai.request.service_tier
gen_ai.openai.response.service_tier: openai.response.service_tier
gen_ai.openai.response.system_fingerprint: openai.response.system_fingerprint
gen_ai.system: gen_ai.provider.name
ios.state: ios.app.state
1.36.0:
1.35.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1698
- rename_attributes:
attribute_map:
az.namespace: azure.resource_provider.namespace
az.service_request_id: azure.service.request.id
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/issues/1800
- rename_metrics:
system.network.connections: system.network.connection.count
1.34.0:
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/2295
- rename_metrics:
cpu.time: system.cpu.time
cpu.utilization: system.cpu.utilization
cpu.frequency: system.cpu.frequency
1.33.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1982
- rename_attributes:
attribute_map:
feature_flag.provider_name: feature_flag.provider.name
# https://github.com/open-telemetry/semantic-conventions/pull/1994
- rename_attributes:
attribute_map:
feature_flag.evaluation.error.message: error.message
1.32.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1989
- rename_attributes:
attribute_map:
feature_flag.evaluation.reason: feature_flag.result.reason
feature_flag.variant: feature_flag.result.variant
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/2042
- rename_metrics:
otel.sdk.span.live.count: otel.sdk.span.live
otel.sdk.span.ended.count: otel.sdk.span.ended
otel.sdk.processor.span.processed.count: otel.sdk.processor.span.processed
otel.sdk.exporter.span.inflight.count: otel.sdk.exporter.span.inflight
otel.sdk.exporter.span.exported.count: otel.sdk.exporter.span.exported
1.31.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1880
- rename_attributes:
attribute_map:
android.state: android.app.state
io.state: ios.app.state
metrics:
changes:
- rename_metrics:
k8s.replication_controller.desired_pods: k8s.replicationcontroller.desired_pods
k8s.replication_controller.available_pods: k8s.replicationcontroller.available_pods
# https://github.com/open-telemetry/semantic-conventions/pull/1896
- rename_metrics:
system.cpu.time: cpu.time
system.cpu.utilization: cpu.utilization
system.cpu.frequency: cpu.frequency
# https://github.com/open-telemetry/semantic-conventions/pull/1896
- rename_attributes:
attribute_map:
system.cpu.logical_number: cpu.logical_number
1.30.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1632
- rename_attributes:
attribute_map:
gen_ai.openai.request.seed: gen_ai.request.seed
system.network.state: network.connection.state
# https://github.com/open-telemetry/semantic-conventions/pull/1624
- rename_attributes:
attribute_map:
code.function: code.function.name
code.filepath: code.file.path
code.lineno: code.line.number
code.column: code.column.number
# https://github.com/open-telemetry/semantic-conventions/pull/1734
- rename_attributes:
attribute_map:
db.system: db.system.name
db.cassandra.coordinator.dc: cassandra.coordinator.dc
db.cassandra.coordinator.id: cassandra.coordinator.id
db.cassandra.consistency_level: cassandra.consistency.level
db.cassandra.idempotence: cassandra.query.idempotent
db.cassandra.page_size: cassandra.page.size
db.cassandra.speculative_execution_count: cassandra.speculative_execution.count
db.cosmosdb.client_id: azure.client.id
db.cosmosdb.connection_mode: azure.cosmosdb.connection.mode
db.cosmosdb.consistency_level: azure.cosmosdb.consistency.level
db.cosmosdb.request_charge: azure.cosmosdb.operation.request_charge
db.cosmosdb.request_content_length: azure.cosmosdb.request.body.size
db.cosmosdb.regions_contacted: azure.cosmosdb.operation.contacted_regions
db.cosmosdb.sub_status_code: azure.cosmosdb.response.sub_status_code
db.elasticsearch.node.name: elasticsearch.node.name
# db.elasticsearch.path_parts is a template attribute, schema transformation
# does not support it, adding as a comment for consistency
# db.elasticsearch.path_parts.<key> -> db.operation.parameter.<key>
metrics:
changes:
- rename_metrics:
db.client.cosmosdb.operation.request_charge: azure.cosmosdb.client.operation.request_charge
db.client.cosmosdb.active_instance.count: azure.cosmosdb.client.active_instance.count
1.29.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1520
- rename_attributes:
attribute_map:
process.executable.build_id.profiling: process.executable.build_id.htlhash
# https://github.com/open-telemetry/semantic-conventions/pull/1383
- rename_attributes:
attribute_map:
vcs.repository.change.id: vcs.change.id
vcs.repository.change.title: vcs.change.title
vcs.repository.ref.name: vcs.ref.head.name
vcs.repository.ref.revision: vcs.ref.head.revision
vcs.repository.ref.type: vcs.ref.head.type
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1492
- rename_attributes:
attribute_map:
system.device: network.interface.name
apply_to_metrics:
- container.network.io
- system.network.dropped
- system.network.errors
- system.network.io
- system.network.connections
1.28.0:
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1422
- rename_metrics:
messaging.client.published.messages: messaging.client.sent.messages
1.27.0:
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1216
- rename_attributes:
attribute_map:
tls.client.server_name: server.address
# https://github.com/open-telemetry/semantic-conventions/pull/1075
- rename_attributes:
attribute_map:
deployment.environment: deployment.environment.name
# https://github.com/open-telemetry/semantic-conventions/pull/1245
- rename_attributes:
attribute_map:
messaging.kafka.message.offset: messaging.kafka.offset
# https://github.com/open-telemetry/semantic-conventions/pull/815
- rename_attributes:
attribute_map:
messaging.kafka.consumer.group: messaging.consumer.group.name
messaging.rocketmq.client_group: messaging.consumer.group.name
messaging.eventhubs.consumer.group: messaging.consumer.group.name
messaging.servicebus.destination.subscription_name: messaging.destination.subscription.name
# https://github.com/open-telemetry/semantic-conventions/pull/1200
- rename_attributes:
attribute_map:
gen_ai.usage.completion_tokens: gen_ai.usage.output_tokens
gen_ai.usage.prompt_tokens: gen_ai.usage.input_tokens
spans:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1002
- rename_attributes:
attribute_map:
db.elasticsearch.cluster.name: db.namespace
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/1125
- rename_attributes:
attribute_map:
db.client.connections.state: db.client.connection.state
apply_to_metrics:
- db.client.connection.count
- rename_attributes:
attribute_map:
db.client.connections.pool.name: db.client.connection.pool.name
apply_to_metrics:
- db.client.connection.count
- db.client.connection.idle.max
- db.client.connection.idle.min
- db.client.connection.max
- db.client.connection.pending_requests
- db.client.connection.timeouts
- db.client.connection.create_time
- db.client.connection.wait_time
- db.client.connection.use_time
# https://github.com/open-telemetry/semantic-conventions/pull/1006
- rename_metrics:
messaging.publish.messages: messaging.client.published.messages
# https://github.com/open-telemetry/semantic-conventions/pull/1026
- rename_attributes:
attribute_map:
system.cpu.state: cpu.mode
process.cpu.state: cpu.mode
container.cpu.state: cpu.mode
apply_to_metrics:
- system.cpu.time
- system.cpu.utilization
- process.cpu.time
- process.cpu.utilization
- container.cpu.time
# https://github.com/open-telemetry/semantic-conventions/pull/1265
- rename_metrics:
jvm.buffer.memory.usage: jvm.buffer.memory.used
1.26.0:
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/966
- rename_metrics:
db.client.connections.usage: db.client.connection.count
db.client.connections.idle.max: db.client.connection.idle.max
db.client.connections.idle.min: db.client.connection.idle.min
db.client.connections.max: db.client.connection.max
db.client.connections.pending_requests: db.client.connection.pending_requests
db.client.connections.timeouts: db.client.connection.timeouts
# https://github.com/open-telemetry/semantic-conventions/pull/948
- rename_attributes:
attribute_map:
messaging.client_id: messaging.client.id
# https://github.com/open-telemetry/semantic-conventions/pull/909
- rename_attributes:
attribute_map:
state: db.client.connections.state
apply_to_metrics:
- db.client.connections.usage
- rename_attributes:
attribute_map:
pool.name: db.client.connections.pool.name
apply_to_metrics:
- db.client.connections.usage
- db.client.connections.idle.max
- db.client.connections.idle.min
- db.client.connections.max
- db.client.connections.pending_requests
- db.client.connections.timeouts
- db.client.connections.create_time
- db.client.connections.wait_time
- db.client.connections.use_time
all:
changes:
# https://github:com/open-telemetry/semantic-conventions/pull/731/
- rename_attributes:
attribute_map:
enduser.id: user.id
1.25.0:
spans:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/911
- rename_attributes:
attribute_map:
db.name: db.namespace
# https://github.com/open-telemetry/semantic-conventions/pull/870
- rename_attributes:
attribute_map:
db.sql.table: db.collection.name
db.mongodb.collection: db.collection.name
db.cosmosdb.container: db.collection.name
db.cassandra.table: db.collection.name
# https://github.com/open-telemetry/semantic-conventions/pull/798
- rename_attributes:
attribute_map:
messaging.kafka.destination.partition: messaging.destination.partition.id
# https://github.com/open-telemetry/semantic-conventions/pull/875
- rename_attributes:
attribute_map:
db.operation: db.operation.name
# https://github.com/open-telemetry/semantic-conventions/pull/913
- rename_attributes:
attribute_map:
messaging.operation: messaging.operation.type
# https://github.com/open-telemetry/semantic-conventions/pull/866
- rename_attributes:
attribute_map:
db.statement: db.query.text
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/484
- rename_attributes:
attribute_map:
system.processes.status: system.process.status
apply_to_metrics:
- system.processes.count
- rename_metrics:
system.processes.count: system.process.count
system.processes.created: system.process.created
# https://github.com/open-telemetry/semantic-conventions/pull/625
- rename_attributes:
attribute_map:
container.labels: container.label
k8s.pod.labels: k8s.pod.label
# https://github.com/open-telemetry/semantic-conventions/pull/330
- rename_metrics:
process.threads: process.thread.count
process.open_file_descriptors: process.open_file_descriptor.count
- rename_attributes:
attribute_map:
state: process.cpu.state
apply_to_metrics:
- process.cpu.time
- process.cpu.utilization
- rename_attributes:
attribute_map:
direction: disk.io.direction
apply_to_metrics:
- process.disk.io
- rename_attributes:
attribute_map:
type: process.context_switch_type
apply_to_metrics:
- process.context_switches
- rename_attributes:
attribute_map:
direction: network.io.direction
apply_to_metrics:
- process.network.io
- rename_attributes:
attribute_map:
type: process.paging.fault_type
apply_to_metrics:
- process.paging.faults
all:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/854
- rename_attributes:
attribute_map:
message.type: rpc.message.type
message.id: rpc.message.id
message.compressed_size: rpc.message.compressed_size
message.uncompressed_size: rpc.message.uncompressed_size
1.24.0:
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/536
- rename_metrics:
jvm.memory.usage: jvm.memory.used
jvm.memory.usage_after_last_gc: jvm.memory.used_after_last_gc
# https://github.com/open-telemetry/semantic-conventions/pull/530
- rename_attributes:
attribute_map:
system.network.io.direction: network.io.direction
system.disk.io.direction: disk.io.direction
1.23.1:
1.23.0:
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/20
- rename_attributes:
attribute_map:
thread.daemon: jvm.thread.daemon
apply_to_metrics:
- jvm.thread.count
1.22.0:
spans:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/229
- rename_attributes:
attribute_map:
messaging.message.payload_size_bytes: messaging.message.body.size
# https://github.com/open-telemetry/opentelemetry-specification/pull/374
- rename_attributes:
attribute_map:
http.resend_count: http.request.resend_count
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/224
- rename_metrics:
http.client.duration: http.client.request.duration
http.server.duration: http.server.request.duration
# https://github.com/open-telemetry/semantic-conventions/pull/241
- rename_metrics:
process.runtime.jvm.memory.usage: jvm.memory.usage
process.runtime.jvm.memory.committed: jvm.memory.committed
process.runtime.jvm.memory.limit: jvm.memory.limit
process.runtime.jvm.memory.usage_after_last_gc: jvm.memory.usage_after_last_gc
process.runtime.jvm.gc.duration: jvm.gc.duration
# also https://github.com/open-telemetry/semantic-conventions/pull/252
process.runtime.jvm.threads.count: jvm.thread.count
# also https://github.com/open-telemetry/semantic-conventions/pull/252
process.runtime.jvm.classes.loaded: jvm.class.loaded
# also https://github.com/open-telemetry/semantic-conventions/pull/252
process.runtime.jvm.classes.unloaded: jvm.class.unloaded
# also https://github.com/open-telemetry/semantic-conventions/pull/252
# and https://github.com/open-telemetry/semantic-conventions/pull/60
process.runtime.jvm.classes.current_loaded: jvm.class.count
process.runtime.jvm.cpu.time: jvm.cpu.time
process.runtime.jvm.cpu.recent_utilization: jvm.cpu.recent_utilization
process.runtime.jvm.memory.init: jvm.memory.init
process.runtime.jvm.system.cpu.utilization: jvm.system.cpu.utilization
process.runtime.jvm.system.cpu.load_1m: jvm.system.cpu.load_1m
# https://github.com/open-telemetry/semantic-conventions/pull/253
process.runtime.jvm.buffer.usage: jvm.buffer.memory.usage
# https://github.com/open-telemetry/semantic-conventions/pull/253
process.runtime.jvm.buffer.limit: jvm.buffer.memory.limit
process.runtime.jvm.buffer.count: jvm.buffer.count
# https://github.com/open-telemetry/semantic-conventions/pull/20
- rename_attributes:
attribute_map:
type: jvm.memory.type
pool: jvm.memory.pool.name
apply_to_metrics:
- jvm.memory.usage
- jvm.memory.committed
- jvm.memory.limit
- jvm.memory.usage_after_last_gc
- jvm.memory.init
- rename_attributes:
attribute_map:
name: jvm.gc.name
action: jvm.gc.action
apply_to_metrics:
- jvm.gc.duration
- rename_attributes:
attribute_map:
daemon: thread.daemon
apply_to_metrics:
- jvm.threads.count
- rename_attributes:
attribute_map:
pool: jvm.buffer.pool.name
apply_to_metrics:
- jvm.buffer.memory.usage
- jvm.buffer.memory.limit
- jvm.buffer.count
# https://github.com/open-telemetry/semantic-conventions/pull/89
- rename_attributes:
attribute_map:
state: system.cpu.state
cpu: system.cpu.logical_number
apply_to_metrics:
- system.cpu.time
- system.cpu.utilization
- rename_attributes:
attribute_map:
state: system.memory.state
apply_to_metrics:
- system.memory.usage
- system.memory.utilization
- rename_attributes:
attribute_map:
state: system.paging.state
apply_to_metrics:
- system.paging.usage
- system.paging.utilization
- rename_attributes:
attribute_map:
type: system.paging.type
direction: system.paging.direction
apply_to_metrics:
- system.paging.faults
- system.paging.operations
- rename_attributes:
attribute_map:
device: system.device
direction: system.disk.direction
apply_to_metrics:
- system.disk.io
- system.disk.operations
- system.disk.io_time
- system.disk.operation_time
- system.disk.merged
- rename_attributes:
attribute_map:
device: system.device
state: system.filesystem.state
type: system.filesystem.type
mode: system.filesystem.mode
mountpoint: system.filesystem.mountpoint
apply_to_metrics:
- system.filesystem.usage
- system.filesystem.utilization
- rename_attributes:
attribute_map:
device: system.device
direction: system.network.direction
protocol: network.protocol
state: system.network.state
apply_to_metrics:
- system.network.dropped
- system.network.packets
- system.network.errors
- system.network.io
- system.network.connections
- rename_attributes:
attribute_map:
status: system.processes.status
apply_to_metrics:
- system.processes.count
# https://github.com/open-telemetry/semantic-conventions/pull/247
- rename_metrics:
http.server.request.size: http.server.request.body.size
http.server.response.size: http.server.response.body.size
resources:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/178
- rename_attributes:
attribute_map:
telemetry.auto.version: telemetry.distro.version
1.21.0:
spans:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/3336
- rename_attributes:
attribute_map:
messaging.kafka.client_id: messaging.client_id
messaging.rocketmq.client_id: messaging.client_id
# https://github.com/open-telemetry/opentelemetry-specification/pull/3402
- rename_attributes:
attribute_map:
# net.peer.(name|port) attributes were usually populated on client side
# so they should be usually translated to server.(address|port)
# net.host.* attributes were only populated on server side
net.host.name: server.address
net.host.port: server.port
# was only populated on client side
net.sock.peer.name: server.socket.domain
# net.sock.peer.(addr|port) mapping is not possible
# since they applied to both client and server side
# were only populated on server side
net.sock.host.addr: server.socket.address
net.sock.host.port: server.socket.port
http.client_ip: client.address
# https://github.com/open-telemetry/opentelemetry-specification/pull/3426
- rename_attributes:
attribute_map:
net.protocol.name: network.protocol.name
net.protocol.version: network.protocol.version
net.host.connection.type: network.connection.type
net.host.connection.subtype: network.connection.subtype
net.host.carrier.name: network.carrier.name
net.host.carrier.mcc: network.carrier.mcc
net.host.carrier.mnc: network.carrier.mnc
net.host.carrier.icc: network.carrier.icc
# https://github.com/open-telemetry/opentelemetry-specification/pull/3355
- rename_attributes:
attribute_map:
http.method: http.request.method
http.status_code: http.response.status_code
http.scheme: url.scheme
http.url: url.full
http.request_content_length: http.request.body.size
http.response_content_length: http.response.body.size
metrics:
changes:
# https://github.com/open-telemetry/semantic-conventions/pull/53
- rename_metrics:
process.runtime.jvm.cpu.utilization: process.runtime.jvm.cpu.recent_utilization
1.20.0:
spans:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/3272
- rename_attributes:
attribute_map:
net.app.protocol.name: net.protocol.name
net.app.protocol.version: net.protocol.version
1.19.0:
spans:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/3209
- rename_attributes:
attribute_map:
faas.execution: faas.invocation_id
# https://github.com/open-telemetry/opentelemetry-specification/pull/3188
- rename_attributes:
attribute_map:
faas.id: cloud.resource_id
# https://github.com/open-telemetry/opentelemetry-specification/pull/3190
- rename_attributes:
attribute_map:
http.user_agent: user_agent.original
resources:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/3190
- rename_attributes:
attribute_map:
browser.user_agent: user_agent.original
1.18.0:
1.17.0:
spans:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/2957
- rename_attributes:
attribute_map:
messaging.consumer_id: messaging.consumer.id
messaging.protocol: net.app.protocol.name
messaging.protocol_version: net.app.protocol.version
messaging.destination: messaging.destination.name
messaging.temp_destination: messaging.destination.temporary
messaging.destination_kind: messaging.destination.kind
messaging.message_id: messaging.message.id
messaging.conversation_id: messaging.message.conversation_id
messaging.message_payload_size_bytes: messaging.message.payload_size_bytes
messaging.message_payload_compressed_size_bytes: messaging.message.payload_compressed_size_bytes
messaging.rabbitmq.routing_key: messaging.rabbitmq.destination.routing_key
messaging.kafka.message_key: messaging.kafka.message.key
messaging.kafka.partition: messaging.kafka.destination.partition
messaging.kafka.tombstone: messaging.kafka.message.tombstone
messaging.rocketmq.message_type: messaging.rocketmq.message.type
messaging.rocketmq.message_tag: messaging.rocketmq.message.tag
messaging.rocketmq.message_keys: messaging.rocketmq.message.keys
messaging.kafka.consumer_group: messaging.kafka.consumer.group
1.16.0:
1.15.0:
spans:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/2743
- rename_attributes:
attribute_map:
http.retry_count: http.resend_count
1.14.0:
1.13.0:
spans:
changes:
# https://github.com/open-telemetry/opentelemetry-specification/pull/2614
- rename_attributes:
attribute_map:
net.peer.ip: net.sock.peer.addr
net.host.ip: net.sock.host.addr
1.12.0:
1.11.0:
1.10.0:
1.9.0:
1.8.0:
spans:
changes:
- rename_attributes:
attribute_map:
db.cassandra.keyspace: db.name
db.hbase.namespace: db.name
1.7.0:
1.6.1:
1.5.0:
1.4.0:

View File

@@ -0,0 +1,157 @@
"""Phase 2 semantic-convention checks across logs and metrics."""
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import requests
from fixtures import querier, types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.metrics import Metrics
DB_CURRENT = "db.system.name"
DB_OLD = "db.system"
METRIC_CURRENT = "container.cpu.usage"
METRIC_OLD = "container.cpu.utilization"
PREFIX = "semconv-phase2"
def _raw_log_bodies(
signoz: types.SigNoz,
token: str,
now: datetime,
expression: str,
) -> set[str]:
response = querier.make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
request_type=querier.RequestType.RAW,
queries=[querier.build_raw_query("A", "logs", limit=100, filter_expression=expression)],
)
assert response.status_code == HTTPStatus.OK, response.text
return {row["data"]["body"] for row in querier.get_rows(response)}
def _field_values(signoz: types.SigNoz, token: str, signal: str, name: str, context: str) -> set[str]:
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=5,
headers={"authorization": f"Bearer {token}"},
params={
"signal": signal,
"name": name,
"fieldContext": context,
"fieldDataType": "string",
},
)
assert response.status_code == HTTPStatus.OK, response.text
return set(response.json()["data"]["values"].get("stringValues") or [])
def test_logs_resolve_db_system_family(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
) -> None:
now = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(minutes=1)
rows = [
("old", {DB_OLD: "postgresql"}),
("current", {DB_CURRENT: "postgresql"}),
("conflict", {DB_OLD: "mysql", DB_CURRENT: "postgresql"}),
("missing", {}),
]
insert_logs(
[
Logs(
timestamp=now + timedelta(seconds=index),
resources={"service.name": PREFIX, **attributes},
attributes=attributes,
body=f"{PREFIX}-{suffix}",
)
for index, (suffix, attributes) in enumerate(rows)
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
present = {f"{PREFIX}-old", f"{PREFIX}-current", f"{PREFIX}-conflict"}
for context in ("attribute", "resource"):
for requested in (DB_CURRENT, DB_OLD):
field = f"{context}.{requested}"
assert _raw_log_bodies(signoz, token, now, f'{field} = "postgresql"') == present
assert _raw_log_bodies(signoz, token, now, f"{field} EXISTS") == present
assert _raw_log_bodies(signoz, token, now, f"{field} NOT EXISTS") == {f"{PREFIX}-missing"}
assert _field_values(signoz, token, "logs", requested, context) == {"postgresql", "mysql"}
def test_metrics_resolve_label_and_metric_name_families(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_metrics: Callable[[list[Metrics]], None],
) -> None:
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
insert_metrics(
[
Metrics(
metric_name=METRIC_CURRENT,
labels={DB_CURRENT: "postgresql"},
timestamp=now - timedelta(seconds=3),
temporality="Unspecified",
type_="Gauge",
is_monotonic=False,
value=10,
),
Metrics(
metric_name=METRIC_OLD,
labels={"db_system": "mysql"},
timestamp=now - timedelta(seconds=2),
temporality="Unspecified",
type_="Gauge",
is_monotonic=False,
value=20,
),
Metrics(
metric_name=METRIC_OLD,
labels={DB_OLD: "mysql", DB_CURRENT: "postgresql", "series": "conflict"},
timestamp=now - timedelta(seconds=1),
temporality="Unspecified",
type_="Gauge",
is_monotonic=False,
value=30,
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
for metric_name in (METRIC_CURRENT, METRIC_OLD):
for requested_label in (DB_CURRENT, DB_OLD):
response = querier.make_scalar_query_request(
signoz,
token,
now,
[
querier.build_scalar_query(
name="A",
signal="metrics",
aggregations=[
querier.build_metrics_aggregation(
metric_name,
"latest",
"sum",
"unspecified",
reduce_to="last",
)
],
group_by=[querier.build_group_by_field(requested_label, "string", "attribute")],
filter_expression=f"attribute.{requested_label} EXISTS",
)
],
)
assert response.status_code == HTTPStatus.OK, response.text
data = {row[0]: row[-1] for row in querier.get_scalar_table_data(response.json())}
assert data == {"postgresql": 40.0, "mysql": 20.0}, (metric_name, requested_label, data)

View File

@@ -0,0 +1,239 @@
"""Phase 1 end-to-end checks for semantic-convention name evolution.
The fixture models a fleet split across SDK generations and deliberately includes
a dual-emitting conflict. Both request spellings must address one logical field,
with the current spelling winning when a row contains both.
"""
from collections.abc import Callable, Generator
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
from typing import Any
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import Aggregation, BuilderQuery, OrderBy, RequestType, TelemetryFieldKey, make_query_request
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
CURRENT = "deployment.environment.name"
OLD = "deployment.environment"
PREFIX = "semconv-phase1"
PRODUCTION_SPANS = {
f"{PREFIX}-old",
f"{PREFIX}-current",
f"{PREFIX}-both",
f"{PREFIX}-conflict",
}
STAGING_SPANS = {f"{PREFIX}-staging"}
MISSING_SPANS = {f"{PREFIX}-missing"}
def _span(timestamp: datetime, suffix: str, environment: dict[str, str]) -> Traces:
service = f"{PREFIX}-{suffix}"
return Traces(
timestamp=timestamp,
duration=timedelta(milliseconds=10),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
name=service,
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": service, **environment},
attributes=dict(environment),
)
@pytest.fixture(name="semconv_phase1_data")
def semconv_phase1_data(
insert_traces: Callable[[list[Traces]], None],
clickhouse: types.TestContainerClickhouse,
) -> Generator[datetime]:
now = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(minutes=2)
insert_traces(
[
_span(now - timedelta(seconds=5), "old", {OLD: "production"}),
_span(now - timedelta(seconds=4), "current", {CURRENT: "production"}),
_span(now - timedelta(seconds=3), "both", {OLD: "production", CURRENT: "production"}),
_span(now - timedelta(seconds=2), "conflict", {OLD: "staging", CURRENT: "production"}),
_span(now - timedelta(seconds=1), "staging", {OLD: "staging"}),
_span(now, "missing", {}),
]
)
# Service-map rows are derived by the collector in production. Seed the
# derived table directly here so the backend alias allowlist is tested in
# isolation; the collector repository owns its write-path integration test.
for environment, suffix in (("production", "production"), ("staging", "staging")):
clickhouse.conn.command(
f"""
INSERT INTO signoz_traces.distributed_dependency_graph_minutes_v2
(src, dest, duration_quantiles_state, error_count, total_count, timestamp,
deployment_environment, k8s_cluster_name, k8s_namespace_name)
SELECT
'{PREFIX}-map-{suffix}', '{PREFIX}-map-child',
quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(1000000)),
toUInt64(0), toUInt64(1), toDateTime({int(now.timestamp())}),
'{environment}', '', ''
"""
)
yield now
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
clickhouse.conn.command(
f"ALTER TABLE signoz_traces.dependency_graph_minutes_v2 ON CLUSTER '{cluster}' "
f"DELETE WHERE startsWith(src, '{PREFIX}-map-') SETTINGS mutations_sync = 1"
)
def _result(response: requests.Response) -> dict[str, Any]:
assert response.status_code == HTTPStatus.OK, response.text
results = response.json()["data"]["data"]["results"]
assert len(results) == 1
return results[0]
def _raw_names(
signoz: types.SigNoz,
token: str,
now: datetime,
expression: str,
) -> set[str]:
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
request_type=RequestType.RAW,
queries=[
BuilderQuery(
signal="traces",
name="A",
limit=100,
filter_expression=expression,
select_fields=[TelemetryFieldKey("span.name")],
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
).to_dict()
],
)
return {row["data"]["name"] for row in (_result(response).get("rows") or [])}
def _metadata_values(signoz: types.SigNoz, token: str, name: str, context: str) -> set[str]:
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=5,
headers={"authorization": f"Bearer {token}"},
params={
"signal": "traces",
"name": name,
"fieldContext": context,
"fieldDataType": "string",
},
)
assert response.status_code == HTTPStatus.OK, response.text
return set(response.json()["data"]["values"].get("stringValues") or [])
def test_semconv_phase1_mixed_sdk_generations( # pylint: disable=too-many-statements
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
semconv_phase1_data: datetime,
) -> None:
now = semconv_phase1_data
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Resource and span-attribute paths share the same matrix. Run every
# operator with both the saved-query (old) and current request spellings.
for context in ("resource", "attribute"):
for requested in (CURRENT, OLD):
field = f"{context}.{requested}"
assert _raw_names(signoz, token, now, f"{field} = 'production'") == PRODUCTION_SPANS
assert _raw_names(signoz, token, now, f"{field} = 'staging'") == STAGING_SPANS
assert _raw_names(signoz, token, now, f"{field} != 'production'") == STAGING_SPANS
assert _raw_names(signoz, token, now, f"{field} EXISTS") == PRODUCTION_SPANS | STAGING_SPANS
assert _raw_names(signoz, token, now, f"{field} NOT EXISTS") == MISSING_SPANS
grouped = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
request_type=RequestType.SCALAR,
queries=[
BuilderQuery(
signal="traces",
name="A",
filter_expression=f"{field} EXISTS",
aggregations=[Aggregation("count()")],
group_by=[TelemetryFieldKey(requested, "string", context)],
order=[OrderBy(TelemetryFieldKey(requested, "string", context), "asc")],
).to_dict()
],
)
result = _result(grouped)
assert result["columns"][0]["name"] == requested, "response identity must match the request spelling"
assert result["data"] == [["production", 4], ["staging", 1]]
assert _metadata_values(signoz, token, CURRENT, context) == {"production", "staging"}
assert _metadata_values(signoz, token, OLD, context) == {"production", "staging"}
keys_response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=5,
headers={"authorization": f"Bearer {token}"},
params={"signal": "traces", "searchText": OLD},
)
assert keys_response.status_code == HTTPStatus.OK, keys_response.text
keys = keys_response.json()["data"]["keys"]
assert CURRENT in keys
assert OLD not in keys
start_ns = str(int((now - timedelta(minutes=2)).timestamp() * 1_000_000_000))
end_ns = str(int((now + timedelta(minutes=1)).timestamp() * 1_000_000_000))
for requested in (CURRENT, OLD):
services_response = requests.post(
signoz.self.host_configs["8080"].get("/api/v2/services"),
timeout=30,
headers={"authorization": f"Bearer {token}"},
json={
"start": start_ns,
"end": end_ns,
"tags": [
{
"Key": requested,
"Operator": "In",
"StringValues": ["production"],
"TagType": "ResourceAttribute",
}
],
},
)
assert services_response.status_code == HTTPStatus.OK, services_response.text
services = {item["serviceName"] for item in services_response.json()["data"]}
assert services == PRODUCTION_SPANS
map_response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/dependency_graph"),
timeout=30,
headers={"authorization": f"Bearer {token}"},
json={
"start": start_ns,
"end": end_ns,
"tags": [
{
"key": requested,
"operator": "In",
"stringValues": ["production"],
"tagType": "ResourceAttribute",
}
],
},
)
assert map_response.status_code == HTTPStatus.OK, map_response.text
assert {edge["parent"] for edge in map_response.json()} == {f"{PREFIX}-map-production"}