mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-06 21:20:42 +01:00
Compare commits
5 Commits
main
...
feat/semco
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b4bb82efb | ||
|
|
487d00b6c9 | ||
|
|
59329d3aaa | ||
|
|
7c58b6af49 | ||
|
|
e730192885 |
15
.github/workflows/goci.yaml
vendored
15
.github/workflows/goci.yaml
vendored
@@ -53,6 +53,21 @@ jobs:
|
||||
with:
|
||||
PRIMUS_REF: main
|
||||
GO_VERSION: 1.24
|
||||
semconv-generated:
|
||||
if: |
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
|
||||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: self-checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: go-install
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.24"
|
||||
- name: check-semconv-generated-files
|
||||
run: go run ./scripts/semconv -check
|
||||
build:
|
||||
if: |
|
||||
github.event_name == 'merge_group' ||
|
||||
|
||||
12
Makefile
12
Makefile
@@ -220,6 +220,14 @@ py-test-teardown: ## Tear down the shared SigNoz backend
|
||||
py-test: ## Runs integration tests
|
||||
@cd tests && uv run pytest --basetemp=./tmp/ -vv --capture=no integration/tests/
|
||||
|
||||
.PHONY: py-test-semconv-phase1
|
||||
py-test-semconv-phase1: py-test-setup ## Rebuild the shared stack and run the semantic-convention Phase 1 matrix
|
||||
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py
|
||||
|
||||
.PHONY: py-test-semconv-phase2
|
||||
py-test-semconv-phase2: py-test-setup ## Rebuild the shared stack and run the Phase 1-2 cross-signal matrices
|
||||
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py integration/tests/queriersemconv/02_cross_signal.py
|
||||
|
||||
.PHONY: py-clean
|
||||
py-clean: ## Clear all pycache and pytest cache from tests directory recursively
|
||||
@echo ">> cleaning python cache files from tests directory"
|
||||
@@ -233,6 +241,10 @@ py-clean: ## Clear all pycache and pytest cache from tests directory recursively
|
||||
##############################################################
|
||||
# generate commands
|
||||
##############################################################
|
||||
.PHONY: semconv-generate
|
||||
semconv-generate: ## Regenerate semantic-convention families for Go and TypeScript
|
||||
@go run ./scripts/semconv
|
||||
|
||||
.PHONY: gen-mocks
|
||||
gen-mocks:
|
||||
@echo ">> Generating mocks"
|
||||
|
||||
59
frontend/src/constants/generated/semconvFamilies.gen.ts
Normal file
59
frontend/src/constants/generated/semconvFamilies.gen.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
// Code generated by scripts/semconv. DO NOT EDIT.
|
||||
|
||||
export type SemconvFamily = {
|
||||
readonly current: string;
|
||||
readonly old: readonly string[];
|
||||
readonly kind: 'attribute' | 'metric';
|
||||
readonly contexts: readonly string[];
|
||||
readonly signals: readonly string[];
|
||||
readonly applyToMetrics: readonly string[];
|
||||
readonly valueMap: Readonly<Record<string, string>>;
|
||||
};
|
||||
|
||||
export const SEMCONV_FAMILIES: readonly SemconvFamily[] = [
|
||||
{
|
||||
current: 'container.cpu.usage',
|
||||
old: ['container.cpu.utilization'],
|
||||
kind: 'metric',
|
||||
contexts: ['metric'],
|
||||
signals: ['metrics'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'db.system.name',
|
||||
old: ['db.system'],
|
||||
kind: 'attribute',
|
||||
contexts: ['attribute', 'resource'],
|
||||
signals: ['logs', 'metrics', 'traces'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'deployment.environment.name',
|
||||
old: ['deployment.environment'],
|
||||
kind: 'attribute',
|
||||
contexts: ['attribute', 'resource'],
|
||||
signals: ['logs', 'metrics', 'traces'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'k8s.node.cpu.usage',
|
||||
old: ['k8s.node.cpu.utilization'],
|
||||
kind: 'metric',
|
||||
contexts: ['metric'],
|
||||
signals: ['metrics'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'k8s.pod.cpu.usage',
|
||||
old: ['k8s.pod.cpu.utilization'],
|
||||
kind: 'metric',
|
||||
contexts: ['metric'],
|
||||
signals: ['metrics'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
] as const;
|
||||
@@ -52,10 +52,10 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
|
||||
chErrors "github.com/SigNoz/signoz/pkg/query-service/errors"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/metrics"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -3190,11 +3190,6 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID v
|
||||
var rows driver.Rows
|
||||
var attributeValues v3.FilterAttributeValueResponse
|
||||
|
||||
normalized := true
|
||||
if constants.IsDotMetricsEnabled {
|
||||
normalized = false
|
||||
}
|
||||
|
||||
reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID))
|
||||
|
||||
if reductionEnabled {
|
||||
@@ -3205,8 +3200,7 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID v
|
||||
if req.Limit != 0 {
|
||||
query = query + fmt.Sprintf(" LIMIT %d;", req.Limit)
|
||||
}
|
||||
names := []string{req.AggregateAttribute}
|
||||
names = append(names, metrics.GetTransitionedMetric(req.AggregateAttribute, normalized))
|
||||
names := semconv.MetricNames(req.AggregateAttribute)
|
||||
|
||||
rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, names, req.FilterAttributeKey, fmt.Sprintf("%%%s%%", req.SearchText), common.PastDayRoundOff())
|
||||
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
var resourceLogOperators = map[v3.FilterOperator]string{
|
||||
@@ -29,13 +31,61 @@ var resourceLogOperators = map[v3.FilterOperator]string{
|
||||
v3.FilterOperatorNotILike: "NOT ILIKE",
|
||||
}
|
||||
|
||||
func resourceSemconvMembers(key string) []string {
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: key,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
})
|
||||
}
|
||||
|
||||
func resourceValueExpression(key string) string {
|
||||
members := resourceSemconvMembers(key)
|
||||
if len(members) == 1 {
|
||||
return fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key)
|
||||
}
|
||||
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(simpleJSONExtractString(labels, '%s'), '')", member))
|
||||
}
|
||||
return "COALESCE(" + strings.Join(values, ", ") + ")"
|
||||
}
|
||||
|
||||
func resourcePresenceExpression(key string, exists bool) string {
|
||||
members := resourceSemconvMembers(key)
|
||||
if len(members) == 1 {
|
||||
if exists {
|
||||
return fmt.Sprintf("simpleJSONHas(labels, '%s')", key)
|
||||
}
|
||||
return fmt.Sprintf("not simpleJSONHas(labels, '%s')", key)
|
||||
}
|
||||
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
if exists {
|
||||
conditions = append(conditions, fmt.Sprintf("simpleJSONHas(labels, '%s')", member))
|
||||
} else {
|
||||
conditions = append(conditions, fmt.Sprintf("not simpleJSONHas(labels, '%s')", member))
|
||||
}
|
||||
}
|
||||
separator := " OR "
|
||||
if !exists {
|
||||
separator = " AND "
|
||||
}
|
||||
return "(" + strings.Join(conditions, separator) + ")"
|
||||
}
|
||||
|
||||
// buildResourceFilter builds a clickhouse filter string for resource labels
|
||||
func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value interface{}) string {
|
||||
// for all operators except contains and like
|
||||
searchKey := fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key)
|
||||
searchKey := resourceValueExpression(key)
|
||||
|
||||
// for contains and like it will be case insensitive
|
||||
lowerSearchKey := fmt.Sprintf("simpleJSONExtractString(lower(labels), '%s')", key)
|
||||
if len(resourceSemconvMembers(key)) > 1 {
|
||||
lowerSearchKey = "lower(" + searchKey + ")"
|
||||
}
|
||||
|
||||
chFmtVal := utils.ClickHouseFormattedValue(value)
|
||||
|
||||
@@ -43,9 +93,9 @@ func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value
|
||||
|
||||
switch op {
|
||||
case v3.FilterOperatorExists:
|
||||
return fmt.Sprintf("simpleJSONHas(labels, '%s')", key)
|
||||
return resourcePresenceExpression(key, true)
|
||||
case v3.FilterOperatorNotExists:
|
||||
return fmt.Sprintf("not simpleJSONHas(labels, '%s')", key)
|
||||
return resourcePresenceExpression(key, false)
|
||||
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex:
|
||||
return fmt.Sprintf(logsOp, searchKey, chFmtVal)
|
||||
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
|
||||
@@ -110,6 +160,38 @@ func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value inter
|
||||
// we can use lower index for =, in etc but it's difficult to do it for !=, NIN etc
|
||||
// if as x != "ABC" we cannot predict something like "not lower(labels) like '%%x%%abc%%'". It has it be "not lower(labels) like '%%x%%ABC%%'"
|
||||
func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{}) string {
|
||||
return buildResourceIndexFilterForKey(key, op, value, true)
|
||||
}
|
||||
|
||||
func buildResourceIndexFilterForKey(key string, op v3.FilterOperator, value interface{}, resolveFamily bool) string {
|
||||
members := []string{key}
|
||||
if resolveFamily {
|
||||
members = resourceSemconvMembers(key)
|
||||
}
|
||||
if len(members) > 1 {
|
||||
switch op {
|
||||
case v3.FilterOperatorNotEqual,
|
||||
v3.FilterOperatorNotLike,
|
||||
v3.FilterOperatorNotILike,
|
||||
v3.FilterOperatorNotContains,
|
||||
v3.FilterOperatorNotExists,
|
||||
v3.FilterOperatorNotRegex,
|
||||
v3.FilterOperatorNotIn:
|
||||
return ""
|
||||
}
|
||||
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
if condition := buildResourceIndexFilterForKey(member, op, value, false); condition != "" {
|
||||
conditions = append(conditions, condition)
|
||||
}
|
||||
}
|
||||
if len(conditions) == 0 {
|
||||
return ""
|
||||
}
|
||||
return "(" + strings.Join(conditions, " OR ") + ")"
|
||||
}
|
||||
|
||||
// not using clickhouseFormattedValue as we don't wan't the quotes
|
||||
strVal := fmt.Sprintf("%s", value)
|
||||
fmtValEscapedForContains := utils.QuoteEscapedStringForContains(strVal, true)
|
||||
@@ -206,14 +288,31 @@ func buildResourceFiltersFromGroupBy(groupBy []v3.AttributeKey) []string {
|
||||
if attr.Type != v3.AttributeKeyTypeResource {
|
||||
continue
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", attr.Key, attr.Key))
|
||||
members := resourceSemconvMembers(attr.Key)
|
||||
if len(members) == 1 {
|
||||
conditions = append(conditions, fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", attr.Key, attr.Key))
|
||||
continue
|
||||
}
|
||||
indexConditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
indexConditions = append(indexConditions, fmt.Sprintf("labels like '%%%s%%'", member))
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf("(%s AND (%s))", resourcePresenceExpression(attr.Key, true), strings.Join(indexConditions, " OR ")))
|
||||
}
|
||||
return conditions
|
||||
}
|
||||
|
||||
func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeKey) string {
|
||||
if aggregateAttribute.Key != "" && aggregateAttribute.Type == v3.AttributeKeyTypeResource {
|
||||
return fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", aggregateAttribute.Key, aggregateAttribute.Key)
|
||||
members := resourceSemconvMembers(aggregateAttribute.Key)
|
||||
if len(members) == 1 {
|
||||
return fmt.Sprintf("(simpleJSONHas(labels, '%s') AND labels like '%%%s%%')", aggregateAttribute.Key, aggregateAttribute.Key)
|
||||
}
|
||||
indexConditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
indexConditions = append(indexConditions, fmt.Sprintf("labels like '%%%s%%'", member))
|
||||
}
|
||||
return fmt.Sprintf("(%s AND (%s))", resourcePresenceExpression(aggregateAttribute.Key, true), strings.Join(indexConditions, " OR "))
|
||||
}
|
||||
|
||||
return ""
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"testing"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func Test_buildResourceFilter(t *testing.T) {
|
||||
@@ -552,3 +554,38 @@ func Test_buildResourceSubQuery(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticConventionResourceFamily(t *testing.T) {
|
||||
const resolvedValue = "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''))"
|
||||
|
||||
for _, requestedName := range []string{"deployment.environment.name", "deployment.environment"} {
|
||||
t.Run(requestedName, func(t *testing.T) {
|
||||
assert.Equal(t, resolvedValue+" = 'production'", buildResourceFilter("=", requestedName, v3.FilterOperatorEqual, "production"))
|
||||
assert.Equal(t, "(simpleJSONHas(labels, 'deployment.environment.name') OR simpleJSONHas(labels, 'deployment.environment'))", buildResourceFilter("", requestedName, v3.FilterOperatorExists, nil))
|
||||
assert.Equal(t, "(not simpleJSONHas(labels, 'deployment.environment.name') AND not simpleJSONHas(labels, 'deployment.environment'))", buildResourceFilter("", requestedName, v3.FilterOperatorNotExists, nil))
|
||||
assert.Equal(t, "(labels like '%deployment.environment.name\":\"production%' OR labels like '%deployment.environment\":\"production%')", buildResourceIndexFilter(requestedName, v3.FilterOperatorEqual, "production"))
|
||||
assert.Empty(t, buildResourceIndexFilter(requestedName, v3.FilterOperatorNotEqual, "production"), "negative family filter must not use a rejecting index hint")
|
||||
})
|
||||
}
|
||||
|
||||
filters, err := buildResourceFiltersFromFilterItems(&v3.FilterSet{Items: []v3.FilterItem{{
|
||||
Key: v3.AttributeKey{
|
||||
Key: "deployment.environment.name",
|
||||
DataType: v3.AttributeKeyDataTypeString,
|
||||
Type: v3.AttributeKeyTypeResource,
|
||||
},
|
||||
Operator: v3.FilterOperatorEqual,
|
||||
Value: "production",
|
||||
}}})
|
||||
require.NoError(t, err, "family filter items must build before their output is inspected")
|
||||
wantFilters := []string{
|
||||
resolvedValue + " = 'production'",
|
||||
"(labels like '%deployment.environment.name\":\"production%' OR labels like '%deployment.environment\":\"production%')",
|
||||
}
|
||||
assert.Equal(t, wantFilters, filters)
|
||||
|
||||
wantPresence := "((simpleJSONHas(labels, 'deployment.environment.name') OR simpleJSONHas(labels, 'deployment.environment')) AND (labels like '%deployment.environment.name%' OR labels like '%deployment.environment%'))"
|
||||
groupBy := buildResourceFiltersFromGroupBy([]v3.AttributeKey{{Key: "deployment.environment", Type: v3.AttributeKeyTypeResource}})
|
||||
assert.Equal(t, []string{wantPresence}, groupBy)
|
||||
assert.Equal(t, wantPresence, buildResourceFiltersFromAggregateAttribute(v3.AttributeKey{Key: "deployment.environment.name", Type: v3.AttributeKeyTypeResource}))
|
||||
}
|
||||
|
||||
@@ -6,16 +6,32 @@ import (
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
var (
|
||||
columns = map[string]struct{}{
|
||||
"deployment_environment": {},
|
||||
"k8s_cluster_name": {},
|
||||
"k8s_namespace_name": {},
|
||||
}
|
||||
columns = serviceMapColumns()
|
||||
)
|
||||
|
||||
func serviceMapColumns() map[string]string {
|
||||
columns := map[string]string{
|
||||
"k8s_cluster_name": "k8s_cluster_name",
|
||||
"k8s_namespace_name": "k8s_namespace_name",
|
||||
}
|
||||
|
||||
// Dependency-graph rows keep their historical physical column name. Both
|
||||
// semantic-convention request spellings target that same derived column.
|
||||
for _, member := range semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: "deployment.environment.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}) {
|
||||
columns[strings.ReplaceAll(member, ".", "_")] = "deployment_environment"
|
||||
}
|
||||
return columns
|
||||
}
|
||||
|
||||
func BuildServiceMapQuery(tags []model.TagQuery) (string, []interface{}) {
|
||||
var filterQuery string
|
||||
var namedArgs []interface{}
|
||||
@@ -24,39 +40,40 @@ func BuildServiceMapQuery(tags []model.TagQuery) (string, []interface{}) {
|
||||
operator := tag.GetOperator()
|
||||
value := tag.GetValues()
|
||||
|
||||
if _, ok := columns[key]; !ok {
|
||||
column, ok := columns[key]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
switch operator {
|
||||
case model.InOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s IN @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s IN @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, value))
|
||||
case model.NotInOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s NOT IN @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s NOT IN @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, value))
|
||||
case model.EqualOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s = @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s = @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, value))
|
||||
case model.NotEqualOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s != @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s != @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, value))
|
||||
case model.ContainsOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%%%s%%", value)))
|
||||
case model.NotContainsOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%%%s%%", value)))
|
||||
case model.StartsWithOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%s%%", value)))
|
||||
case model.NotStartsWithOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", key, key)
|
||||
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", column, key)
|
||||
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%s%%", value)))
|
||||
case model.ExistsOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s IS NOT NULL", key)
|
||||
filterQuery += fmt.Sprintf(" AND %s IS NOT NULL", column)
|
||||
case model.NotExistsOperator:
|
||||
filterQuery += fmt.Sprintf(" AND %s IS NULL", key)
|
||||
filterQuery += fmt.Sprintf(" AND %s IS NULL", column)
|
||||
}
|
||||
}
|
||||
return filterQuery, namedArgs
|
||||
|
||||
35
pkg/query-service/app/services/map_test.go
Normal file
35
pkg/query-service/app/services/map_test.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildServiceMapQueryAcceptsEnvironmentFamily(t *testing.T) {
|
||||
for _, requestedName := range []string{"deployment.environment.name", "deployment.environment"} {
|
||||
t.Run(requestedName, func(t *testing.T) {
|
||||
tags := []model.TagQuery{model.NewTagQueryString(model.TagQueryParam{
|
||||
Key: requestedName,
|
||||
StringValues: []string{"production"},
|
||||
Operator: model.InOperator,
|
||||
TagType: model.ResourceAttributeTagType,
|
||||
})}
|
||||
|
||||
query, args := BuildServiceMapQuery(tags)
|
||||
argName := "deployment_environment"
|
||||
if requestedName == "deployment.environment.name" {
|
||||
argName = "deployment_environment_name"
|
||||
}
|
||||
assert.Equal(t, " AND deployment_environment IN @"+argName, query)
|
||||
require.Len(t, args, 1)
|
||||
named, ok := args[0].(driver.NamedValue)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, argName, named.Name)
|
||||
assert.Equal(t, []interface{}{"production"}, named.Value)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package metrics
|
||||
|
||||
var MetricsUnderTransition = map[string]string{
|
||||
"k8s_pod_cpu_utilization": "k8s_pod_cpu_usage",
|
||||
"k8s_node_cpu_utilization": "k8s_node_cpu_usage",
|
||||
"container_cpu_utilization": "container_cpu_usage",
|
||||
}
|
||||
|
||||
var DotMetricsUnderTransition = map[string]string{
|
||||
"k8s.pod.cpu.utilization": "k8s.pod.cpu.usage",
|
||||
"k8s.node.cpu.utilization": "k8s.node.cpu.usage",
|
||||
"container.cpu.utilization": "container.cpu.usage",
|
||||
}
|
||||
|
||||
func GetTransitionedMetric(metric string, normalized bool) string {
|
||||
if normalized {
|
||||
if _, ok := MetricsUnderTransition[metric]; ok {
|
||||
return MetricsUnderTransition[metric]
|
||||
}
|
||||
return metric
|
||||
} else {
|
||||
if _, ok := DotMetricsUnderTransition[metric]; ok {
|
||||
return DotMetricsUnderTransition[metric]
|
||||
}
|
||||
return metric
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/metrics"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
)
|
||||
|
||||
// ValidateAndCastValue validates and casts the value of a key to the corresponding data type of the key
|
||||
@@ -234,12 +234,12 @@ func ClickHouseFormattedValue(v interface{}) string {
|
||||
|
||||
func ClickHouseFormattedMetricNames(v interface{}) string {
|
||||
if name, ok := v.(string); ok {
|
||||
transitionedMetrics := metrics.GetTransitionedMetric(name, !constants.IsDotMetricsEnabled)
|
||||
if transitionedMetrics != name {
|
||||
return ClickHouseFormattedValue([]interface{}{transitionedMetrics})
|
||||
} else {
|
||||
return ClickHouseFormattedValue([]interface{}{name})
|
||||
members := semconv.MetricNames(name)
|
||||
values := make([]interface{}, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, member)
|
||||
}
|
||||
return ClickHouseFormattedValue(values)
|
||||
}
|
||||
|
||||
return ClickHouseFormattedValue(v)
|
||||
|
||||
@@ -2,13 +2,29 @@ package querybuilder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
func physicalSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return []string{key.Name}
|
||||
}
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: key.Signal,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
// ExistsExpression renders the existence predicate for a key resolved to the given
|
||||
// columns (negated when exists is false). Comparisons are against constants rendered
|
||||
// as literals, so the expression carries no bind args and can guard column expressions
|
||||
@@ -43,11 +59,26 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
|
||||
if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil {
|
||||
columnName = evolutionsEntries[0].ColumnName
|
||||
}
|
||||
rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name)
|
||||
if exists {
|
||||
return rawPath + " IS NOT NULL", nil
|
||||
members := physicalSemconvMembers(key)
|
||||
paths := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
paths = append(paths, fmt.Sprintf("%s.`%s`", columnName, member))
|
||||
}
|
||||
return rawPath + " IS NULL", nil
|
||||
if len(paths) == 1 {
|
||||
if exists {
|
||||
return paths[0] + " IS NOT NULL", nil
|
||||
}
|
||||
return paths[0] + " IS NULL", nil
|
||||
}
|
||||
guards := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
guards = append(guards, path+" IS NOT NULL")
|
||||
}
|
||||
rawPath := "(" + strings.Join(guards, " OR ") + ")"
|
||||
if exists {
|
||||
return rawPath, nil
|
||||
}
|
||||
return "NOT " + rawPath, nil
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumFixedString:
|
||||
if exists {
|
||||
@@ -88,8 +119,16 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
if key.Materialized {
|
||||
members := physicalSemconvMembers(key)
|
||||
operands := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
operands = append(operands, fmt.Sprintf("mapContains(%s, '%s')", column.Name, member))
|
||||
}
|
||||
leftOperand := strings.Join(operands, " OR ")
|
||||
if len(operands) > 1 {
|
||||
leftOperand = "(" + leftOperand + ")"
|
||||
}
|
||||
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
|
||||
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
}
|
||||
if exists {
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -38,6 +39,7 @@ type filterExpressionVisitor struct {
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
skipResourceFilter bool
|
||||
skipFullTextFilter bool
|
||||
exactSemconv bool
|
||||
variables map[string]qbtypes.VariableItem
|
||||
|
||||
keysWithWarnings map[string]bool
|
||||
@@ -58,6 +60,7 @@ type FilterExprVisitorOpts struct {
|
||||
FullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
SkipResourceFilter bool
|
||||
SkipFullTextFilter bool
|
||||
ExactSemconv bool
|
||||
Variables map[string]qbtypes.VariableItem
|
||||
StartNs uint64
|
||||
EndNs uint64
|
||||
@@ -75,6 +78,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
|
||||
fullTextColumn: opts.FullTextColumn,
|
||||
skipResourceFilter: opts.SkipResourceFilter,
|
||||
skipFullTextFilter: opts.SkipFullTextFilter,
|
||||
exactSemconv: opts.ExactSemconv,
|
||||
variables: opts.Variables,
|
||||
keysWithWarnings: make(map[string]bool),
|
||||
startNs: opts.StartNs,
|
||||
@@ -379,7 +383,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
// VisitComparison handles all comparison operators.
|
||||
func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
|
||||
key := v.Visit(ctx.Key()).(*telemetrytypes.TelemetryFieldKey)
|
||||
matching := MatchingFieldKeys(key, v.fieldKeys)
|
||||
matching := v.matchingFieldKeys(key)
|
||||
|
||||
// Handle EXISTS specially
|
||||
if ctx.EXISTS() != nil {
|
||||
@@ -730,7 +734,7 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
|
||||
conds, ok := v.buildConditions(key, MatchingFieldKeys(key, v.fieldKeys), operator, value)
|
||||
conds, ok := v.buildConditions(key, v.matchingFieldKeys(key), operator, value)
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -923,7 +927,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
|
||||
// buildConditions invokes the condition builder for a filter term, folding its
|
||||
// warnings/errors into visitor state; returns false if an error was recorded.
|
||||
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) ([]string, bool) {
|
||||
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder)
|
||||
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter, ExactSemconv: v.exactSemconv}, op, value, v.builder)
|
||||
if err != nil {
|
||||
_, _, _, _, errURL, _ := errors.Unwrapb(err)
|
||||
assignIfEmpty(&v.mainErrorURL, errURL)
|
||||
@@ -982,27 +986,116 @@ func assignIfEmpty(s *string, value string) {
|
||||
// MatchingFieldKeys returns the field keys from the map that match the given key,
|
||||
// honoring any context/data type the user specified.
|
||||
func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
fieldKeysForName := []*telemetrytypes.TelemetryFieldKey{}
|
||||
return matchingFieldKeys(field, fieldKeys, true)
|
||||
}
|
||||
|
||||
// match by name; keep items whose context and data type match (unspecified matches any)
|
||||
for _, item := range fieldKeys[field.Name] {
|
||||
if (field.FieldContext == telemetrytypes.FieldContextUnspecified || field.FieldContext == item.FieldContext) &&
|
||||
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || field.FieldDataType == item.FieldDataType) {
|
||||
fieldKeysForName = append(fieldKeysForName, item)
|
||||
// MatchingFieldKeysExact matches only the requested physical spelling.
|
||||
func MatchingFieldKeysExact(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return matchingFieldKeys(field, fieldKeys, false)
|
||||
}
|
||||
|
||||
func matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, resolveSemconv bool) []*telemetrytypes.TelemetryFieldKey {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: field.Signal,
|
||||
FieldContext: field.FieldContext,
|
||||
}
|
||||
members := []string{field.Name}
|
||||
if resolveSemconv {
|
||||
members = semconv.AttributeMembers(selector)
|
||||
}
|
||||
isFamily := len(members) > 1
|
||||
fieldKeysForName := make([]*telemetrytypes.TelemetryFieldKey, 0)
|
||||
indexByIdentity := make(map[string]int)
|
||||
|
||||
appendMatches := func(lookupName string, memberName string, contextAlreadyMatched bool) {
|
||||
for _, item := range fieldKeys[lookupName] {
|
||||
if !contextAlreadyMatched && field.FieldContext != telemetrytypes.FieldContextUnspecified && field.FieldContext != item.FieldContext {
|
||||
continue
|
||||
}
|
||||
if field.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && field.FieldDataType != item.FieldDataType {
|
||||
continue
|
||||
}
|
||||
|
||||
// A wildcard lookup may have found a same-named field in a scope where
|
||||
// this family does not apply. Keep exact names, but reject cross-member
|
||||
// matches outside the generated family scope.
|
||||
if resolveSemconv && memberName != field.Name {
|
||||
itemSelector := telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: item.Signal,
|
||||
FieldContext: item.FieldContext,
|
||||
}
|
||||
if !slices.Contains(semconv.AttributeMembers(itemSelector), memberName) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
physicalMembers := item.SemconvMembers
|
||||
if len(physicalMembers) == 0 {
|
||||
physicalMembers = []string{memberName}
|
||||
}
|
||||
identity := item.Signal.StringValue() + ";" + item.FieldContext.StringValue() + ";" + item.FieldDataType.StringValue()
|
||||
if isFamily {
|
||||
if index, found := indexByIdentity[identity]; found {
|
||||
for _, physicalMember := range physicalMembers {
|
||||
if !slices.Contains(fieldKeysForName[index].SemconvMembers, physicalMember) {
|
||||
fieldKeysForName[index].SemconvMembers = append(fieldKeysForName[index].SemconvMembers, physicalMember)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
indexByIdentity[identity] = len(fieldKeysForName)
|
||||
}
|
||||
resolved := *item
|
||||
// The requested spelling is the response identity. Field mappers use
|
||||
// it to resolve the available family members current-first.
|
||||
if isFamily {
|
||||
resolved.Name = field.Name
|
||||
resolved.SemconvMembers = slices.Clone(physicalMembers)
|
||||
} else if !resolveSemconv {
|
||||
resolved.SemconvMembers = []string{field.Name}
|
||||
}
|
||||
fieldKeysForName = append(fieldKeysForName, &resolved)
|
||||
}
|
||||
}
|
||||
|
||||
// A context may have been split off a name that legitimately contained it (e.g.
|
||||
// `attribute.key`); also look up the context-prefixed name so both readings resolve.
|
||||
// Members are current-first, so metadata from the current key wins when
|
||||
// both spellings describe the same signal/context/type.
|
||||
for _, member := range members {
|
||||
appendMatches(member, member, false)
|
||||
}
|
||||
|
||||
// A context may have been split off a name that legitimately contained it
|
||||
// (e.g. `attribute.key`); preserve that historical alternate reading for
|
||||
// every family member.
|
||||
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
contextPrefixedFieldName := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)
|
||||
for _, item := range fieldKeys[contextPrefixedFieldName] {
|
||||
// Context already matched via the lookup key; only data type needs checking.
|
||||
if field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || item.FieldDataType == field.FieldDataType {
|
||||
fieldKeysForName = append(fieldKeysForName, item)
|
||||
}
|
||||
for _, member := range members {
|
||||
appendMatches(fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), member), member, true)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldKeysForName
|
||||
}
|
||||
|
||||
func (v *filterExpressionVisitor) matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
if v.exactSemconv {
|
||||
return MatchingFieldKeysExact(field, v.fieldKeys)
|
||||
}
|
||||
return MatchingFieldKeys(field, v.fieldKeys)
|
||||
}
|
||||
|
||||
// ExactSemconvKeys returns copies pinned to their physical names, preventing a
|
||||
// field mapper from expanding a synthesized or metadata-free key into a family.
|
||||
func ExactSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
result := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
if key == nil {
|
||||
continue
|
||||
}
|
||||
resolved := *key
|
||||
resolved.SemconvMembers = []string{key.Name}
|
||||
result = append(result, &resolved)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestPrepareWhereClause_EmptyVariableList ensures PrepareWhereClause errors when a variable has an empty list value.
|
||||
@@ -685,6 +686,58 @@ func TestVisitKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchingFieldKeysResolvesSemconvFamily(t *testing.T) {
|
||||
current := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Description: "current metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
old := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Description: "old metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {current},
|
||||
old.Name: {old},
|
||||
}
|
||||
|
||||
for _, requestedName := range []string{current.Name, old.Name} {
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
requestedName,
|
||||
telemetrytypes.FieldContextResource,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
matches := MatchingFieldKeys(requested, fieldKeys)
|
||||
require.Len(t, matches, 1, "family lookup must return one field before its metadata is inspected")
|
||||
assert.Equal(t, requestedName, matches[0].Name)
|
||||
assert.Equal(t, "current metadata", matches[0].Description)
|
||||
assert.Equal(t, []string{current.Name, old.Name}, matches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
// A current-name query still resolves when metadata has seen only the old
|
||||
// spelling. The returned name remains the request identity.
|
||||
requested := telemetrytypes.NewTelemetryFieldKey(
|
||||
current.Name,
|
||||
telemetrytypes.FieldContextResource,
|
||||
telemetrytypes.FieldDataTypeString,
|
||||
)
|
||||
matches := MatchingFieldKeys(requested, map[string][]*telemetrytypes.TelemetryFieldKey{old.Name: {old}})
|
||||
require.Len(t, matches, 1, "family lookup must return one field before its metadata is inspected")
|
||||
assert.Equal(t, current.Name, matches[0].Name)
|
||||
assert.Equal(t, "old metadata", matches[0].Description)
|
||||
assert.Equal(t, []string{old.Name}, matches[0].SemconvMembers)
|
||||
|
||||
exactMatches := MatchingFieldKeysExact(requested, fieldKeys)
|
||||
require.Len(t, exactMatches, 1, "exact lookup must return one field before its metadata is inspected")
|
||||
assert.Equal(t, current.Name, exactMatches[0].Name)
|
||||
assert.Equal(t, []string{current.Name}, exactMatches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// TestVisitComparison
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
3
pkg/semconv/doc.go
Normal file
3
pkg/semconv/doc.go
Normal file
@@ -0,0 +1,3 @@
|
||||
// Package semconv resolves historical OpenTelemetry semantic-convention names
|
||||
// into logical field families.
|
||||
package semconv
|
||||
48
pkg/semconv/families_gen.go
Normal file
48
pkg/semconv/families_gen.go
Normal file
@@ -0,0 +1,48 @@
|
||||
// Code generated by scripts/semconv. DO NOT EDIT.
|
||||
|
||||
package semconv
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
var families = []Family{
|
||||
{
|
||||
Current: "container.cpu.usage",
|
||||
Old: []string{"container.cpu.utilization"},
|
||||
Kind: KindMetric,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "db.system.name",
|
||||
Old: []string{"db.system"},
|
||||
Kind: KindAttribute,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalLogs, telemetrytypes.SignalMetrics, telemetrytypes.SignalTraces},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "deployment.environment.name",
|
||||
Old: []string{"deployment.environment"},
|
||||
Kind: KindAttribute,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalLogs, telemetrytypes.SignalMetrics, telemetrytypes.SignalTraces},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "k8s.node.cpu.usage",
|
||||
Old: []string{"k8s.node.cpu.utilization"},
|
||||
Kind: KindMetric,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "k8s.pod.cpu.usage",
|
||||
Old: []string{"k8s.pod.cpu.utilization"},
|
||||
Kind: KindMetric,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
}
|
||||
265
pkg/semconv/semconv.go
Normal file
265
pkg/semconv/semconv.go
Normal file
@@ -0,0 +1,265 @@
|
||||
package semconv
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
//go:generate go run ../../scripts/semconv
|
||||
|
||||
// Kind identifies whether a family describes an attribute or a metric name.
|
||||
type Kind struct {
|
||||
valuer.String
|
||||
}
|
||||
|
||||
// Family is one logical telemetry field. Old is ordered from the most recent
|
||||
// predecessor to the oldest one and therefore also defines fallback order.
|
||||
type Family struct {
|
||||
Current string
|
||||
Old []string
|
||||
Kind Kind
|
||||
Contexts []telemetrytypes.FieldContext
|
||||
Signals []telemetrytypes.Signal
|
||||
ApplyToMetrics []string
|
||||
ValueMap map[string]string
|
||||
}
|
||||
|
||||
type metricSpelling uint8
|
||||
|
||||
const (
|
||||
metricSpellingDotted metricSpelling = iota
|
||||
metricSpellingNormalized
|
||||
)
|
||||
|
||||
var (
|
||||
KindAttribute = Kind{String: valuer.NewString("attribute")}
|
||||
KindMetric = Kind{String: valuer.NewString("metric")}
|
||||
)
|
||||
|
||||
var memberToFamilies, familyMembers = buildIndexes()
|
||||
|
||||
// Enum returns the acceptable values for Kind.
|
||||
func (Kind) Enum() []any {
|
||||
return []any{KindAttribute, KindMetric}
|
||||
}
|
||||
|
||||
// Lookup returns the enabled family containing selector.Name for kind. The
|
||||
// slices and map in the returned value reference immutable generated data and
|
||||
// must be treated as read-only. Use All when a mutable copy is required.
|
||||
func Lookup(kind Kind, selector telemetrytypes.FieldKeySelector) (Family, bool) {
|
||||
idx, ok := lookupIndex(kind, selector)
|
||||
if !ok {
|
||||
return Family{}, false
|
||||
}
|
||||
return families[idx], true
|
||||
}
|
||||
|
||||
// Members returns the current name first, followed by historical names in
|
||||
// fallback order. For enabled families the returned precomputed slice is
|
||||
// read-only. A name outside an enabled family is returned unchanged.
|
||||
func Members(kind Kind, selector telemetrytypes.FieldKeySelector) []string {
|
||||
idx, ok := lookupIndex(kind, selector)
|
||||
if !ok {
|
||||
return []string{selector.Name}
|
||||
}
|
||||
return familyMembers[idx]
|
||||
}
|
||||
|
||||
// AttributeMembers returns the physical attribute spellings that may represent
|
||||
// selector.Name. Metrics have used both dotted and normalized label layouts;
|
||||
// resource labels have additionally used a resource_ prefix. Keeping that
|
||||
// storage detail here prevents metrics readers from maintaining local
|
||||
// transition tables.
|
||||
func AttributeMembers(selector telemetrytypes.FieldKeySelector) []string {
|
||||
if selector.Signal != telemetrytypes.SignalMetrics {
|
||||
return Members(KindAttribute, selector)
|
||||
}
|
||||
|
||||
lookupSelector := selector
|
||||
lookupSelector.Name = strings.TrimPrefix(selector.Name, "resource_")
|
||||
family, style, ok := lookupMetricSpelling(KindAttribute, lookupSelector)
|
||||
if !ok {
|
||||
return []string{selector.Name}
|
||||
}
|
||||
|
||||
logicalMembers := familyMembers[family]
|
||||
result := make([]string, 0, len(logicalMembers)*4)
|
||||
for _, member := range logicalMembers {
|
||||
dotted := member
|
||||
normalized := normalizeMetricSpelling(member)
|
||||
variants := []string{dotted, normalized}
|
||||
if style == metricSpellingNormalized {
|
||||
variants[0], variants[1] = variants[1], variants[0]
|
||||
}
|
||||
|
||||
if selector.FieldContext == telemetrytypes.FieldContextResource ||
|
||||
selector.FieldContext == telemetrytypes.FieldContextUnspecified ||
|
||||
strings.HasPrefix(selector.Name, "resource_") {
|
||||
for _, variant := range variants {
|
||||
result = appendUniqueString(result, "resource_"+variant)
|
||||
}
|
||||
}
|
||||
for _, variant := range variants {
|
||||
result = appendUniqueString(result, variant)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// MetricNames returns the current and historical storage names for a metric.
|
||||
// The input's dotted or normalized style is preserved because both layouts are
|
||||
// valid metric identities and must not be mixed in one query.
|
||||
func MetricNames(name string) []string {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
}
|
||||
family, style, ok := lookupMetricSpelling(KindMetric, selector)
|
||||
if !ok {
|
||||
return []string{name}
|
||||
}
|
||||
|
||||
logicalMembers := familyMembers[family]
|
||||
result := make([]string, 0, len(logicalMembers))
|
||||
for _, member := range logicalMembers {
|
||||
if style == metricSpellingNormalized {
|
||||
member = normalizeMetricSpelling(member)
|
||||
}
|
||||
result = appendUniqueString(result, member)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// CurrentAttribute returns the canonical dotted name for an attribute
|
||||
// spelling, or selector.Name if no enabled family matches.
|
||||
func CurrentAttribute(selector telemetrytypes.FieldKeySelector) string {
|
||||
if selector.Signal != telemetrytypes.SignalMetrics {
|
||||
return Current(KindAttribute, selector)
|
||||
}
|
||||
|
||||
lookupSelector := selector
|
||||
lookupSelector.Name = strings.TrimPrefix(selector.Name, "resource_")
|
||||
family, _, ok := lookupMetricSpelling(KindAttribute, lookupSelector)
|
||||
if !ok {
|
||||
return selector.Name
|
||||
}
|
||||
return families[family].Current
|
||||
}
|
||||
|
||||
// Current returns the current name for selector.Name, or the input name when
|
||||
// it does not belong to an enabled family.
|
||||
func Current(kind Kind, selector telemetrytypes.FieldKeySelector) string {
|
||||
idx, ok := lookupIndex(kind, selector)
|
||||
if !ok {
|
||||
return selector.Name
|
||||
}
|
||||
return families[idx].Current
|
||||
}
|
||||
|
||||
// All returns a defensive copy of every enabled family.
|
||||
func All() []Family {
|
||||
result := make([]Family, len(families))
|
||||
for i := range families {
|
||||
result[i] = cloneFamily(families[i])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func buildIndexes() (map[string][]int, [][]string) {
|
||||
index := make(map[string][]int)
|
||||
members := make([][]string, len(families))
|
||||
for i, family := range families {
|
||||
members[i] = make([]string, 0, len(family.Old)+1)
|
||||
members[i] = append(members[i], family.Current)
|
||||
members[i] = append(members[i], family.Old...)
|
||||
index[family.Current] = append(index[family.Current], i)
|
||||
for _, old := range family.Old {
|
||||
index[old] = append(index[old], i)
|
||||
}
|
||||
}
|
||||
return index, members
|
||||
}
|
||||
|
||||
func lookupIndex(kind Kind, selector telemetrytypes.FieldKeySelector) (int, bool) {
|
||||
for _, idx := range memberToFamilies[selector.Name] {
|
||||
if matchesSelector(families[idx], kind, selector) {
|
||||
return idx, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func lookupMetricSpelling(kind Kind, selector telemetrytypes.FieldKeySelector) (int, metricSpelling, bool) {
|
||||
if idx, ok := lookupIndex(kind, selector); ok {
|
||||
return idx, metricSpellingDotted, true
|
||||
}
|
||||
|
||||
for idx, family := range families {
|
||||
if !matchesSelector(family, kind, selector) {
|
||||
continue
|
||||
}
|
||||
for _, member := range familyMembers[idx] {
|
||||
if normalizeMetricSpelling(member) == selector.Name {
|
||||
return idx, metricSpellingNormalized, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, metricSpellingDotted, false
|
||||
}
|
||||
|
||||
func normalizeMetricSpelling(name string) string {
|
||||
return strings.ReplaceAll(name, ".", "_")
|
||||
}
|
||||
|
||||
func appendUniqueString(values []string, value string) []string {
|
||||
if value == "" || slices.Contains(values, value) {
|
||||
return values
|
||||
}
|
||||
return append(values, value)
|
||||
}
|
||||
|
||||
func matchesSelector(family Family, kind Kind, selector telemetrytypes.FieldKeySelector) bool {
|
||||
if family.Kind != kind {
|
||||
return false
|
||||
}
|
||||
|
||||
if selector.Signal != telemetrytypes.SignalUnspecified && len(family.Signals) > 0 {
|
||||
if !slices.Contains(family.Signals, selector.Signal) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && len(family.Contexts) > 0 {
|
||||
if !slices.Contains(family.Contexts, selector.FieldContext) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
if selector.Signal == telemetrytypes.SignalMetrics && len(family.ApplyToMetrics) > 0 {
|
||||
if selector.MetricContext == nil {
|
||||
return false
|
||||
}
|
||||
return slices.Contains(family.ApplyToMetrics, selector.MetricContext.MetricName)
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func cloneFamily(family Family) Family {
|
||||
valueMap := family.ValueMap
|
||||
family.Old = slices.Clone(family.Old)
|
||||
family.Contexts = slices.Clone(family.Contexts)
|
||||
family.Signals = slices.Clone(family.Signals)
|
||||
family.ApplyToMetrics = slices.Clone(family.ApplyToMetrics)
|
||||
if valueMap != nil {
|
||||
family.ValueMap = make(map[string]string, len(valueMap))
|
||||
for old, current := range valueMap {
|
||||
family.ValueMap[old] = current
|
||||
}
|
||||
}
|
||||
return family
|
||||
}
|
||||
114
pkg/semconv/semconv_test.go
Normal file
114
pkg/semconv/semconv_test.go
Normal file
@@ -0,0 +1,114 @@
|
||||
package semconv
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMembersReturnsCurrentBeforeHistoricalName(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{"deployment.environment.name", "deployment.environment"},
|
||||
Members(KindAttribute, selector),
|
||||
"members should use current-first fallback order",
|
||||
)
|
||||
}
|
||||
|
||||
func TestCurrentReturnsCanonicalName(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
assert.Equal(t,
|
||||
"deployment.environment.name",
|
||||
Current(KindAttribute, selector),
|
||||
"historical name should resolve to the current family name",
|
||||
)
|
||||
}
|
||||
|
||||
func TestMembersReturnsInputWhenKindDoesNotMatch(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
}
|
||||
|
||||
assert.Equal(t,
|
||||
[]string{"deployment.environment"},
|
||||
Members(KindMetric, selector),
|
||||
"an attribute family must not match a metric-name lookup",
|
||||
)
|
||||
}
|
||||
|
||||
func TestAllReturnsDefensiveCopies(t *testing.T) {
|
||||
first := All()
|
||||
require.NotEmpty(t, first, "generated families must not be empty")
|
||||
first[0].Old[0] = "mutated"
|
||||
|
||||
second := All()
|
||||
assert.NotEqual(t, "mutated", second[0].Old[0], "All must not expose mutable generated data")
|
||||
}
|
||||
|
||||
func TestAttributeMembersIncludesMetricResourceStorageSpellings(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{
|
||||
"resource_db.system.name", "resource_db_system_name", "db.system.name", "db_system_name",
|
||||
"resource_db.system", "resource_db_system", "db.system", "db_system",
|
||||
}, AttributeMembers(selector), "resource metric attributes should cover every historical storage layout")
|
||||
}
|
||||
|
||||
func TestCurrentAttributeResolvesNormalizedMetricResourceSpelling(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "resource_db_system",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
assert.Equal(t, "db.system.name", CurrentAttribute(selector), "normalized resource spelling should resolve to the dotted current name")
|
||||
}
|
||||
|
||||
func TestAttributeMembersPreservesNormalizedMetricPointStyle(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "db_system",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{
|
||||
"db_system_name", "db.system.name", "db_system", "db.system",
|
||||
}, AttributeMembers(selector), "normalized point attribute should remain the preferred storage spelling")
|
||||
}
|
||||
|
||||
func TestMetricNamesPreservesDottedStyle(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
[]string{"k8s.pod.cpu.usage", "k8s.pod.cpu.utilization"},
|
||||
MetricNames("k8s.pod.cpu.usage"),
|
||||
"dotted metric input should produce dotted family names",
|
||||
)
|
||||
}
|
||||
|
||||
func TestMetricNamesPreservesNormalizedStyle(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
[]string{"k8s_pod_cpu_usage", "k8s_pod_cpu_utilization"},
|
||||
MetricNames("k8s_pod_cpu_utilization"),
|
||||
"normalized metric input should produce normalized family names",
|
||||
)
|
||||
}
|
||||
|
||||
func TestMetricNamesReturnsUnknownNameUnchanged(t *testing.T) {
|
||||
assert.Equal(t, []string{"custom_metric"}, MetricNames("custom_metric"), "unknown metrics should not be expanded")
|
||||
}
|
||||
@@ -235,6 +235,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewFillDashboardSpecCollectionsFactory(sqlstore, dashboardStore),
|
||||
sqlmigration.NewScrubEmailChannelTransportFactory(sqlstore),
|
||||
sqlmigration.NewAddDashboardTuplesFactory(sqlstore),
|
||||
sqlmigration.NewMigrateDeploymentEnvironmentQuickFilterFactory(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
const deploymentEnvironmentCurrent = "deployment.environment.name"
|
||||
|
||||
type migrateDeploymentEnvironmentQuickFilter struct {
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
type semconvQuickFilterRow struct {
|
||||
bun.BaseModel `bun:"table:quick_filter"`
|
||||
|
||||
ID string `bun:"id"`
|
||||
Filter string `bun:"filter"`
|
||||
}
|
||||
|
||||
func NewMigrateDeploymentEnvironmentQuickFilterFactory() factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("migrate_semconv_quick_filter"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, _ Config) (SQLMigration, error) {
|
||||
return &migrateDeploymentEnvironmentQuickFilter{logger: settings.Logger}, nil
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func (migration *migrateDeploymentEnvironmentQuickFilter) Register(migrations *migrate.Migrations) error {
|
||||
return migrations.Register(migration.Up, migration.Down)
|
||||
}
|
||||
|
||||
func deploymentEnvironmentOld() string {
|
||||
members := semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: deploymentEnvironmentCurrent,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
})
|
||||
if len(members) < 2 {
|
||||
return deploymentEnvironmentCurrent
|
||||
}
|
||||
return members[1]
|
||||
}
|
||||
|
||||
func rewriteQuickFilterSemconv(filterJSON, from, to string) (string, bool, error) {
|
||||
var filters []map[string]any
|
||||
if err := json.Unmarshal([]byte(filterJSON), &filters); err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
|
||||
changed := false
|
||||
for _, filter := range filters {
|
||||
if key, ok := filter["key"].(string); ok && key == from {
|
||||
filter["key"] = to
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return filterJSON, false, nil
|
||||
}
|
||||
|
||||
rewritten, err := json.Marshal(filters)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return string(rewritten), true, nil
|
||||
}
|
||||
|
||||
func (migration *migrateDeploymentEnvironmentQuickFilter) migrate(ctx context.Context, db *bun.DB, from, to string) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
rows := make([]*semconvQuickFilterRow, 0)
|
||||
if err := tx.NewSelect().
|
||||
Model(&rows).
|
||||
Where("signal IN (?)", bun.In([]string{"traces", "api_monitoring", "exceptions"})).
|
||||
Scan(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
rewritten, changed, err := rewriteQuickFilterSemconv(row.Filter, from, to)
|
||||
if err != nil {
|
||||
// Quick filters are user-editable. One malformed legacy row must not
|
||||
// prevent the application from starting or block every other org's
|
||||
// migration.
|
||||
if migration.logger != nil {
|
||||
migration.logger.WarnContext(ctx, "skipping quick filter with unreadable filter JSON",
|
||||
slog.String("quick_filter_id", row.ID), slog.Any("error", err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
if _, err := tx.NewUpdate().
|
||||
Model((*semconvQuickFilterRow)(nil)).
|
||||
Set("filter = ?", rewritten).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Where("id = ?", row.ID).
|
||||
Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (migration *migrateDeploymentEnvironmentQuickFilter) Up(ctx context.Context, db *bun.DB) error {
|
||||
return migration.migrate(ctx, db, deploymentEnvironmentOld(), deploymentEnvironmentCurrent)
|
||||
}
|
||||
|
||||
func (migration *migrateDeploymentEnvironmentQuickFilter) Down(ctx context.Context, db *bun.DB) error {
|
||||
return migration.migrate(ctx, db, deploymentEnvironmentCurrent, deploymentEnvironmentOld())
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestRewriteQuickFilterSemconv(t *testing.T) {
|
||||
oldName := deploymentEnvironmentOld()
|
||||
input := `[{"key":"service.name","dataType":"string","type":"resource"},{"key":"` + oldName + `","dataType":"string","type":"resource","custom":true}]`
|
||||
|
||||
rewritten, changed, err := rewriteQuickFilterSemconv(input, oldName, deploymentEnvironmentCurrent)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, changed)
|
||||
|
||||
var filters []map[string]any
|
||||
require.NoError(t, json.Unmarshal([]byte(rewritten), &filters))
|
||||
assert.Equal(t, "service.name", filters[0]["key"])
|
||||
assert.Equal(t, deploymentEnvironmentCurrent, filters[1]["key"])
|
||||
assert.Equal(t, true, filters[1]["custom"], "unknown filter properties must be preserved")
|
||||
|
||||
restored, changed, err := rewriteQuickFilterSemconv(rewritten, deploymentEnvironmentCurrent, oldName)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, changed)
|
||||
require.NoError(t, json.Unmarshal([]byte(restored), &filters))
|
||||
assert.Equal(t, oldName, filters[1]["key"])
|
||||
}
|
||||
|
||||
func TestRewriteQuickFilterSemconvNoop(t *testing.T) {
|
||||
input := `[{"key":"service.name","dataType":"string","type":"resource"}]`
|
||||
rewritten, changed, err := rewriteQuickFilterSemconv(input, deploymentEnvironmentOld(), deploymentEnvironmentCurrent)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, changed)
|
||||
assert.Equal(t, input, rewritten)
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
@@ -31,6 +32,15 @@ const (
|
||||
OthersMultiTemporality = `IF(LOWER(temporality) LIKE LOWER('delta'), %s, %s) AS per_series_value`
|
||||
)
|
||||
|
||||
func metricNameValues(name string) []any {
|
||||
members := semconv.MetricNames(name)
|
||||
values := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, member)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
type StatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
@@ -341,7 +351,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LTE("unix_milli", end),
|
||||
)
|
||||
@@ -385,7 +395,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
sb.In("metric_name", metricNameValues(agg.MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -427,7 +437,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
sb.In("metric_name", metricNameValues(agg.MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -505,7 +515,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -560,7 +570,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
}
|
||||
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LTE("unix_milli", end),
|
||||
)
|
||||
@@ -638,7 +648,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -679,7 +689,7 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
baseSb.Where(
|
||||
baseSb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
baseSb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
baseSb.GTE("unix_milli", start),
|
||||
baseSb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -770,7 +780,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
|
||||
@@ -13,9 +13,21 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMetricNameValuesResolveFamily(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
[]any{"k8s.pod.cpu.usage", "k8s.pod.cpu.utilization"},
|
||||
metricNameValues("k8s.pod.cpu.usage"),
|
||||
)
|
||||
assert.Equal(t,
|
||||
[]any{"container_cpu_usage", "container_cpu_utilization"},
|
||||
metricNameValues("container_cpu_utilization"),
|
||||
)
|
||||
}
|
||||
|
||||
func TestStatementBuilder(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
@@ -44,6 +44,73 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
|
||||
return fmt.Sprintf(`%%%s%%`, key.Name)
|
||||
}
|
||||
|
||||
func memberKey(key *telemetrytypes.TelemetryFieldKey, name string) *telemetrytypes.TelemetryFieldKey {
|
||||
member := *key
|
||||
member.Name = name
|
||||
return &member
|
||||
}
|
||||
|
||||
func keyIndexCondition(sb *sqlbuilder.SelectBuilder, column string, key *telemetrytypes.TelemetryFieldKey, members []string) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
conditions = append(conditions, sb.Like(column, keyIndexFilter(memberKey(key, member))))
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
|
||||
func valueIndexCondition(
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
column string,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
members []string,
|
||||
op qbtypes.FilterOperator,
|
||||
value any,
|
||||
caseInsensitive bool,
|
||||
) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
patterns := valueForIndexFilter(op, memberKey(key, member), value)
|
||||
switch values := patterns.(type) {
|
||||
case []string:
|
||||
for _, pattern := range values {
|
||||
conditions = append(conditions, sb.Like(column, pattern))
|
||||
}
|
||||
default:
|
||||
if caseInsensitive {
|
||||
conditions = append(conditions, sb.ILike(column, values))
|
||||
} else {
|
||||
conditions = append(conditions, sb.Like(column, values))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
|
||||
func memberPresenceCondition(sb *sqlbuilder.SelectBuilder, column string, members []string, exists bool) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
field := fmt.Sprintf("simpleJSONHas(%s, '%s')", column, member)
|
||||
if exists {
|
||||
conditions = append(conditions, sb.E(field, true))
|
||||
} else {
|
||||
conditions = append(conditions, sb.NE(field, true))
|
||||
}
|
||||
}
|
||||
if exists {
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
return sb.And(conditions...)
|
||||
}
|
||||
|
||||
// SkipResourceFilter is not applicable here: the fingerprint table only stores resource attributes.
|
||||
func (b *defaultConditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
@@ -115,8 +182,10 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
// as we have not changed the resource column in the resource fingerprint table.
|
||||
column := columns[0]
|
||||
|
||||
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
|
||||
valueForIndexFilter := valueForIndexFilter(op, key, value)
|
||||
members := resourceSemconvMembers(key)
|
||||
isFamily := len(members) > 1
|
||||
keyIdxFilter := keyIndexCondition(sb, column.Name, key, members)
|
||||
singleValueIndexFilter := valueForIndexFilter(op, memberKey(key, members[0]), value)
|
||||
|
||||
fieldName, err := b.fm.FieldFor(ctx, valuer.UUID{}, startNs, endNs, key)
|
||||
if err != nil {
|
||||
@@ -128,12 +197,15 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.E(fieldName, formattedValue),
|
||||
keyIdxFilter,
|
||||
sb.Like(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, false),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
if isFamily {
|
||||
return sb.NE(fieldName, formattedValue), nil
|
||||
}
|
||||
return sb.And(
|
||||
sb.NE(fieldName, formattedValue),
|
||||
sb.NotLike(column.Name, valueForIndexFilter),
|
||||
sb.NotLike(column.Name, singleValueIndexFilter),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.And(sb.GT(fieldName, formattedValue), keyIdxFilter), nil
|
||||
@@ -148,7 +220,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.ILike(fieldName, formattedValue),
|
||||
keyIdxFilter,
|
||||
sb.ILike(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, true),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotLike, qbtypes.FilterOperatorNotILike:
|
||||
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
|
||||
@@ -185,13 +257,11 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
mainCondition := sb.Or(inConditions...)
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.Like(column.Name, v))
|
||||
}
|
||||
}
|
||||
mainCondition = sb.And(mainCondition, keyIdxFilter, sb.Or(valConditions...))
|
||||
mainCondition = sb.And(
|
||||
mainCondition,
|
||||
keyIdxFilter,
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, false),
|
||||
)
|
||||
|
||||
return mainCondition, nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -204,8 +274,11 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
mainCondition := sb.And(notInConditions...)
|
||||
if isFamily {
|
||||
return mainCondition, nil
|
||||
}
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := valueForIndexFilter.([]string); ok {
|
||||
if valuesForIndexFilter, ok := singleValueIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.NotLike(column.Name, v))
|
||||
}
|
||||
@@ -215,13 +288,11 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
|
||||
case qbtypes.FilterOperatorExists:
|
||||
return sb.And(
|
||||
sb.E(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
|
||||
memberPresenceCondition(sb, column.Name, members, true),
|
||||
keyIdxFilter,
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotExists:
|
||||
return sb.And(
|
||||
sb.NE(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
|
||||
), nil
|
||||
return memberPresenceCondition(sb, column.Name, members, false), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
return sb.And(
|
||||
@@ -237,7 +308,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.ILike(fieldName, fmt.Sprintf(`%%%s%%`, formattedValue)),
|
||||
keyIdxFilter,
|
||||
sb.ILike(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, key, members, op, value, true),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
|
||||
|
||||
@@ -198,6 +198,75 @@ func TestConditionBuilder(t *testing.T) {
|
||||
expected: "match(simpleJSONExtractString(labels, 'k8s.namespace.name'), ?) AND labels LIKE ?",
|
||||
expectedArgs: []any{"ban.*", "%k8s.namespace.name%"},
|
||||
},
|
||||
{
|
||||
name: "semantic convention family equality uses current-first fallback",
|
||||
key: &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
},
|
||||
op: qbtypes.FilterOperatorEqual,
|
||||
value: "production",
|
||||
expected: "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), '')) = ? AND (labels LIKE ? OR labels LIKE ?) AND (labels LIKE ? OR labels LIKE ?)",
|
||||
expectedArgs: []any{
|
||||
"production",
|
||||
"%deployment.environment.name%",
|
||||
"%deployment.environment%",
|
||||
`%deployment.environment.name":"production%`,
|
||||
`%deployment.environment":"production%`,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "old semantic convention request uses only current metadata member",
|
||||
key: &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name"},
|
||||
},
|
||||
op: qbtypes.FilterOperatorEqual,
|
||||
value: "production",
|
||||
expected: "simpleJSONExtractString(labels, 'deployment.environment.name') = ? AND labels LIKE ? AND labels LIKE ?",
|
||||
expectedArgs: []any{"production", "%deployment.environment.name%", `%deployment.environment.name":"production%`},
|
||||
},
|
||||
{
|
||||
name: "semantic convention family negative filter does not reject fallback rows",
|
||||
key: &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
},
|
||||
op: qbtypes.FilterOperatorNotEqual,
|
||||
value: "staging",
|
||||
expected: "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), '')) <> ?",
|
||||
expectedArgs: []any{"staging"},
|
||||
},
|
||||
{
|
||||
name: "semantic convention family exists checks every member",
|
||||
key: &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
},
|
||||
op: qbtypes.FilterOperatorExists,
|
||||
expected: "(simpleJSONHas(labels, 'deployment.environment.name') = ? OR simpleJSONHas(labels, 'deployment.environment') = ?) AND (labels LIKE ? OR labels LIKE ?)",
|
||||
expectedArgs: []any{true, true, "%deployment.environment.name%", "%deployment.environment%"},
|
||||
},
|
||||
{
|
||||
name: "semantic convention family not exists checks every member",
|
||||
key: &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
},
|
||||
op: qbtypes.FilterOperatorNotExists,
|
||||
expected: "(simpleJSONHas(labels, 'deployment.environment.name') <> ? AND simpleJSONHas(labels, 'deployment.environment') <> ?)",
|
||||
expectedArgs: []any{true, true},
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
|
||||
@@ -3,8 +3,10 @@ package resourcefilter
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -32,6 +34,20 @@ func NewFieldMapper() *defaultFieldMapper {
|
||||
return &defaultFieldMapper{}
|
||||
}
|
||||
|
||||
func resourceSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
})
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
@@ -66,7 +82,15 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
return "", err
|
||||
}
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
members := resourceSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(simpleJSONExtractString(%s, '%s'), '')", columns[0].Name, member))
|
||||
}
|
||||
return "COALESCE(" + strings.Join(values, ", ") + ")", nil
|
||||
}
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, members[0]), nil
|
||||
}
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
package telemetrymetadata
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
type BackwardCompatibleKeyMap map[string]string
|
||||
|
||||
var (
|
||||
TracesBackwardCompatKeys = BackwardCompatibleKeyMap{
|
||||
"net.peer.name": "server.address",
|
||||
"server.address": "net.peer.name",
|
||||
"http.url": "url.full",
|
||||
"url.full": "http.url",
|
||||
}
|
||||
|
||||
// LogsBackwardCompatKeys contains bidirectional mappings for logs.
|
||||
// Currently empty, can be extended in the future.
|
||||
LogsBackwardCompatKeys = BackwardCompatibleKeyMap{}
|
||||
|
||||
// MetricsBackwardCompatKeys contains bidirectional mappings for metrics.
|
||||
// Currently empty, can be extended in the future.
|
||||
MetricsBackwardCompatKeys = BackwardCompatibleKeyMap{}
|
||||
)
|
||||
|
||||
func GetBackwardCompatKeysForSignal(signal telemetrytypes.Signal) BackwardCompatibleKeyMap {
|
||||
switch signal {
|
||||
case telemetrytypes.SignalTraces:
|
||||
return TracesBackwardCompatKeys
|
||||
case telemetrytypes.SignalLogs:
|
||||
return LogsBackwardCompatKeys
|
||||
case telemetrytypes.SignalMetrics:
|
||||
return MetricsBackwardCompatKeys
|
||||
default:
|
||||
return BackwardCompatibleKeyMap{}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/audittelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/logstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metertelemetryschema"
|
||||
@@ -151,6 +153,129 @@ func (t *telemetryMetaStore) tracesTblStatementToFieldKeys(ctx context.Context)
|
||||
return materialisedKeys, nil
|
||||
}
|
||||
|
||||
func attributeSemconvMembers(name string, signal telemetrytypes.Signal, fieldContext telemetrytypes.FieldContext) []string {
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: name,
|
||||
Signal: signal,
|
||||
FieldContext: fieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
func traceSemconvMembers(name string, fieldContext telemetrytypes.FieldContext) []string {
|
||||
return attributeSemconvMembers(name, telemetrytypes.SignalTraces, fieldContext)
|
||||
}
|
||||
|
||||
func semconvDuplicateFactor(signal telemetrytypes.Signal) int {
|
||||
factor := 1
|
||||
for _, family := range semconv.All() {
|
||||
if family.Kind != semconv.KindAttribute {
|
||||
continue
|
||||
}
|
||||
if _, ok := semconv.Lookup(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: family.Current,
|
||||
Signal: signal,
|
||||
}); ok {
|
||||
factor = max(factor, len(attributeSemconvMembers(family.Current, signal, telemetrytypes.FieldContextUnspecified)))
|
||||
}
|
||||
}
|
||||
return factor
|
||||
}
|
||||
|
||||
func traceSemconvDuplicateFactor() int {
|
||||
return semconvDuplicateFactor(telemetrytypes.SignalTraces)
|
||||
}
|
||||
|
||||
func inStrings(sb *sqlbuilder.SelectBuilder, column string, values []string) string {
|
||||
args := make([]any, 0, len(values))
|
||||
for _, value := range values {
|
||||
args = append(args, value)
|
||||
}
|
||||
return sb.In(column, args...)
|
||||
}
|
||||
|
||||
// canonicalizeTraceSemconvKeys presents one current-name key for each family.
|
||||
// If metadata contains both spellings, metadata attached to the current name
|
||||
// wins; otherwise the old entry is copied under the current response name.
|
||||
func canonicalizeTraceSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return canonicalizeSemconvKeys(keys, telemetrytypes.SignalTraces)
|
||||
}
|
||||
|
||||
// canonicalizeSemconvKeys presents one current-name key for each enabled
|
||||
// attribute family and records only the physical spellings actually present in
|
||||
// metadata. That lets field mappers retain their optimized single-key SQL for
|
||||
// homogeneous ranges while mixed ranges coalesce current-first.
|
||||
func canonicalizeSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey, signal telemetrytypes.Signal) []*telemetrytypes.TelemetryFieldKey {
|
||||
result := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
indexByIdentity := make(map[string]int)
|
||||
currentSourceByIdentity := make(map[string]bool)
|
||||
|
||||
for _, key := range keys {
|
||||
if key.Signal != signal {
|
||||
result = append(result, key)
|
||||
continue
|
||||
}
|
||||
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: signal,
|
||||
FieldContext: key.FieldContext,
|
||||
}
|
||||
members := semconv.AttributeMembers(selector)
|
||||
current := semconv.CurrentAttribute(selector)
|
||||
if len(members) == 1 && current == key.Name {
|
||||
result = append(result, key)
|
||||
continue
|
||||
}
|
||||
|
||||
resolved := *key
|
||||
resolved.Name = current
|
||||
resolved.SemconvMembers = []string{key.Name}
|
||||
identity := resolved.Name + ";" + resolved.Signal.StringValue() + ";" + resolved.FieldContext.StringValue() + ";" + resolved.FieldDataType.StringValue()
|
||||
physicalName := strings.TrimPrefix(key.Name, "resource_")
|
||||
fromCurrent := physicalName == current || physicalName == strings.ReplaceAll(current, ".", "_")
|
||||
|
||||
if index, found := indexByIdentity[identity]; found {
|
||||
physicalMembers := result[index].SemconvMembers
|
||||
if fromCurrent && !currentSourceByIdentity[identity] {
|
||||
result[index] = &resolved
|
||||
currentSourceByIdentity[identity] = true
|
||||
}
|
||||
for _, member := range physicalMembers {
|
||||
if !slices.Contains(result[index].SemconvMembers, member) {
|
||||
result[index].SemconvMembers = append(result[index].SemconvMembers, member)
|
||||
}
|
||||
}
|
||||
if !slices.Contains(result[index].SemconvMembers, key.Name) {
|
||||
result[index].SemconvMembers = append(result[index].SemconvMembers, key.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
indexByIdentity[identity] = len(result)
|
||||
currentSourceByIdentity[identity] = fromCurrent
|
||||
result = append(result, &resolved)
|
||||
}
|
||||
|
||||
for _, key := range result {
|
||||
if len(key.SemconvMembers) < 2 {
|
||||
continue
|
||||
}
|
||||
present := make(map[string]bool, len(key.SemconvMembers))
|
||||
for _, member := range key.SemconvMembers {
|
||||
present[member] = true
|
||||
}
|
||||
ordered := make([]string, 0, len(key.SemconvMembers))
|
||||
for _, member := range attributeSemconvMembers(key.Name, signal, key.FieldContext) {
|
||||
if present[member] {
|
||||
ordered = append(ordered, member)
|
||||
}
|
||||
}
|
||||
key.SemconvMembers = ordered
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// getTracesKeys returns the keys from the spans that match the field selection criteria.
|
||||
func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
@@ -202,10 +327,23 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
|
||||
// key part of the selector
|
||||
fieldKeyConds := []string{}
|
||||
members := traceSemconvMembers(fieldKeySelector.Name, fieldKeySelector.FieldContext)
|
||||
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("tagKey", fieldKeySelector.Name))
|
||||
if len(members) == 1 {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("tagKey", members[0]))
|
||||
} else {
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldKeyConds = append(fieldKeyConds, sb.In("tagKey", memberValues...))
|
||||
}
|
||||
} else {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
|
||||
memberConditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConditions = append(memberConditions, sb.ILike("tagKey", "%"+escapeForLike(member)+"%"))
|
||||
}
|
||||
fieldKeyConds = append(fieldKeyConds, sb.Or(memberConditions...))
|
||||
}
|
||||
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
@@ -238,8 +376,10 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
mainSb.From(mainSb.BuilderAs(sb, "sub_query"))
|
||||
mainSb.GroupBy("tag_key", "tag_type", "tag_data_type")
|
||||
mainSb.OrderBy("priority")
|
||||
// query one extra to check if we hit the limit
|
||||
mainSb.Limit(limit + 1)
|
||||
// Family members collapse after the database query. In the worst case each
|
||||
// logical key occupies one row per family member, so fetch enough physical
|
||||
// rows to return the requested logical page, plus one to detect truncation.
|
||||
mainSb.Limit(limit*traceSemconvDuplicateFactor() + 1)
|
||||
|
||||
query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
@@ -249,14 +389,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
}
|
||||
defer rows.Close()
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
rowCount := 0
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
@@ -285,8 +418,11 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTracesKeys.Error())
|
||||
}
|
||||
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
keys = canonicalizeTraceSemconvKeys(keys)
|
||||
complete := len(keys) <= limit
|
||||
if !complete {
|
||||
keys = keys[:limit]
|
||||
}
|
||||
|
||||
staticKeys := []string{"isRoot", "isEntryPoint"}
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
|
||||
@@ -460,10 +596,19 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
continue
|
||||
}
|
||||
fieldKeyConds := []string{}
|
||||
members := attributeSemconvMembers(sel.Name, telemetrytypes.SignalLogs, fieldContext)
|
||||
if sel.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("name", sel.Name))
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldKeyConds = append(fieldKeyConds, sb.In("name", memberValues...))
|
||||
} else {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.ILike("name", "%"+escapeForLike(sel.Name)+"%"))
|
||||
memberConds := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConds = append(memberConds, sb.ILike("name", "%"+escapeForLike(member)+"%"))
|
||||
}
|
||||
fieldKeyConds = append(fieldKeyConds, sb.Or(memberConds...))
|
||||
}
|
||||
if sel.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("datatype", sel.FieldDataType.TagDataType()))
|
||||
@@ -556,6 +701,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalLogs)
|
||||
mainQuery := fmt.Sprintf(`
|
||||
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
|
||||
FROM (
|
||||
@@ -564,7 +710,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
GROUP BY tag_key, tag_type, tag_data_type
|
||||
ORDER BY priority
|
||||
LIMIT %d
|
||||
`, strings.Join(queries, " UNION ALL "), limit+1)
|
||||
`, strings.Join(queries, " UNION ALL "), dbLimit+1)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
@@ -585,7 +731,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -633,7 +779,16 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
}
|
||||
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit
|
||||
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalLogs)
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
complete = false
|
||||
}
|
||||
mapOfKeys = make(map[string]*telemetrytypes.TelemetryFieldKey, len(keys))
|
||||
for _, key := range keys {
|
||||
mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
staticKeys := []string{}
|
||||
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
|
||||
@@ -941,10 +1096,19 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
conds := []string{}
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
fieldConds := []string{}
|
||||
members := attributeSemconvMembers(fieldKeySelector.Name, telemetrytypes.SignalMetrics, fieldKeySelector.FieldContext)
|
||||
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("attr_name", memberValues...))
|
||||
} else {
|
||||
fieldConds = append(fieldConds, sb.ILike("attr_name", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
|
||||
memberConds := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConds = append(memberConds, sb.ILike("attr_name", "%"+escapeForLike(member)+"%"))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.Or(memberConds...))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.NotLike("attr_name", "\\_\\_%"))
|
||||
|
||||
@@ -960,7 +1124,12 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
|
||||
if fieldKeySelector.MetricContext != nil {
|
||||
if fieldKeySelector.MetricContext.MetricName != "" {
|
||||
fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
|
||||
metricNames := semconv.MetricNames(fieldKeySelector.MetricContext.MetricName)
|
||||
metricValues := make([]any, 0, len(metricNames))
|
||||
for _, metricName := range metricNames {
|
||||
metricValues = append(metricValues, metricName)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("metric_name", metricValues...))
|
||||
}
|
||||
if fieldKeySelector.MetricContext.MetricNamespace != "" {
|
||||
fieldConds = append(fieldConds, sb.Like("metric_name", escapeForLike(fieldKeySelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -982,7 +1151,8 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
mainSb.GroupBy("name", "field_context", "field_data_type")
|
||||
mainSb.OrderBy("priority")
|
||||
// query one extra to check if we hit the limit
|
||||
mainSb.Limit(limit + 1)
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
|
||||
mainSb.Limit(dbLimit + 1)
|
||||
|
||||
query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
@@ -997,7 +1167,7 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1022,7 +1192,12 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit
|
||||
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalMetrics)
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
complete = false
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
}
|
||||
@@ -1044,16 +1219,30 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
var limit int
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
fieldConds := []string{}
|
||||
members := attributeSemconvMembers(fieldKeySelector.Name, telemetrytypes.SignalMetrics, fieldKeySelector.FieldContext)
|
||||
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("attr_name", memberValues...))
|
||||
} else {
|
||||
fieldConds = append(fieldConds, sb.Like("attr_name", "%"+fieldKeySelector.Name+"%"))
|
||||
memberConds := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConds = append(memberConds, sb.Like("attr_name", "%"+member+"%"))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.Or(memberConds...))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.NotLike("attr_name", "\\_\\_%"))
|
||||
|
||||
if fieldKeySelector.MetricContext != nil {
|
||||
if fieldKeySelector.MetricContext.MetricName != "" {
|
||||
fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
|
||||
metricNames := semconv.MetricNames(fieldKeySelector.MetricContext.MetricName)
|
||||
metricValues := make([]any, 0, len(metricNames))
|
||||
for _, metricName := range metricNames {
|
||||
metricValues = append(metricValues, metricName)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("metric_name", metricValues...))
|
||||
}
|
||||
if fieldKeySelector.MetricContext.MetricNamespace != "" {
|
||||
fieldConds = append(fieldConds, sb.Like("metric_name", escapeForLike(fieldKeySelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -1068,7 +1257,8 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
sb.Limit(limit)
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
|
||||
sb.Limit(dbLimit + 1)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
@@ -1082,7 +1272,7 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1091,9 +1281,14 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
|
||||
}
|
||||
fieldContext := telemetrytypes.FieldContextAttribute
|
||||
if strings.HasPrefix(name, "resource_") {
|
||||
fieldContext = telemetrytypes.FieldContextResource
|
||||
}
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: fieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1102,46 +1297,17 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit
|
||||
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalMetrics)
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
complete = false
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
|
||||
}
|
||||
|
||||
// applyBackwardCompatibleKeys adds backward compatible key aliases to the map.
|
||||
func applyBackwardCompatibleKeys(mapOfKeys map[string][]*telemetrytypes.TelemetryFieldKey) {
|
||||
// Get backward compatible keys for all signals
|
||||
backwardCompatKeysBySignal := map[telemetrytypes.Signal]BackwardCompatibleKeyMap{
|
||||
telemetrytypes.SignalTraces: GetBackwardCompatKeysForSignal(telemetrytypes.SignalTraces),
|
||||
telemetrytypes.SignalLogs: GetBackwardCompatKeysForSignal(telemetrytypes.SignalLogs),
|
||||
telemetrytypes.SignalMetrics: GetBackwardCompatKeysForSignal(telemetrytypes.SignalMetrics),
|
||||
}
|
||||
|
||||
// Iterate over existing keys and add aliases if they exist in backward compat mapping
|
||||
for srcKey, srcKeys := range mapOfKeys {
|
||||
for _, srcKeyEntry := range srcKeys {
|
||||
backwardCompatKeys := backwardCompatKeysBySignal[srcKeyEntry.Signal]
|
||||
if backwardCompatKeys == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if aliasKey, ok := backwardCompatKeys[srcKey]; ok {
|
||||
if _, aliasExists := mapOfKeys[aliasKey]; !aliasExists {
|
||||
aliasKeyEntry := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: aliasKey,
|
||||
Signal: srcKeyEntry.Signal,
|
||||
FieldContext: srcKeyEntry.FieldContext,
|
||||
FieldDataType: srcKeyEntry.FieldDataType,
|
||||
}
|
||||
mapOfKeys[aliasKey] = []*telemetrytypes.TelemetryFieldKey{aliasKeyEntry}
|
||||
}
|
||||
// Found the alias for this signal, no need to check other entries
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
if len(selectors) == 0 {
|
||||
return keys
|
||||
@@ -1272,7 +1438,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
|
||||
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
|
||||
}
|
||||
|
||||
applyBackwardCompatibleKeys(mapOfKeys)
|
||||
mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
|
||||
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
|
||||
@@ -1353,7 +1518,6 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
|
||||
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
|
||||
}
|
||||
|
||||
applyBackwardCompatibleKeys(mapOfKeys)
|
||||
mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
|
||||
if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
|
||||
mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
|
||||
@@ -1367,7 +1531,18 @@ func (t *telemetryMetaStore) GetKey(ctx context.Context, orgID valuer.UUID, fiel
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keys[fieldKeySelector.Name], nil
|
||||
members := semconv.Members(semconv.KindAttribute, *fieldKeySelector)
|
||||
resolved := make([]*telemetrytypes.TelemetryFieldKey, 0)
|
||||
seen := make(map[*telemetrytypes.TelemetryFieldKey]bool)
|
||||
for _, member := range members {
|
||||
for _, key := range keys[member] {
|
||||
if !seen[key] {
|
||||
resolved = append(resolved, key)
|
||||
seen[key] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *telemetrytypes.FieldValueSelector) ([]string, bool, error) {
|
||||
@@ -1486,7 +1661,6 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, orgID valuer.
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
}
|
||||
// query one extra to check if we hit the limit
|
||||
sb.Limit(limit + 1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -1542,7 +1716,16 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
members := traceSemconvMembers(fieldValueSelector.Name, fieldValueSelector.FieldContext)
|
||||
if len(members) == 1 {
|
||||
sb.Where(sb.E("tag_key", members[0]))
|
||||
} else {
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
sb.Where(sb.In("tag_key", memberValues...))
|
||||
}
|
||||
}
|
||||
|
||||
// now look at the field context
|
||||
@@ -1632,7 +1815,12 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
members := attributeSemconvMembers(fieldValueSelector.Name, telemetrytypes.SignalLogs, fieldValueSelector.FieldContext)
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
sb.Where(sb.In("tag_key", memberValues...))
|
||||
}
|
||||
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
@@ -1830,7 +2018,9 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID val
|
||||
From(t.metricsDBName + "." + t.metricsFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("attr_name", fieldValueSelector.Name))
|
||||
sb.Where(inStrings(sb, "attr_name", attributeSemconvMembers(
|
||||
fieldValueSelector.Name, telemetrytypes.SignalMetrics, fieldValueSelector.FieldContext,
|
||||
)))
|
||||
}
|
||||
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
@@ -1842,7 +2032,7 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID val
|
||||
}
|
||||
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
|
||||
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
|
||||
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
|
||||
}
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
|
||||
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -1976,7 +2166,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValuesForTable(ctx context.C
|
||||
From(t.metricsDBName + "." + tableName)
|
||||
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
|
||||
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
|
||||
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
|
||||
}
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
|
||||
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -2039,12 +2229,14 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
From(t.meterDBName + "." + t.meterFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("attr.1", fieldValueSelector.Name))
|
||||
sb.Where(inStrings(sb, "attr.1", attributeSemconvMembers(
|
||||
fieldValueSelector.Name, telemetrytypes.SignalMetrics, fieldValueSelector.FieldContext,
|
||||
)))
|
||||
}
|
||||
sb.Where(sb.NotLike("attr.1", "\\_\\_%"))
|
||||
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
|
||||
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
|
||||
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
|
||||
}
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
|
||||
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -2063,8 +2255,10 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
}
|
||||
// query one extra to check if we hit the limit
|
||||
sb.Limit(limit + 1)
|
||||
// A value can be present under several physical spellings; over-fetch and
|
||||
// de-duplicate after scanning so one family cannot consume the result limit.
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
|
||||
sb.Limit(dbLimit + 1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
@@ -2074,11 +2268,13 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
defer rows.Close()
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
seen := make(map[string]bool)
|
||||
rowCount := 0
|
||||
uniqueCount := 0
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -2087,12 +2283,16 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterValues.Error())
|
||||
}
|
||||
if len(attribute) > 1 {
|
||||
values.StringValues = append(values.StringValues, attribute[1])
|
||||
if !seen[attribute[1]] && uniqueCount < limit {
|
||||
values.StringValues = append(values.StringValues, attribute[1])
|
||||
seen[attribute[1]] = true
|
||||
uniqueCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit && uniqueCount < limit
|
||||
return values, complete, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -83,3 +84,121 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
t.Errorf("there were unfulfilled expectations: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalizeTraceSemconvKeys(t *testing.T) {
|
||||
oldResource := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Description: "old resource metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
currentResource := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
Description: "current resource metadata",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
oldAttribute := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
oldLogResource := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
result := canonicalizeTraceSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
|
||||
oldResource,
|
||||
currentResource,
|
||||
oldAttribute,
|
||||
oldLogResource,
|
||||
})
|
||||
|
||||
require.Len(t, result, 3)
|
||||
assert.Equal(t, "deployment.environment.name", result[0].Name)
|
||||
assert.Equal(t, "current resource metadata", result[0].Description)
|
||||
assert.Equal(t, []string{"deployment.environment.name", "deployment.environment"}, result[0].SemconvMembers)
|
||||
assert.Equal(t, "deployment.environment.name", result[1].Name)
|
||||
assert.Equal(t, telemetrytypes.FieldContextAttribute, result[1].FieldContext)
|
||||
assert.Equal(t, []string{"deployment.environment"}, result[1].SemconvMembers)
|
||||
assert.Equal(t, "deployment.environment", result[2].Name, "phase 1 must not rewrite raw log metadata")
|
||||
}
|
||||
|
||||
func TestCanonicalizeLogAndMetricSemconvKeys(t *testing.T) {
|
||||
logKeys := canonicalizeSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
Name: "db.system",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
}, telemetrytypes.SignalLogs)
|
||||
require.Len(t, logKeys, 1)
|
||||
assert.Equal(t, "db.system.name", logKeys[0].Name)
|
||||
assert.Equal(t, []string{"db.system.name", "db.system"}, logKeys[0].SemconvMembers)
|
||||
|
||||
metricKeys := canonicalizeSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
Name: "resource_db_system",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
}, telemetrytypes.SignalMetrics)
|
||||
require.Len(t, metricKeys, 1)
|
||||
assert.Equal(t, "db.system.name", metricKeys[0].Name)
|
||||
assert.Equal(t, []string{"db.system.name", "resource_db_system"}, metricKeys[0].SemconvMembers)
|
||||
}
|
||||
|
||||
func TestGetSpanFieldValuesMergesSemconvFamily(t *testing.T) {
|
||||
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, ®exMatcher{})
|
||||
mock := mockTelemetryStore.Mock()
|
||||
|
||||
metadata := NewTelemetryMetaStore(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockTelemetryStore,
|
||||
flaggertest.New(t),
|
||||
)
|
||||
|
||||
mock.ExpectQuery(`SELECT DISTINCT string_value, number_value FROM signoz_traces\.distributed_tag_attributes_v2 WHERE tag_key IN \(\?, \?\) AND tag_type = \? AND tag_data_type = \? LIMIT \?`).
|
||||
WithArgs("deployment.environment.name", "deployment.environment", "resource", "string", 51).
|
||||
WillReturnRows(cmock.NewRows([]cmock.ColumnType{
|
||||
{Name: "string_value", Type: "String"},
|
||||
{Name: "number_value", Type: "Float64"},
|
||||
}, [][]any{
|
||||
{"production", float64(0)},
|
||||
{"staging", float64(0)},
|
||||
{"production", float64(0)},
|
||||
}))
|
||||
|
||||
values, complete, err := metadata.GetAllValues(context.Background(), valuer.UUID{}, &telemetrytypes.FieldValueSelector{
|
||||
FieldKeySelector: &telemetrytypes.FieldKeySelector{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Name: "deployment.environment",
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, complete)
|
||||
assert.Equal(t, []string{"production", "staging"}, values.StringValues)
|
||||
assert.NoError(t, mock.ExpectationsWereMet(), "all expected metadata queries should be executed")
|
||||
}
|
||||
|
||||
@@ -89,10 +89,12 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
|
||||
|
||||
// Create and store the TelemetryFieldKey
|
||||
field := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: fieldName,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
Materialized: true,
|
||||
Name: fieldName,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
Materialized: true,
|
||||
MaterializedColumnName: columnName,
|
||||
MaterializedSemconv: strings.Count(defaultExprStr, "['") > 1,
|
||||
}
|
||||
|
||||
v.Fields = append(v.Fields, field)
|
||||
|
||||
@@ -5,8 +5,25 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExtractFieldKeyPreservesHistoricalMaterializedColumnName(t *testing.T) {
|
||||
statement := `CREATE TABLE signoz_traces.signoz_index_v3
|
||||
(
|
||||
attributes_string Map(LowCardinality(String), String),
|
||||
` + "`attribute_string_db$$system`" + ` LowCardinality(String)
|
||||
DEFAULT if(mapContains(attributes_string, 'db.system.name'), attributes_string['db.system.name'], attributes_string['db.system'])
|
||||
) ENGINE = MergeTree ORDER BY tuple()`
|
||||
keys, err := ExtractFieldKeysFromTblStatement(statement)
|
||||
require.NoError(t, err, "table statement should parse")
|
||||
require.Len(t, keys, 1, "table statement should contain one materialized key")
|
||||
assert.Equal(t, "db.system.name", keys[0].Name)
|
||||
assert.Equal(t, "attribute_string_db$$system", keys[0].MaterializedColumnName)
|
||||
assert.True(t, keys[0].MaterializedSemconv)
|
||||
}
|
||||
|
||||
func TestExtractFieldKeysFromTblStatement(t *testing.T) {
|
||||
|
||||
var statement = `CREATE TABLE signoz_logs.logs_v2
|
||||
|
||||
@@ -453,6 +453,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) ([]string, []string, error) {
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
if options.ExactSemconv {
|
||||
matches = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
|
||||
}
|
||||
skipResourceFilter := options.SkipResourceFilter
|
||||
|
||||
// search() resolves its own (optional) scope; handle it before key resolution.
|
||||
@@ -499,6 +502,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
|
||||
}
|
||||
}
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.ExactSemconvKeys(keys)
|
||||
}
|
||||
|
||||
if skipResourceFilter && !synthesized {
|
||||
filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -67,6 +68,20 @@ type fieldMapper struct {
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
func logSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
func NewFieldMapper(fl flagger.Flagger) qbtypes.FieldMapper {
|
||||
return &fieldMapper{fl: fl}
|
||||
}
|
||||
@@ -141,8 +156,20 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
|
||||
members := logSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
values := make([]string, 0, len(members))
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s.`%s`::String, '')", columnName, member))
|
||||
guards = append(guards, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
existExpr = append(existExpr, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, members[0]))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, members[0]))
|
||||
}
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if key.Name == messageSubField {
|
||||
exprs = append(exprs, messageSubColumn)
|
||||
@@ -181,13 +208,32 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
members := logSemconvMembers(key)
|
||||
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExpr = append(existExpr, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
|
||||
} else if len(members) > 1 {
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
guards = append(guards, fmt.Sprintf("mapContains(%s, '%s')", columnName, member))
|
||||
}
|
||||
if valueType.GetType() == schema.ColumnTypeEnumString {
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s['%s'], '')", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
} else {
|
||||
branches := make([]string, 0, len(members)*2+1)
|
||||
for i, member := range members {
|
||||
branches = append(branches, guards[i], fmt.Sprintf("%s['%s']", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
|
||||
}
|
||||
existExpr = append(existExpr, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, members[0]))
|
||||
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, members[0]))
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
|
||||
@@ -2,6 +2,7 @@ package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +15,32 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFieldForSemconvFamily(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper(flaggertest.New(t)).(*fieldMapper)
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := fm.FieldFor(ctx, valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "COALESCE(NULLIF(attributes_string['db.system.name'], ''), NULLIF(attributes_string['db.system'], ''))", expression)
|
||||
assert.Less(t, strings.Index(expression, "db.system.name"), strings.Index(expression, "db.system']"), "current spelling must win")
|
||||
|
||||
exists, err := fm.existsExpressionFor(ctx, valuer.UUID{}, 0, 0, key, true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "(mapContains(attributes_string, 'db.system.name') OR mapContains(attributes_string, 'db.system'))", exists)
|
||||
|
||||
exact := *key
|
||||
exact.SemconvMembers = []string{"db.system"}
|
||||
expression, err = fm.FieldFor(ctx, valuer.UUID{}, 0, 0, &exact)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "attributes_string['db.system']", expression)
|
||||
}
|
||||
|
||||
func TestGetColumn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -135,10 +136,22 @@ func (c *conditionBuilder) conditionFor(
|
||||
return "true", nil
|
||||
}
|
||||
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
|
||||
members := metricAttributeMembers(key)
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
guards = append(guards, fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", member))
|
||||
}
|
||||
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
|
||||
guard := strings.Join(guards, " OR ")
|
||||
if len(guards) > 1 {
|
||||
guard = "(" + guard + ")"
|
||||
}
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return guard, nil
|
||||
}
|
||||
if len(guards) == 1 {
|
||||
return "not " + guard, nil
|
||||
}
|
||||
return "NOT " + guard, nil
|
||||
}
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
|
||||
}
|
||||
@@ -151,7 +164,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
_ qbtypes.ConditionBuilderOptions,
|
||||
options qbtypes.ConditionBuilderOptions,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
@@ -162,7 +175,15 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
requestedKey := *key
|
||||
if requestedKey.Signal == telemetrytypes.SignalUnspecified {
|
||||
requestedKey.Signal = telemetrytypes.SignalMetrics
|
||||
}
|
||||
key = &requestedKey
|
||||
keys := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
|
||||
}
|
||||
var warnings []string
|
||||
if len(keys) == 0 {
|
||||
if _, isColumn := timeSeriesV4Columns[key.Name]; isColumn {
|
||||
@@ -180,6 +201,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.ExactSemconvKeys(keys)
|
||||
}
|
||||
|
||||
conds := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
|
||||
@@ -390,3 +390,45 @@ func TestConditionForKeyNotInMetadata(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionForSemconvMetricLabels(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
requested := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
current := requested
|
||||
legacyNormalized := requested
|
||||
legacyNormalized.Name = "resource_db_system"
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {¤t},
|
||||
legacyNormalized.Name: {&legacyNormalized},
|
||||
}
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conditions, warnings, err := conditionBuilder.ConditionFor(
|
||||
ctx, valuer.UUID{}, 0, 0, &requested, fieldKeys, qbtypes.ConditionBuilderOptions{},
|
||||
qbtypes.FilterOperatorEqual, "postgresql", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, warnings)
|
||||
sb.Where(conditions...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, query, "COALESCE(NULLIF(JSONExtractString(labels, 'db.system.name'), ''), NULLIF(JSONExtractString(labels, 'resource_db_system'), '')) = ?")
|
||||
assert.Equal(t, []any{"postgresql"}, args)
|
||||
|
||||
sb = sqlbuilder.NewSelectBuilder()
|
||||
conditions, _, err = conditionBuilder.ConditionFor(
|
||||
ctx, valuer.UUID{}, 0, 0, &requested, fieldKeys, qbtypes.ConditionBuilderOptions{ExactSemconv: true},
|
||||
qbtypes.FilterOperatorExists, nil, sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
query, _ = sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, query, "has(JSONExtractKeys(labels), 'db.system.name')")
|
||||
assert.NotContains(t, query, "resource_db_system")
|
||||
}
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -38,6 +40,23 @@ var (
|
||||
|
||||
type fieldMapper struct{}
|
||||
|
||||
func metricAttributeMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource &&
|
||||
key.FieldContext != telemetrytypes.FieldContextScope &&
|
||||
key.FieldContext != telemetrytypes.FieldContextAttribute &&
|
||||
key.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
// CandidateKeys returns nil: metrics has no attribute-map fallback, so a context-missing
|
||||
// key stays unresolved and the caller errors.
|
||||
func (m *fieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, _ *telemetrytypes.TelemetryFieldKey, _ any, _ map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
@@ -80,19 +99,30 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endN
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return metricLabelExpression(columns[0].Name, metricAttributeMembers(key)), nil
|
||||
case telemetrytypes.FieldContextMetric:
|
||||
return columns[0].Name, nil
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
if slices.Contains(IntrinsicFields, key.Name) {
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return metricLabelExpression(columns[0].Name, metricAttributeMembers(key)), nil
|
||||
}
|
||||
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
func metricLabelExpression(columnName string, members []string) string {
|
||||
if len(members) == 1 {
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columnName, members[0])
|
||||
}
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(JSONExtractString(%s, '%s'), '')", columnName, member))
|
||||
}
|
||||
return "COALESCE(" + strings.Join(values, ", ") + ")"
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package metricstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
@@ -12,6 +13,32 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMetricLabelSemconvSpellings(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
for _, member := range []string{
|
||||
"resource_db.system.name", "resource_db_system_name", "db.system.name", "db_system_name",
|
||||
"resource_db.system", "resource_db_system", "db.system", "db_system",
|
||||
} {
|
||||
assert.Contains(t, expression, "'"+member+"'")
|
||||
}
|
||||
assert.Less(t, strings.Index(expression, "resource_db.system.name"), strings.Index(expression, "resource_db.system'"))
|
||||
|
||||
exact := *key
|
||||
exact.SemconvMembers = []string{"resource_db_system"}
|
||||
expression, err = fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, &exact)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "JSONExtractString(labels, 'resource_db_system')", expression)
|
||||
}
|
||||
|
||||
func TestGetColumn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -154,6 +154,15 @@ func (c *conditionBuilder) conditionFor(
|
||||
// in the query builder, `exists` and `not exists` are used for
|
||||
// key membership checks, so depending on the column type, the condition changes
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
// A semantic-convention family is represented by one current-first value
|
||||
// expression, but presence still has to inspect every physical member. In
|
||||
// particular, using ExistsExpression below with the requested key would add
|
||||
// a mapContains check for only that spelling and reject fallback-only rows.
|
||||
if isTraceSemconvFamily(key) {
|
||||
if fm, ok := c.fm.(*fieldMapper); ok {
|
||||
return fm.existsExpressionFor(ctx, orgID, startNs, endNs, key, operator == qbtypes.FilterOperatorExists)
|
||||
}
|
||||
}
|
||||
columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -211,6 +220,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
if options.ExactSemconv {
|
||||
matches = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
|
||||
}
|
||||
skipResourceFilter := options.SkipResourceFilter
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, matches)
|
||||
@@ -256,6 +268,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
synthesized = true
|
||||
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
|
||||
}
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.ExactSemconvKeys(keys)
|
||||
}
|
||||
|
||||
// When a resource sub-query already covers the term, drop resource keys from the main
|
||||
// query. Synthesized keys are exempt: the sub-query skips keys absent from metadata.
|
||||
|
||||
@@ -210,6 +210,32 @@ func TestConditionFor(t *testing.T) {
|
||||
expectedSQL: "NOT mapContains(attributes_string, 'user.id')",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Equal operator - semantic convention family",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorEqual,
|
||||
value: "production",
|
||||
expectedSQL: "(COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], '')) = ? AND ((mapContains(attributes_string, 'deployment.environment.name') OR mapContains(attributes_string, 'deployment.environment'))))",
|
||||
expectedArgs: []any{"production"},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Not Exists operator - semantic convention family",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name", "deployment.environment"},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorNotExists,
|
||||
expectedSQL: "NOT (((mapContains(attributes_string, 'deployment.environment.name') OR mapContains(attributes_string, 'deployment.environment'))))",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Exists operator - json field",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -167,6 +168,32 @@ func NewFieldMapper() *fieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func traceSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
func isTraceSemconvFamily(key *telemetrytypes.TelemetryFieldKey) bool {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return false
|
||||
}
|
||||
_, ok := semconv.Lookup(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
return ok
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
@@ -291,10 +318,25 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
|
||||
}
|
||||
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
|
||||
// once clickHouse dependency is updated, we need to check if we can remove it.
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
|
||||
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
|
||||
members := traceSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
values := make([]string, 0, len(members))
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
// The String cast is required because ClickHouse does not allow
|
||||
// Variant/Dynamic values in GROUP BY.
|
||||
value := fmt.Sprintf("%s.`%s`::String", columnName, member)
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s, '')", value))
|
||||
guards = append(guards, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
|
||||
// once ClickHouse is updated, check whether this cast can be removed.
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, members[0]))
|
||||
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, members[0]))
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumUInt64,
|
||||
schema.ColumnTypeEnumUInt32,
|
||||
@@ -319,13 +361,32 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
members := traceSemconvMembers(key)
|
||||
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
|
||||
} else if len(members) > 1 {
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
guards = append(guards, fmt.Sprintf("mapContains(%s, '%s')", columnName, member))
|
||||
}
|
||||
if valueType.GetType() == schema.ColumnTypeEnumString {
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s['%s'], '')", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
} else {
|
||||
branches := make([]string, 0, len(members)*2)
|
||||
for i, member := range members {
|
||||
branches = append(branches, guards[i], fmt.Sprintf("%s['%s']", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
|
||||
}
|
||||
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
|
||||
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, members[0]))
|
||||
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, members[0]))
|
||||
}
|
||||
default:
|
||||
return nil, nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)
|
||||
@@ -529,6 +590,25 @@ func (m *fieldMapper) existsExpressionFor(
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
exists bool,
|
||||
) (string, error) {
|
||||
if isTraceSemconvFamily(key) {
|
||||
_, existExprs, _, err := m.resolveColumnExprs(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(existExprs) == 0 {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no existence expression found for field %s", key.Name)
|
||||
}
|
||||
parts := make([]string, 0, len(existExprs))
|
||||
for _, expression := range existExprs {
|
||||
parts = append(parts, "("+expression+")")
|
||||
}
|
||||
combined := strings.Join(parts, " OR ")
|
||||
if exists {
|
||||
return combined, nil
|
||||
}
|
||||
return "NOT (" + combined + ")", nil
|
||||
}
|
||||
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -80,7 +82,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
Materialized: true,
|
||||
Evolutions: mockEvolution,
|
||||
},
|
||||
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
|
||||
expectedResult: "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(resources_string['deployment.environment'], '')), NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -120,6 +122,51 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldForSemconvFamily(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
start := uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano())
|
||||
end := uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano())
|
||||
|
||||
for _, requestedName := range []string{"deployment.environment.name", "deployment.environment"} {
|
||||
attributeKey := telemetrytypes.TelemetryFieldKey{
|
||||
Name: requestedName,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
attributeExpression, err := fm.FieldFor(ctx, valuer.UUID{}, start, end, &attributeKey)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t,
|
||||
"COALESCE(NULLIF(attributes_string['deployment.environment.name'], ''), NULLIF(attributes_string['deployment.environment'], ''))",
|
||||
attributeExpression,
|
||||
)
|
||||
|
||||
resourceKey := telemetrytypes.TelemetryFieldKey{
|
||||
Name: requestedName,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
Evolutions: MockEvolutionData(time.Date(2024, 6, 2, 0, 0, 0, 0, time.UTC)),
|
||||
}
|
||||
resourceExpression, err := fm.FieldFor(ctx, valuer.UUID{}, start, end, &resourceKey)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t,
|
||||
"multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(resources_string['deployment.environment'], '')), NULL)",
|
||||
resourceExpression,
|
||||
)
|
||||
}
|
||||
|
||||
oldRequestWithCurrentOnlyMetadata := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
SemconvMembers: []string{"deployment.environment.name"},
|
||||
}
|
||||
expression, err := fm.FieldFor(ctx, valuer.UUID{}, start, end, &oldRequestWithCurrentOnlyMetadata)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "attributes_string['deployment.environment.name']", expression)
|
||||
}
|
||||
|
||||
func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
releaseTime := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
@@ -176,7 +223,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
},
|
||||
tsStart: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2025, 7, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedResult: "resource.`deployment.environment`::String",
|
||||
expectedResult: "COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, ''))",
|
||||
},
|
||||
{
|
||||
name: "Window straddles release - materialized resource",
|
||||
@@ -189,7 +236,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
|
||||
expectedResult: "multiIf((resource.`deployment.environment.name` IS NOT NULL OR resource.`deployment.environment` IS NOT NULL), COALESCE(NULLIF(resource.`deployment.environment.name`::String, ''), NULLIF(resource.`deployment.environment`::String, '')), (mapContains(resources_string, 'deployment.environment.name') OR mapContains(resources_string, 'deployment.environment')), COALESCE(NULLIF(resources_string['deployment.environment.name'], ''), NULLIF(resources_string['deployment.environment'], '')), NULL)",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -303,3 +350,27 @@ func TestColumnExpressionForTimestampAttributeCollision(t *testing.T) {
|
||||
assert.Contains(t, result, "attributes_number['timestamp']")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDBSystemFamilyUsesSemconvAwareMaterializedColumn(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
MaterializedColumnName: "attribute_string_db$$system",
|
||||
MaterializedSemconv: true,
|
||||
SemconvMembers: []string{"db.system.name", "db.system"},
|
||||
}
|
||||
|
||||
expression, err := fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "`attribute_string_db$$system`", expression)
|
||||
|
||||
exists, err := querybuilder.ExistsExpression(
|
||||
[]*schema.Column{indexV3Columns["attributes_string"]}, key, 0, 0, expression, true,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "`attribute_string_db$$system_exists`", exists)
|
||||
}
|
||||
|
||||
@@ -45,6 +45,10 @@ type ConditionBuilder interface {
|
||||
type ConditionBuilderOptions struct {
|
||||
// SkipResourceFilter drops the resource context from the candidate set.
|
||||
SkipResourceFilter bool
|
||||
// ExactSemconv disables semantic-convention family expansion. It is an
|
||||
// internal escape hatch for diagnostics and migrations that must address one
|
||||
// physical spelling only; public query APIs continue to resolve families.
|
||||
ExactSemconv bool
|
||||
}
|
||||
type AggExprRewriter interface {
|
||||
// Rewrite rewrites the aggregation expression to be used in the query.
|
||||
|
||||
@@ -141,7 +141,7 @@ func NewSignalFilterFromStorableQuickFilter(storableQuickFilter *StorableQuickFi
|
||||
func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
|
||||
tracesFilters := []map[string]interface{}{
|
||||
{"key": "duration_nano", "dataType": "float64", "type": "tag"},
|
||||
{"key": "deployment.environment", "dataType": "string", "type": "resource"},
|
||||
{"key": "deployment.environment.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "hasError", "dataType": "bool", "type": "tag"},
|
||||
{"key": "service.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "name", "dataType": "string", "type": "tag"},
|
||||
@@ -166,13 +166,13 @@ func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
|
||||
}
|
||||
|
||||
apiMonitoringFilters := []map[string]interface{}{
|
||||
{"key": "deployment.environment", "dataType": "string", "type": "resource"},
|
||||
{"key": "deployment.environment.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "service.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "rpc.method", "dataType": "string", "type": "tag"},
|
||||
}
|
||||
|
||||
exceptionsFilters := []map[string]interface{}{
|
||||
{"key": "deployment.environment", "dataType": "string", "type": "resource"},
|
||||
{"key": "deployment.environment.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "service.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "host.name", "dataType": "string", "type": "resource"},
|
||||
{"key": "k8s.cluster.name", "dataType": "string", "type": "resource"},
|
||||
|
||||
37
pkg/types/quickfiltertypes/filter_test.go
Normal file
37
pkg/types/quickfiltertypes/filter_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package quickfiltertypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDefaultTraceQuickFiltersUseCurrentEnvironmentName(t *testing.T) {
|
||||
filters, err := NewDefaultQuickFilter(valuer.GenerateUUID())
|
||||
require.NoError(t, err)
|
||||
|
||||
traceSignals := map[string]bool{
|
||||
SignalTraces.StringValue(): true,
|
||||
SignalApiMonitoring.StringValue(): true,
|
||||
SignalExceptions.StringValue(): true,
|
||||
}
|
||||
for _, filter := range filters {
|
||||
if !traceSignals[filter.Signal.StringValue()] {
|
||||
continue
|
||||
}
|
||||
var keys []v3.AttributeKey
|
||||
require.NoError(t, json.Unmarshal([]byte(filter.Filter), &keys))
|
||||
found := false
|
||||
for _, key := range keys {
|
||||
if key.Key == "deployment.environment.name" {
|
||||
found = true
|
||||
}
|
||||
assert.NotEqual(t, "deployment.environment", key.Key)
|
||||
}
|
||||
assert.True(t, found, "missing environment quick filter for %s", filter.Signal.StringValue())
|
||||
}
|
||||
}
|
||||
@@ -46,8 +46,16 @@ type TelemetryFieldKey struct {
|
||||
JSONPlan JSONAccessPlan `json:"-"`
|
||||
Indexes []TelemetryFieldKeySkipIndex `json:"-"`
|
||||
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
|
||||
// MaterializedColumnName preserves the physical column when its DEFAULT
|
||||
// expression resolves a newer semantic-convention key than the historical
|
||||
// column identifier (for example db.system.name in db$$system).
|
||||
MaterializedColumnName string `json:"-"`
|
||||
// MaterializedSemconv is true when the column's DEFAULT expression already
|
||||
// coalesces every enabled family member and is therefore safe for a family query.
|
||||
MaterializedSemconv bool `json:"-"`
|
||||
|
||||
Evolutions []*EvolutionEntry `json:"-"`
|
||||
Evolutions []*EvolutionEntry `json:"-"`
|
||||
SemconvMembers []string `json:"-"`
|
||||
}
|
||||
|
||||
func (f *TelemetryFieldKey) KeyNameContainsArray() bool {
|
||||
@@ -126,8 +134,11 @@ func (f *TelemetryFieldKey) OverrideMetadataFrom(src *TelemetryFieldKey) {
|
||||
f.FieldDataType = src.FieldDataType
|
||||
f.Indexes = src.Indexes
|
||||
f.Materialized = src.Materialized
|
||||
f.MaterializedColumnName = src.MaterializedColumnName
|
||||
f.MaterializedSemconv = src.MaterializedSemconv
|
||||
f.JSONPlan = src.JSONPlan
|
||||
f.Evolutions = src.Evolutions
|
||||
f.SemconvMembers = src.SemconvMembers
|
||||
}
|
||||
|
||||
func (f *TelemetryFieldKey) Equal(key *TelemetryFieldKey) bool {
|
||||
@@ -202,6 +213,9 @@ func TelemetryFieldKeyToText(key *TelemetryFieldKey) string {
|
||||
}
|
||||
|
||||
func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
|
||||
if key.MaterializedColumnName != "" {
|
||||
return fmt.Sprintf("`%s`", key.MaterializedColumnName)
|
||||
}
|
||||
return fmt.Sprintf("`%s_%s_%s`",
|
||||
key.FieldContext.String,
|
||||
fieldDataTypes[key.FieldDataType.StringValue()].StringValue(),
|
||||
@@ -210,6 +224,9 @@ func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
|
||||
}
|
||||
|
||||
func FieldKeyToMaterializedColumnNameForExists(key *TelemetryFieldKey) string {
|
||||
if key.MaterializedColumnName != "" {
|
||||
return fmt.Sprintf("`%s_exists`", key.MaterializedColumnName)
|
||||
}
|
||||
return fmt.Sprintf("`%s_%s_%s_exists`",
|
||||
key.FieldContext.String,
|
||||
fieldDataTypes[key.FieldDataType.StringValue()].StringValue(),
|
||||
|
||||
721
scripts/semconv/generate.go
Normal file
721
scripts/semconv/generate.go
Normal file
@@ -0,0 +1,721 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"go/format"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const (
|
||||
kindAttribute = "attribute"
|
||||
kindMetric = "metric"
|
||||
)
|
||||
|
||||
type stringListFlag []string
|
||||
|
||||
func (f *stringListFlag) String() string { return strings.Join(*f, ",") }
|
||||
func (f *stringListFlag) Set(value string) error {
|
||||
*f = append(*f, value)
|
||||
return nil
|
||||
}
|
||||
|
||||
type schemaFile struct {
|
||||
FileFormat string `yaml:"file_format"`
|
||||
SchemaURL string `yaml:"schema_url"`
|
||||
Versions map[string]schemaVersion `yaml:"versions"`
|
||||
}
|
||||
|
||||
type schemaVersion struct {
|
||||
All changeSection `yaml:"all"`
|
||||
Resources changeSection `yaml:"resources"`
|
||||
Spans changeSection `yaml:"spans"`
|
||||
Logs changeSection `yaml:"logs"`
|
||||
Metrics changeSection `yaml:"metrics"`
|
||||
}
|
||||
|
||||
type changeSection struct {
|
||||
Changes []schemaChange `yaml:"changes"`
|
||||
}
|
||||
|
||||
type schemaChange struct {
|
||||
RenameAttributes *attributeRename `yaml:"rename_attributes"`
|
||||
RenameMetrics map[string]string `yaml:"rename_metrics"`
|
||||
}
|
||||
|
||||
type attributeRename struct {
|
||||
AttributeMap map[string]string `yaml:"attribute_map"`
|
||||
ApplyToMetrics []string `yaml:"apply_to_metrics"`
|
||||
}
|
||||
|
||||
type overlayFile struct {
|
||||
DefaultEnabled bool `yaml:"default_enabled"`
|
||||
// Families is keyed only by current name. One name cannot carry separate
|
||||
// policies for attribute and metric families; set kind explicitly whenever
|
||||
// a metric-name family is configured.
|
||||
Families map[string]overlayFamily `yaml:"families"`
|
||||
}
|
||||
|
||||
type overlayFamily struct {
|
||||
Enabled *bool `yaml:"enabled"`
|
||||
Kind string `yaml:"kind"`
|
||||
Old []string `yaml:"old"`
|
||||
AddOld []string `yaml:"add_old"`
|
||||
ExcludeOld []string `yaml:"exclude_old"`
|
||||
Contexts []string `yaml:"contexts"`
|
||||
Signals []string `yaml:"signals"`
|
||||
AddContexts []string `yaml:"add_contexts"`
|
||||
AddSignals []string `yaml:"add_signals"`
|
||||
ApplyToMetrics []string `yaml:"apply_to_metrics"`
|
||||
AddApplyToMetrics []string `yaml:"add_apply_to_metrics"`
|
||||
ValueMap map[string]string `yaml:"value_map"`
|
||||
}
|
||||
|
||||
type edge struct {
|
||||
old string
|
||||
current string
|
||||
kind string
|
||||
contexts []string
|
||||
signals []string
|
||||
allContexts bool
|
||||
allSignals bool
|
||||
applyToMetrics []string
|
||||
}
|
||||
|
||||
type graphKey struct{ kind, name string }
|
||||
|
||||
type generatedFamily struct {
|
||||
Current string
|
||||
Old []string
|
||||
Kind string
|
||||
Contexts []string
|
||||
Signals []string
|
||||
ApplyToMetrics []string
|
||||
ValueMap map[string]string
|
||||
}
|
||||
|
||||
func main() {
|
||||
root, err := findRepoRoot()
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
|
||||
var schemaPaths stringListFlag
|
||||
flag.Var(&schemaPaths, "schema", "schema source (repeatable)")
|
||||
overlayPath := flag.String("overlay", filepath.Join(root, "scripts/semconv/overlay.yaml"), "SigNoz overlay")
|
||||
goOutput := flag.String("go-out", filepath.Join(root, "pkg/semconv/families_gen.go"), "generated Go output")
|
||||
tsOutput := flag.String("ts-out", filepath.Join(root, "frontend/src/constants/generated/semconvFamilies.gen.ts"), "generated TypeScript output")
|
||||
check := flag.Bool("check", false, "fail if generated files are stale")
|
||||
flag.Parse()
|
||||
|
||||
if len(schemaPaths) == 0 {
|
||||
schemaPaths = append(schemaPaths, filepath.Join(root, "scripts/semconv/schema-1.42.0.yaml"))
|
||||
}
|
||||
|
||||
families, err := generate(schemaPaths, *overlayPath)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
goBytes, err := renderGo(families)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
tsBytes := renderTypeScript(families)
|
||||
|
||||
if *check {
|
||||
if err := checkFile(*goOutput, goBytes); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if err := checkFile(*tsOutput, tsBytes); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err := os.WriteFile(*goOutput, goBytes, 0o644); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(*tsOutput, tsBytes, 0o644); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func findRepoRoot() (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
|
||||
return dir, nil
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return "", errors.New("could not find repository root")
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
|
||||
func generate(schemaPaths []string, overlayPath string) ([]generatedFamily, error) {
|
||||
var schemas []schemaFile
|
||||
for _, path := range schemaPaths {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read schema %s: %w", path, err)
|
||||
}
|
||||
var schema schemaFile
|
||||
if err := decodeKnownFields(data, &schema); err != nil {
|
||||
return nil, fmt.Errorf("parse schema %s: %w", path, err)
|
||||
}
|
||||
schemas = append(schemas, schema)
|
||||
}
|
||||
|
||||
overlayData, err := os.ReadFile(overlayPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read overlay: %w", err)
|
||||
}
|
||||
var overlay overlayFile
|
||||
if err := decodeKnownFields(overlayData, &overlay); err != nil {
|
||||
return nil, fmt.Errorf("parse overlay: %w", err)
|
||||
}
|
||||
|
||||
return buildFamilies(schemas, overlay)
|
||||
}
|
||||
|
||||
func decodeKnownFields(data []byte, target any) error {
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(data))
|
||||
decoder.KnownFields(true)
|
||||
return decoder.Decode(target)
|
||||
}
|
||||
|
||||
func collectEdges(schemas []schemaFile) ([]edge, error) {
|
||||
var edges []edge
|
||||
for _, schema := range schemas {
|
||||
versions := make([]string, 0, len(schema.Versions))
|
||||
versionParts := make(map[string][3]int, len(schema.Versions))
|
||||
for version := range schema.Versions {
|
||||
parts, err := parseSchemaVersion(version)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
versions = append(versions, version)
|
||||
versionParts[version] = parts
|
||||
}
|
||||
sort.Slice(versions, func(i, j int) bool {
|
||||
return compareVersionParts(versionParts[versions[i]], versionParts[versions[j]]) < 0
|
||||
})
|
||||
for _, versionName := range versions {
|
||||
version := schema.Versions[versionName]
|
||||
var versionEdges []edge
|
||||
sections := []struct {
|
||||
name string
|
||||
section changeSection
|
||||
}{
|
||||
{name: "all", section: version.All},
|
||||
{name: "resources", section: version.Resources},
|
||||
{name: "spans", section: version.Spans},
|
||||
{name: "logs", section: version.Logs},
|
||||
{name: "metrics", section: version.Metrics},
|
||||
}
|
||||
for _, scoped := range sections {
|
||||
contexts, signals, allContexts, allSignals, err := scopeForSection(scoped.name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, change := range scoped.section.Changes {
|
||||
if change.RenameAttributes != nil {
|
||||
for _, old := range sortedMapKeys(change.RenameAttributes.AttributeMap) {
|
||||
versionEdges = append(versionEdges, edge{
|
||||
old: old, current: change.RenameAttributes.AttributeMap[old], kind: kindAttribute,
|
||||
contexts: contexts, signals: signals,
|
||||
allContexts: allContexts, allSignals: allSignals,
|
||||
applyToMetrics: change.RenameAttributes.ApplyToMetrics,
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, old := range sortedMapKeys(change.RenameMetrics) {
|
||||
versionEdges = append(versionEdges, edge{
|
||||
old: old, current: change.RenameMetrics[old], kind: kindMetric,
|
||||
contexts: []string{"metric"}, signals: []string{"metrics"},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := rejectSameVersionChains(versionName, versionEdges); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
edges = append(edges, versionEdges...)
|
||||
}
|
||||
}
|
||||
return edges, nil
|
||||
}
|
||||
|
||||
func rejectSameVersionChains(version string, edges []edge) error {
|
||||
oldNames := make(map[graphKey]struct{}, len(edges))
|
||||
for _, item := range edges {
|
||||
oldNames[graphKey{kind: item.kind, name: item.old}] = struct{}{}
|
||||
}
|
||||
for _, item := range edges {
|
||||
if _, ok := oldNames[graphKey{kind: item.kind, name: item.current}]; ok {
|
||||
return fmt.Errorf(
|
||||
"schema version %q contains a same-version %s rename chain through %q",
|
||||
version,
|
||||
item.kind,
|
||||
item.current,
|
||||
)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseSchemaVersion(version string) ([3]int, error) {
|
||||
parts := strings.Split(version, ".")
|
||||
if len(parts) != 3 {
|
||||
return [3]int{}, fmt.Errorf("schema version %q must contain major, minor, and patch numbers", version)
|
||||
}
|
||||
|
||||
var parsed [3]int
|
||||
for i, part := range parts {
|
||||
value, err := strconv.Atoi(part)
|
||||
if err != nil || value < 0 {
|
||||
return [3]int{}, fmt.Errorf("schema version %q contains invalid numeric component %q", version, part)
|
||||
}
|
||||
parsed[i] = value
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func compareVersionParts(left, right [3]int) int {
|
||||
for i := range left {
|
||||
if left[i] < right[i] {
|
||||
return -1
|
||||
}
|
||||
if left[i] > right[i] {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func scopeForSection(section string) (contexts, signals []string, allContexts, allSignals bool, err error) {
|
||||
switch section {
|
||||
case "all":
|
||||
return nil, nil, true, true, nil
|
||||
case "resources":
|
||||
return []string{"resource"}, nil, false, true, nil
|
||||
case "spans":
|
||||
return []string{"attribute"}, []string{"traces"}, false, false, nil
|
||||
case "logs":
|
||||
return []string{"attribute"}, []string{"logs"}, false, false, nil
|
||||
case "metrics":
|
||||
return []string{"attribute"}, []string{"metrics"}, false, false, nil
|
||||
default:
|
||||
return nil, nil, false, false, fmt.Errorf("unsupported schema section %q", section)
|
||||
}
|
||||
}
|
||||
|
||||
func buildFamilies(schemas []schemaFile, overlay overlayFile) ([]generatedFamily, error) {
|
||||
edges, err := collectEdges(schemas)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
next := make(map[graphKey]string)
|
||||
for _, item := range edges {
|
||||
key := graphKey{kind: item.kind, name: item.old}
|
||||
if existing, ok := next[key]; ok && existing == item.current {
|
||||
// Repeated entries are common in chained schema histories. Treat an
|
||||
// identical edge as a no-op so it cannot sever a later edge in the
|
||||
// same chain (A -> B, B -> C, then a repeated A -> B).
|
||||
continue
|
||||
}
|
||||
// Schema history occasionally repeats an old name with a newer direct
|
||||
// destination or rolls a rename back. Edges are collected
|
||||
// oldest-to-newest, so the latest published current name must be a root.
|
||||
delete(next, graphKey{kind: item.kind, name: item.current})
|
||||
next[key] = item.current
|
||||
}
|
||||
|
||||
type familyState struct {
|
||||
family generatedFamily
|
||||
distance map[string]int
|
||||
allContexts bool
|
||||
allSignals bool
|
||||
}
|
||||
states := map[graphKey]*familyState{}
|
||||
for _, item := range edges {
|
||||
root, distance, err := rootFor(next, item.kind, item.old)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key := graphKey{kind: item.kind, name: root}
|
||||
state := states[key]
|
||||
if state == nil {
|
||||
state = &familyState{
|
||||
family: generatedFamily{Current: root, Kind: item.kind},
|
||||
distance: map[string]int{},
|
||||
}
|
||||
states[key] = state
|
||||
}
|
||||
if prior, ok := state.distance[item.old]; !ok || distance < prior {
|
||||
state.distance[item.old] = distance
|
||||
}
|
||||
state.allContexts = state.allContexts || item.allContexts
|
||||
state.allSignals = state.allSignals || item.allSignals
|
||||
state.family.Contexts = appendUnique(state.family.Contexts, item.contexts...)
|
||||
state.family.Signals = appendUnique(state.family.Signals, item.signals...)
|
||||
state.family.ApplyToMetrics = appendUnique(state.family.ApplyToMetrics, item.applyToMetrics...)
|
||||
}
|
||||
|
||||
for _, state := range states {
|
||||
for old := range state.distance {
|
||||
if old != state.family.Current {
|
||||
state.family.Old = append(state.family.Old, old)
|
||||
}
|
||||
}
|
||||
sort.Slice(state.family.Old, func(i, j int) bool {
|
||||
left, right := state.family.Old[i], state.family.Old[j]
|
||||
if state.distance[left] != state.distance[right] {
|
||||
return state.distance[left] < state.distance[right]
|
||||
}
|
||||
return left < right
|
||||
})
|
||||
if state.allContexts {
|
||||
state.family.Contexts = nil
|
||||
} else {
|
||||
sort.Strings(state.family.Contexts)
|
||||
}
|
||||
if state.allSignals {
|
||||
state.family.Signals = nil
|
||||
} else {
|
||||
sort.Strings(state.family.Signals)
|
||||
}
|
||||
sort.Strings(state.family.ApplyToMetrics)
|
||||
}
|
||||
|
||||
for _, current := range sortedMapKeys(overlay.Families) {
|
||||
policy := overlay.Families[current]
|
||||
kind, err := normalizedOverlayKind(current, policy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policy.Kind = kind
|
||||
overlay.Families[current] = policy
|
||||
key := graphKey{kind: kind, name: current}
|
||||
state := states[key]
|
||||
if state == nil {
|
||||
if len(policy.Old) == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"overlay family %q with kind %q is absent from schemas and has no old members",
|
||||
current,
|
||||
kind,
|
||||
)
|
||||
}
|
||||
state = &familyState{
|
||||
family: generatedFamily{Current: current, Kind: kind, Old: append([]string(nil), policy.Old...)},
|
||||
distance: map[string]int{},
|
||||
}
|
||||
states[key] = state
|
||||
}
|
||||
applyOverlay(&state.family, policy)
|
||||
}
|
||||
|
||||
var result []generatedFamily
|
||||
for key, state := range states {
|
||||
policy, hasPolicy := overlay.Families[key.name]
|
||||
enabled := overlay.DefaultEnabled
|
||||
if hasPolicy && policy.Kind != key.kind {
|
||||
hasPolicy = false
|
||||
}
|
||||
if hasPolicy && policy.Enabled != nil {
|
||||
enabled = *policy.Enabled
|
||||
}
|
||||
if !enabled {
|
||||
continue
|
||||
}
|
||||
if len(state.family.Old) == 0 {
|
||||
return nil, fmt.Errorf(
|
||||
"enabled family %q with kind %q has no old members",
|
||||
state.family.Current,
|
||||
state.family.Kind,
|
||||
)
|
||||
}
|
||||
sort.Strings(state.family.Contexts)
|
||||
sort.Strings(state.family.Signals)
|
||||
sort.Strings(state.family.ApplyToMetrics)
|
||||
result = append(result, state.family)
|
||||
}
|
||||
|
||||
sort.Slice(result, func(i, j int) bool {
|
||||
if result[i].Current != result[j].Current {
|
||||
return result[i].Current < result[j].Current
|
||||
}
|
||||
return result[i].Kind < result[j].Kind
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func rootFor(next map[graphKey]string, kind, name string) (string, int, error) {
|
||||
seen := map[string]bool{}
|
||||
distance := 0
|
||||
for {
|
||||
if seen[name] {
|
||||
return "", 0, fmt.Errorf("rename cycle for %s %q", kind, name)
|
||||
}
|
||||
seen[name] = true
|
||||
current, ok := next[graphKey{kind: kind, name: name}]
|
||||
if !ok {
|
||||
return name, distance, nil
|
||||
}
|
||||
name = current
|
||||
distance++
|
||||
}
|
||||
}
|
||||
|
||||
func normalizedOverlayKind(current string, policy overlayFamily) (string, error) {
|
||||
kind := policy.Kind
|
||||
if kind == "" {
|
||||
kind = kindAttribute
|
||||
}
|
||||
if kind != kindAttribute && kind != kindMetric {
|
||||
return "", fmt.Errorf("overlay family %q has unsupported kind %q", current, kind)
|
||||
}
|
||||
return kind, nil
|
||||
}
|
||||
|
||||
func applyOverlay(family *generatedFamily, policy overlayFamily) {
|
||||
if policy.Kind != "" {
|
||||
family.Kind = policy.Kind
|
||||
}
|
||||
if policy.Old != nil {
|
||||
family.Old = append([]string(nil), policy.Old...)
|
||||
}
|
||||
family.Old = appendUnique(family.Old, policy.AddOld...)
|
||||
if len(policy.ExcludeOld) > 0 {
|
||||
excluded := make(map[string]bool, len(policy.ExcludeOld))
|
||||
for _, old := range policy.ExcludeOld {
|
||||
excluded[old] = true
|
||||
}
|
||||
family.Old = deleteMatching(family.Old, excluded)
|
||||
}
|
||||
if policy.Contexts != nil {
|
||||
family.Contexts = append([]string(nil), policy.Contexts...)
|
||||
}
|
||||
if policy.Signals != nil {
|
||||
family.Signals = append([]string(nil), policy.Signals...)
|
||||
}
|
||||
family.Contexts = appendUnique(family.Contexts, policy.AddContexts...)
|
||||
family.Signals = appendUnique(family.Signals, policy.AddSignals...)
|
||||
if policy.ApplyToMetrics != nil {
|
||||
family.ApplyToMetrics = append([]string(nil), policy.ApplyToMetrics...)
|
||||
}
|
||||
family.ApplyToMetrics = appendUnique(family.ApplyToMetrics, policy.AddApplyToMetrics...)
|
||||
if policy.ValueMap != nil {
|
||||
family.ValueMap = make(map[string]string, len(policy.ValueMap))
|
||||
for old, current := range policy.ValueMap {
|
||||
family.ValueMap[old] = current
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func appendUnique(values []string, additions ...string) []string {
|
||||
seen := make(map[string]bool, len(values)+len(additions))
|
||||
for _, value := range values {
|
||||
seen[value] = true
|
||||
}
|
||||
for _, value := range additions {
|
||||
if value == "" || seen[value] {
|
||||
continue
|
||||
}
|
||||
seen[value] = true
|
||||
values = append(values, value)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func deleteMatching(values []string, excluded map[string]bool) []string {
|
||||
result := values[:0]
|
||||
for _, value := range values {
|
||||
if !excluded[value] {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func renderGo(families []generatedFamily) ([]byte, error) {
|
||||
var out bytes.Buffer
|
||||
out.WriteString("// Code generated by scripts/semconv. DO NOT EDIT.\n\n")
|
||||
out.WriteString("package semconv\n\n")
|
||||
needsTelemetryTypes := false
|
||||
for _, family := range families {
|
||||
if len(family.Contexts) > 0 || len(family.Signals) > 0 {
|
||||
needsTelemetryTypes = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if needsTelemetryTypes {
|
||||
out.WriteString("import \"github.com/SigNoz/signoz/pkg/types/telemetrytypes\"\n\n")
|
||||
}
|
||||
out.WriteString("var families = []Family{\n")
|
||||
for _, family := range families {
|
||||
contexts, err := goFieldContextSlice(family.Contexts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("render family %q: %w", family.Current, err)
|
||||
}
|
||||
signals, err := goSignalSlice(family.Signals)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("render family %q: %w", family.Current, err)
|
||||
}
|
||||
out.WriteString("\t{\n")
|
||||
fmt.Fprintf(&out, "\t\tCurrent: %s,\n", strconv.Quote(family.Current))
|
||||
fmt.Fprintf(&out, "\t\tOld: %s,\n", goStringSlice(family.Old))
|
||||
if family.Kind == kindMetric {
|
||||
out.WriteString("\t\tKind: KindMetric,\n")
|
||||
} else {
|
||||
out.WriteString("\t\tKind: KindAttribute,\n")
|
||||
}
|
||||
fmt.Fprintf(&out, "\t\tContexts: %s,\n", contexts)
|
||||
fmt.Fprintf(&out, "\t\tSignals: %s,\n", signals)
|
||||
fmt.Fprintf(&out, "\t\tApplyToMetrics: %s,\n", goStringSlice(family.ApplyToMetrics))
|
||||
if len(family.ValueMap) > 0 {
|
||||
out.WriteString("\t\tValueMap: map[string]string{\n")
|
||||
keys := sortedMapKeys(family.ValueMap)
|
||||
for _, key := range keys {
|
||||
fmt.Fprintf(&out, "\t\t\t%s: %s,\n", strconv.Quote(key), strconv.Quote(family.ValueMap[key]))
|
||||
}
|
||||
out.WriteString("\t\t},\n")
|
||||
}
|
||||
out.WriteString("\t},\n")
|
||||
}
|
||||
out.WriteString("}\n")
|
||||
return format.Source(out.Bytes())
|
||||
}
|
||||
|
||||
func goStringSlice(values []string) string {
|
||||
if len(values) == 0 {
|
||||
return "nil"
|
||||
}
|
||||
quoted := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
quoted[i] = strconv.Quote(value)
|
||||
}
|
||||
return "[]string{" + strings.Join(quoted, ", ") + "}"
|
||||
}
|
||||
|
||||
func goFieldContextSlice(values []string) (string, error) {
|
||||
if len(values) == 0 {
|
||||
return "nil", nil
|
||||
}
|
||||
constants := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
switch value {
|
||||
case "metric":
|
||||
constants[i] = "telemetrytypes.FieldContextMetric"
|
||||
case "resource":
|
||||
constants[i] = "telemetrytypes.FieldContextResource"
|
||||
case "attribute":
|
||||
constants[i] = "telemetrytypes.FieldContextAttribute"
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported field context %q", value)
|
||||
}
|
||||
}
|
||||
return "[]telemetrytypes.FieldContext{" + strings.Join(constants, ", ") + "}", nil
|
||||
}
|
||||
|
||||
func goSignalSlice(values []string) (string, error) {
|
||||
if len(values) == 0 {
|
||||
return "nil", nil
|
||||
}
|
||||
constants := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
switch value {
|
||||
case "traces":
|
||||
constants[i] = "telemetrytypes.SignalTraces"
|
||||
case "logs":
|
||||
constants[i] = "telemetrytypes.SignalLogs"
|
||||
case "metrics":
|
||||
constants[i] = "telemetrytypes.SignalMetrics"
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported signal %q", value)
|
||||
}
|
||||
}
|
||||
return "[]telemetrytypes.Signal{" + strings.Join(constants, ", ") + "}", nil
|
||||
}
|
||||
|
||||
func renderTypeScript(families []generatedFamily) []byte {
|
||||
var out bytes.Buffer
|
||||
out.WriteString("// Code generated by scripts/semconv. DO NOT EDIT.\n\n")
|
||||
out.WriteString("export type SemconvFamily = {\n")
|
||||
out.WriteString("\treadonly current: string;\n\treadonly old: readonly string[];\n")
|
||||
out.WriteString("\treadonly kind: 'attribute' | 'metric';\n")
|
||||
out.WriteString("\treadonly contexts: readonly string[];\n\treadonly signals: readonly string[];\n")
|
||||
out.WriteString("\treadonly applyToMetrics: readonly string[];\n")
|
||||
out.WriteString("\treadonly valueMap: Readonly<Record<string, string>>;\n};\n\n")
|
||||
out.WriteString("export const SEMCONV_FAMILIES: readonly SemconvFamily[] = [\n")
|
||||
for _, family := range families {
|
||||
out.WriteString("\t{\n")
|
||||
fmt.Fprintf(&out, "\t\tcurrent: %s,\n", tsString(family.Current))
|
||||
fmt.Fprintf(&out, "\t\told: %s,\n", tsStringSlice(family.Old))
|
||||
fmt.Fprintf(&out, "\t\tkind: %s,\n", tsString(family.Kind))
|
||||
fmt.Fprintf(&out, "\t\tcontexts: %s,\n", tsStringSlice(family.Contexts))
|
||||
fmt.Fprintf(&out, "\t\tsignals: %s,\n", tsStringSlice(family.Signals))
|
||||
fmt.Fprintf(&out, "\t\tapplyToMetrics: %s,\n", tsStringSlice(family.ApplyToMetrics))
|
||||
out.WriteString("\t\tvalueMap: {")
|
||||
keys := sortedMapKeys(family.ValueMap)
|
||||
for i, key := range keys {
|
||||
if i > 0 {
|
||||
out.WriteString(", ")
|
||||
}
|
||||
fmt.Fprintf(&out, "%s: %s", tsString(key), tsString(family.ValueMap[key]))
|
||||
}
|
||||
out.WriteString("},\n\t},\n")
|
||||
}
|
||||
out.WriteString("] as const;\n")
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func tsString(value string) string {
|
||||
quoted := strconv.Quote(value)
|
||||
return "'" + strings.ReplaceAll(quoted[1:len(quoted)-1], "'", `\'`) + "'"
|
||||
}
|
||||
func tsStringSlice(values []string) string {
|
||||
quoted := make([]string, len(values))
|
||||
for i, value := range values {
|
||||
quoted[i] = tsString(value)
|
||||
}
|
||||
return "[" + strings.Join(quoted, ", ") + "]"
|
||||
}
|
||||
|
||||
func sortedMapKeys[T any](values map[string]T) []string {
|
||||
keys := make([]string, 0, len(values))
|
||||
for key := range values {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
func checkFile(path string, expected []byte) error {
|
||||
actual, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("generated file %s is missing: run go run ./scripts/semconv", path)
|
||||
}
|
||||
if !bytes.Equal(actual, expected) {
|
||||
return fmt.Errorf("generated file %s is stale: run go run ./scripts/semconv", path)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
366
scripts/semconv/generate_test.go
Normal file
366
scripts/semconv/generate_test.go
Normal file
@@ -0,0 +1,366 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSchemaDecoderRejectsUnsupportedSection(t *testing.T) {
|
||||
var schema schemaFile
|
||||
err := decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
span_events:
|
||||
changes:
|
||||
- rename_events:
|
||||
event_map:
|
||||
old: current
|
||||
`), &schema)
|
||||
|
||||
assert.ErrorContains(t, err, "field span_events not found", "unsupported schema sections must fail generation")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesRejectsMalformedSchemaVersion(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
latest:
|
||||
spans:
|
||||
changes: []
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
_, err := buildFamilies([]schemaFile{schema}, overlayFile{})
|
||||
assert.ErrorContains(t, err, `schema version "latest"`, "malformed versions must not be silently reordered")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesResolvesRenameChain(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
4.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
a: b
|
||||
3.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
b: c
|
||||
x: c
|
||||
2.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
a: b
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
a: b
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
enabled := true
|
||||
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
|
||||
"c": {Enabled: &enabled},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []generatedFamily{{
|
||||
Current: "c",
|
||||
Old: []string{"b", "x", "a"},
|
||||
Kind: kindAttribute,
|
||||
Contexts: []string{"attribute"},
|
||||
Signals: []string{"traces"},
|
||||
}}, families, "predecessors should be ordered by distance and then name")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesMapsSchemaSectionsToScopes(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
resources:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
resource.old: resource.current
|
||||
logs:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
log.old: log.current
|
||||
metrics:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
state: cpu.mode
|
||||
apply_to_metrics: [system.cpu.time]
|
||||
- rename_metrics:
|
||||
old.metric: current.metric
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
enabled := true
|
||||
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
|
||||
"resource.current": {Enabled: &enabled},
|
||||
"log.current": {Enabled: &enabled},
|
||||
"cpu.mode": {Enabled: &enabled},
|
||||
"current.metric": {Enabled: &enabled, Kind: kindMetric},
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []generatedFamily{
|
||||
{
|
||||
Current: "cpu.mode", Old: []string{"state"}, Kind: kindAttribute,
|
||||
Contexts: []string{"attribute"}, Signals: []string{"metrics"},
|
||||
ApplyToMetrics: []string{"system.cpu.time"},
|
||||
},
|
||||
{
|
||||
Current: "current.metric", Old: []string{"old.metric"}, Kind: kindMetric,
|
||||
Contexts: []string{"metric"}, Signals: []string{"metrics"},
|
||||
},
|
||||
{
|
||||
Current: "log.current", Old: []string{"log.old"}, Kind: kindAttribute,
|
||||
Contexts: []string{"attribute"}, Signals: []string{"logs"},
|
||||
},
|
||||
{
|
||||
Current: "resource.current", Old: []string{"resource.old"}, Kind: kindAttribute,
|
||||
Contexts: []string{"resource"},
|
||||
},
|
||||
}, families, "schema sections should produce their documented signal and context scopes")
|
||||
}
|
||||
|
||||
func TestOverlayAddsFamilyWithoutSchemaHistory(t *testing.T) {
|
||||
enabled := true
|
||||
families, err := buildFamilies(nil, overlayFile{Families: map[string]overlayFamily{
|
||||
"added.current": {
|
||||
Enabled: &enabled,
|
||||
Old: []string{"added.old"},
|
||||
Contexts: []string{"resource"},
|
||||
Signals: []string{"traces"},
|
||||
},
|
||||
}})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []generatedFamily{{
|
||||
Current: "added.current",
|
||||
Old: []string{"added.old"},
|
||||
Kind: kindAttribute,
|
||||
Contexts: []string{"resource"},
|
||||
Signals: []string{"traces"},
|
||||
}}, families, "an explicit overlay family should not require schema history")
|
||||
}
|
||||
|
||||
func TestOverlayOverridesGeneratedFamily(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
old: current
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
enabled := true
|
||||
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
|
||||
"current": {
|
||||
Enabled: &enabled,
|
||||
AddOld: []string{"older"},
|
||||
ExcludeOld: []string{"old"},
|
||||
AddContexts: []string{"resource"},
|
||||
AddSignals: []string{"logs"},
|
||||
ValueMap: map[string]string{"legacy": "current"},
|
||||
},
|
||||
}})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []generatedFamily{{
|
||||
Current: "current",
|
||||
Old: []string{"older"},
|
||||
Kind: kindAttribute,
|
||||
Contexts: []string{"attribute", "resource"},
|
||||
Signals: []string{"logs", "traces"},
|
||||
ValueMap: map[string]string{"legacy": "current"},
|
||||
}}, families, "overlay additions and exclusions should be applied to the generated family")
|
||||
}
|
||||
|
||||
func TestOverlayDisablesFamilyWhenDefaultIsEnabled(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
old: current
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
disabled := false
|
||||
families, err := buildFamilies([]schemaFile{schema}, overlayFile{
|
||||
DefaultEnabled: true,
|
||||
Families: map[string]overlayFamily{
|
||||
"current": {Enabled: &disabled},
|
||||
},
|
||||
})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, families, "an explicitly disabled family must override default_enabled")
|
||||
}
|
||||
|
||||
func TestRenderGoIsDeterministic(t *testing.T) {
|
||||
families := []generatedFamily{{
|
||||
Current: "current", Old: []string{"old"}, Kind: kindAttribute,
|
||||
ValueMap: map[string]string{"b": "2", "a": "1"},
|
||||
}}
|
||||
|
||||
first, err := renderGo(families)
|
||||
require.NoError(t, err)
|
||||
second, err := renderGo(families)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first, second, "Go generation must not depend on map iteration order")
|
||||
}
|
||||
|
||||
func TestRenderGoUsesCanonicalTelemetryTypes(t *testing.T) {
|
||||
families := []generatedFamily{{
|
||||
Current: "current", Old: []string{"old"}, Kind: kindAttribute,
|
||||
Contexts: []string{"resource"}, Signals: []string{"traces"},
|
||||
}}
|
||||
|
||||
output, err := renderGo(families)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(output), "telemetrytypes.FieldContextResource", "generated contexts should use telemetrytypes")
|
||||
assert.Contains(t, string(output), "telemetrytypes.SignalTraces", "generated signals should use telemetrytypes")
|
||||
}
|
||||
|
||||
func TestRenderTypeScriptIsDeterministic(t *testing.T) {
|
||||
families := []generatedFamily{{
|
||||
Current: "current", Old: []string{"old"}, Kind: kindAttribute,
|
||||
ValueMap: map[string]string{"b": "2", "a": "1"},
|
||||
}}
|
||||
|
||||
assert.Equal(t, renderTypeScript(families), renderTypeScript(families), "TypeScript generation must not depend on map iteration order")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesHandlesRenameRollback(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
2.0.0:
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
temporary: original
|
||||
1.0.0:
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
original: temporary
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
enabled := true
|
||||
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
|
||||
"original": {Enabled: &enabled, Kind: kindMetric},
|
||||
}})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []generatedFamily{{
|
||||
Current: "original",
|
||||
Old: []string{"temporary"},
|
||||
Kind: kindMetric,
|
||||
Contexts: []string{"metric"},
|
||||
Signals: []string{"metrics"},
|
||||
}}, families, "the latest rollback destination should remain the family root")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesRejectsSameVersionRenameChain(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
x: y
|
||||
y: z
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
_, err := buildFamilies([]schemaFile{schema}, overlayFile{})
|
||||
assert.ErrorContains(t, err, `same-version attribute rename chain through "y"`, "order-sensitive same-version chains must be rejected")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesRejectsOverlayFamilyWithoutHistory(t *testing.T) {
|
||||
enabled := true
|
||||
_, err := buildFamilies(nil, overlayFile{Families: map[string]overlayFamily{
|
||||
"missing": {Enabled: &enabled},
|
||||
}})
|
||||
|
||||
assert.ErrorContains(t, err, `overlay family "missing" with kind "attribute" is absent`, "an overlay cannot invent a family without old members")
|
||||
}
|
||||
|
||||
func TestBuildFamiliesRejectsEnabledFamilyWithoutOldMembers(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
old: current
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
enabled := true
|
||||
_, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
|
||||
"current": {Enabled: &enabled, ExcludeOld: []string{"old"}},
|
||||
}})
|
||||
assert.ErrorContains(t, err, `enabled family "current" with kind "attribute" has no old members`, "exclude_old cannot empty an enabled family")
|
||||
}
|
||||
|
||||
func TestOverlayKindDefaultsToAttribute(t *testing.T) {
|
||||
var schema schemaFile
|
||||
require.NoError(t, decodeKnownFields([]byte(`
|
||||
versions:
|
||||
1.0.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
attribute.old: shared.current
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
metric.old: shared.current
|
||||
`), &schema), "test schema must decode")
|
||||
|
||||
enabled := true
|
||||
families, err := buildFamilies([]schemaFile{schema}, overlayFile{Families: map[string]overlayFamily{
|
||||
"shared.current": {Enabled: &enabled},
|
||||
}})
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []generatedFamily{{
|
||||
Current: "shared.current",
|
||||
Old: []string{"attribute.old"},
|
||||
Kind: kindAttribute,
|
||||
Contexts: []string{"attribute"},
|
||||
Signals: []string{"traces"},
|
||||
}}, families, "a kind-less overlay policy should affect only the attribute family")
|
||||
}
|
||||
|
||||
func TestCheckFileReportsStaleOutput(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "generated.go")
|
||||
require.NoError(t, os.WriteFile(path, []byte("old"), 0o600), "test output must be writable")
|
||||
|
||||
assert.ErrorContains(t, checkFile(path, []byte("new")), "is stale", "check mode must reject stale generated output")
|
||||
}
|
||||
|
||||
func TestTypeScriptStringEscapesControlCharacters(t *testing.T) {
|
||||
assert.Equal(t, `'line\n\t\x01\'\\end'`, tsString("line\n\t\x01'\\end"), "generated TypeScript strings must remain valid literals")
|
||||
}
|
||||
37
scripts/semconv/overlay.yaml
Normal file
37
scripts/semconv/overlay.yaml
Normal file
@@ -0,0 +1,37 @@
|
||||
# SigNoz semantic-convention rollout policy.
|
||||
#
|
||||
# Families are keyed by their current OpenTelemetry name. Schema-derived
|
||||
# families are disabled by default so rollout remains explicit and reversible.
|
||||
default_enabled: false
|
||||
|
||||
families:
|
||||
deployment.environment.name:
|
||||
enabled: true
|
||||
contexts: [resource, attribute]
|
||||
signals: [traces, logs, metrics]
|
||||
db.system.name:
|
||||
enabled: true
|
||||
contexts: [resource, attribute]
|
||||
signals: [traces, logs, metrics]
|
||||
|
||||
# These metric renames predate the schema history vendored above. Keep them
|
||||
# in the same generated registry so every v5 metric query uses one source of
|
||||
# truth instead of the legacy hand-written transition table.
|
||||
k8s.pod.cpu.usage:
|
||||
enabled: true
|
||||
kind: metric
|
||||
old: [k8s.pod.cpu.utilization]
|
||||
contexts: [metric]
|
||||
signals: [metrics]
|
||||
k8s.node.cpu.usage:
|
||||
enabled: true
|
||||
kind: metric
|
||||
old: [k8s.node.cpu.utilization]
|
||||
contexts: [metric]
|
||||
signals: [metrics]
|
||||
container.cpu.usage:
|
||||
enabled: true
|
||||
kind: metric
|
||||
old: [container.cpu.utilization]
|
||||
contexts: [metric]
|
||||
signals: [metrics]
|
||||
760
scripts/semconv/schema-1.42.0.yaml
Normal file
760
scripts/semconv/schema-1.42.0.yaml
Normal file
@@ -0,0 +1,760 @@
|
||||
|
||||
|
||||
file_format: 1.1.0
|
||||
schema_url: https://opentelemetry.io/schemas/1.42.0
|
||||
versions:
|
||||
1.42.0:
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
v8js.memory.heap.limit: v8js.memory.heap.space.size
|
||||
1.41.1:
|
||||
1.41.0:
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
k8s.container.cpu.limit: k8s.container.cpu.limit.desired
|
||||
k8s.container.cpu.limit_utilization: k8s.container.cpu.limit.utilization
|
||||
k8s.container.cpu.request: k8s.container.cpu.request.desired
|
||||
k8s.container.cpu.request_utilization: k8s.container.cpu.request.utilization
|
||||
k8s.container.memory.limit: k8s.container.memory.limit.desired
|
||||
k8s.container.memory.request: k8s.container.memory.request.desired
|
||||
1.40.0:
|
||||
all:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
feature_flag.evaluation.error.message: feature_flag.error.message
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
system.memory.shared: system.memory.linux.shared
|
||||
1.39.0:
|
||||
all:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
linux.memory.slab.state: system.memory.linux.slab.state
|
||||
peer.service: service.peer.name
|
||||
rpc.connect_rpc.error_code: rpc.response.status_code
|
||||
rpc.connect_rpc.request.metadata: rpc.request.metadata
|
||||
rpc.connect_rpc.response.metadata: rpc.response.metadata
|
||||
rpc.grpc.request.metadata: rpc.request.metadata
|
||||
rpc.grpc.response.metadata: rpc.response.metadata
|
||||
rpc.jsonrpc.request_id: jsonrpc.request.id
|
||||
rpc.jsonrpc.version: jsonrpc.protocol.version
|
||||
rpc.system: rpc.system.name
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
process.open_file_descriptor.count: process.unix.file_descriptor.count
|
||||
system.linux.memory.available: system.memory.linux.available
|
||||
system.linux.memory.slab.usage: system.memory.linux.slab.usage
|
||||
1.38.0:
|
||||
all:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
process.context_switch_type: process.context_switch.type
|
||||
process.paging.fault_type: system.paging.fault.type
|
||||
system.cpu.logical_number: cpu.logical_number
|
||||
system.paging.type: system.paging.fault.type
|
||||
system.process.status: process.state
|
||||
system.processes.status: process.state
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
k8s.cronjob.active_jobs: k8s.cronjob.job.active
|
||||
k8s.daemonset.current_scheduled_nodes: k8s.daemonset.node.current_scheduled
|
||||
k8s.daemonset.desired_scheduled_nodes: k8s.daemonset.node.desired_scheduled
|
||||
k8s.daemonset.misscheduled_nodes: k8s.daemonset.node.misscheduled
|
||||
k8s.daemonset.ready_nodes: k8s.daemonset.node.ready
|
||||
k8s.deployment.available_pods: k8s.deployment.pod.available
|
||||
k8s.deployment.desired_pods: k8s.deployment.pod.desired
|
||||
k8s.hpa.current_pods: k8s.hpa.pod.current
|
||||
k8s.hpa.desired_pods: k8s.hpa.pod.desired
|
||||
k8s.hpa.max_pods: k8s.hpa.pod.max
|
||||
k8s.hpa.min_pods: k8s.hpa.pod.min
|
||||
k8s.job.active_pods: k8s.job.pod.active
|
||||
k8s.job.desired_successful_pods: k8s.job.pod.desired_successful
|
||||
k8s.job.failed_pods: k8s.job.pod.failed
|
||||
k8s.job.max_parallel_pods: k8s.job.pod.max_parallel
|
||||
k8s.job.successful_pods: k8s.job.pod.successful
|
||||
k8s.node.allocatable.cpu: k8s.node.cpu.allocatable
|
||||
k8s.node.allocatable.ephemeral_storage: k8s.node.ephemeral_storage.allocatable
|
||||
k8s.node.allocatable.memory: k8s.node.memory.allocatable
|
||||
k8s.node.allocatable.pods: k8s.node.pod.allocatable
|
||||
k8s.replicaset.available_pods: k8s.replicaset.pod.available
|
||||
k8s.replicaset.desired_pods: k8s.replicaset.pod.desired
|
||||
k8s.replication_controller.available_pods: k8s.replicationcontroller.pod.available
|
||||
k8s.replication_controller.desired_pods: k8s.replicationcontroller.pod.desired
|
||||
k8s.replicationcontroller.available_pods: k8s.replicationcontroller.pod.available
|
||||
k8s.replicationcontroller.desired_pods: k8s.replicationcontroller.pod.desired
|
||||
k8s.statefulset.current_pods: k8s.statefulset.pod.current
|
||||
k8s.statefulset.desired_pods: k8s.statefulset.pod.desired
|
||||
k8s.statefulset.ready_pods: k8s.statefulset.pod.ready
|
||||
k8s.statefulset.updated_pods: k8s.statefulset.pod.updated
|
||||
v8js.heap.space.available_size: v8js.memory.heap.space.available_size
|
||||
v8js.heap.space.physical_size: v8js.memory.heap.space.physical_size
|
||||
1.37.0:
|
||||
all:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
android.state: android.app.state
|
||||
container.runtime: container.runtime.name
|
||||
enduser.role: user.roles
|
||||
gen_ai.openai.request.service_tier: openai.request.service_tier
|
||||
gen_ai.openai.response.service_tier: openai.response.service_tier
|
||||
gen_ai.openai.response.system_fingerprint: openai.response.system_fingerprint
|
||||
gen_ai.system: gen_ai.provider.name
|
||||
ios.state: ios.app.state
|
||||
1.36.0:
|
||||
1.35.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1698
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
az.namespace: azure.resource_provider.namespace
|
||||
az.service_request_id: azure.service.request.id
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/issues/1800
|
||||
- rename_metrics:
|
||||
system.network.connections: system.network.connection.count
|
||||
1.34.0:
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/2295
|
||||
- rename_metrics:
|
||||
cpu.time: system.cpu.time
|
||||
cpu.utilization: system.cpu.utilization
|
||||
cpu.frequency: system.cpu.frequency
|
||||
1.33.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1982
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
feature_flag.provider_name: feature_flag.provider.name
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1994
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
feature_flag.evaluation.error.message: error.message
|
||||
1.32.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1989
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
feature_flag.evaluation.reason: feature_flag.result.reason
|
||||
feature_flag.variant: feature_flag.result.variant
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/2042
|
||||
- rename_metrics:
|
||||
otel.sdk.span.live.count: otel.sdk.span.live
|
||||
otel.sdk.span.ended.count: otel.sdk.span.ended
|
||||
otel.sdk.processor.span.processed.count: otel.sdk.processor.span.processed
|
||||
otel.sdk.exporter.span.inflight.count: otel.sdk.exporter.span.inflight
|
||||
otel.sdk.exporter.span.exported.count: otel.sdk.exporter.span.exported
|
||||
1.31.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1880
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
android.state: android.app.state
|
||||
io.state: ios.app.state
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
k8s.replication_controller.desired_pods: k8s.replicationcontroller.desired_pods
|
||||
k8s.replication_controller.available_pods: k8s.replicationcontroller.available_pods
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1896
|
||||
- rename_metrics:
|
||||
system.cpu.time: cpu.time
|
||||
system.cpu.utilization: cpu.utilization
|
||||
system.cpu.frequency: cpu.frequency
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1896
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
system.cpu.logical_number: cpu.logical_number
|
||||
1.30.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1632
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
gen_ai.openai.request.seed: gen_ai.request.seed
|
||||
system.network.state: network.connection.state
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1624
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
code.function: code.function.name
|
||||
code.filepath: code.file.path
|
||||
code.lineno: code.line.number
|
||||
code.column: code.column.number
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1734
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.system: db.system.name
|
||||
db.cassandra.coordinator.dc: cassandra.coordinator.dc
|
||||
db.cassandra.coordinator.id: cassandra.coordinator.id
|
||||
db.cassandra.consistency_level: cassandra.consistency.level
|
||||
db.cassandra.idempotence: cassandra.query.idempotent
|
||||
db.cassandra.page_size: cassandra.page.size
|
||||
db.cassandra.speculative_execution_count: cassandra.speculative_execution.count
|
||||
db.cosmosdb.client_id: azure.client.id
|
||||
db.cosmosdb.connection_mode: azure.cosmosdb.connection.mode
|
||||
db.cosmosdb.consistency_level: azure.cosmosdb.consistency.level
|
||||
db.cosmosdb.request_charge: azure.cosmosdb.operation.request_charge
|
||||
db.cosmosdb.request_content_length: azure.cosmosdb.request.body.size
|
||||
db.cosmosdb.regions_contacted: azure.cosmosdb.operation.contacted_regions
|
||||
db.cosmosdb.sub_status_code: azure.cosmosdb.response.sub_status_code
|
||||
db.elasticsearch.node.name: elasticsearch.node.name
|
||||
# db.elasticsearch.path_parts is a template attribute, schema transformation
|
||||
# does not support it, adding as a comment for consistency
|
||||
# db.elasticsearch.path_parts.<key> -> db.operation.parameter.<key>
|
||||
metrics:
|
||||
changes:
|
||||
- rename_metrics:
|
||||
db.client.cosmosdb.operation.request_charge: azure.cosmosdb.client.operation.request_charge
|
||||
db.client.cosmosdb.active_instance.count: azure.cosmosdb.client.active_instance.count
|
||||
|
||||
1.29.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1520
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
process.executable.build_id.profiling: process.executable.build_id.htlhash
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1383
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
vcs.repository.change.id: vcs.change.id
|
||||
vcs.repository.change.title: vcs.change.title
|
||||
vcs.repository.ref.name: vcs.ref.head.name
|
||||
vcs.repository.ref.revision: vcs.ref.head.revision
|
||||
vcs.repository.ref.type: vcs.ref.head.type
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1492
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
system.device: network.interface.name
|
||||
apply_to_metrics:
|
||||
- container.network.io
|
||||
- system.network.dropped
|
||||
- system.network.errors
|
||||
- system.network.io
|
||||
- system.network.connections
|
||||
1.28.0:
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1422
|
||||
- rename_metrics:
|
||||
messaging.client.published.messages: messaging.client.sent.messages
|
||||
1.27.0:
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1216
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
tls.client.server_name: server.address
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1075
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
deployment.environment: deployment.environment.name
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1245
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.kafka.message.offset: messaging.kafka.offset
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/815
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.kafka.consumer.group: messaging.consumer.group.name
|
||||
messaging.rocketmq.client_group: messaging.consumer.group.name
|
||||
messaging.eventhubs.consumer.group: messaging.consumer.group.name
|
||||
messaging.servicebus.destination.subscription_name: messaging.destination.subscription.name
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1200
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
gen_ai.usage.completion_tokens: gen_ai.usage.output_tokens
|
||||
gen_ai.usage.prompt_tokens: gen_ai.usage.input_tokens
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1002
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.elasticsearch.cluster.name: db.namespace
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1125
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.client.connections.state: db.client.connection.state
|
||||
apply_to_metrics:
|
||||
- db.client.connection.count
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.client.connections.pool.name: db.client.connection.pool.name
|
||||
apply_to_metrics:
|
||||
- db.client.connection.count
|
||||
- db.client.connection.idle.max
|
||||
- db.client.connection.idle.min
|
||||
- db.client.connection.max
|
||||
- db.client.connection.pending_requests
|
||||
- db.client.connection.timeouts
|
||||
- db.client.connection.create_time
|
||||
- db.client.connection.wait_time
|
||||
- db.client.connection.use_time
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1006
|
||||
- rename_metrics:
|
||||
messaging.publish.messages: messaging.client.published.messages
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1026
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
system.cpu.state: cpu.mode
|
||||
process.cpu.state: cpu.mode
|
||||
container.cpu.state: cpu.mode
|
||||
apply_to_metrics:
|
||||
- system.cpu.time
|
||||
- system.cpu.utilization
|
||||
- process.cpu.time
|
||||
- process.cpu.utilization
|
||||
- container.cpu.time
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/1265
|
||||
- rename_metrics:
|
||||
jvm.buffer.memory.usage: jvm.buffer.memory.used
|
||||
1.26.0:
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/966
|
||||
- rename_metrics:
|
||||
db.client.connections.usage: db.client.connection.count
|
||||
db.client.connections.idle.max: db.client.connection.idle.max
|
||||
db.client.connections.idle.min: db.client.connection.idle.min
|
||||
db.client.connections.max: db.client.connection.max
|
||||
db.client.connections.pending_requests: db.client.connection.pending_requests
|
||||
db.client.connections.timeouts: db.client.connection.timeouts
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/948
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.client_id: messaging.client.id
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/909
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
state: db.client.connections.state
|
||||
apply_to_metrics:
|
||||
- db.client.connections.usage
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
pool.name: db.client.connections.pool.name
|
||||
apply_to_metrics:
|
||||
- db.client.connections.usage
|
||||
- db.client.connections.idle.max
|
||||
- db.client.connections.idle.min
|
||||
- db.client.connections.max
|
||||
- db.client.connections.pending_requests
|
||||
- db.client.connections.timeouts
|
||||
- db.client.connections.create_time
|
||||
- db.client.connections.wait_time
|
||||
- db.client.connections.use_time
|
||||
all:
|
||||
changes:
|
||||
# https://github:com/open-telemetry/semantic-conventions/pull/731/
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
enduser.id: user.id
|
||||
|
||||
1.25.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/911
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.name: db.namespace
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/870
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.sql.table: db.collection.name
|
||||
db.mongodb.collection: db.collection.name
|
||||
db.cosmosdb.container: db.collection.name
|
||||
db.cassandra.table: db.collection.name
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/798
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.kafka.destination.partition: messaging.destination.partition.id
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/875
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.operation: db.operation.name
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/913
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.operation: messaging.operation.type
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/866
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.statement: db.query.text
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/484
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
system.processes.status: system.process.status
|
||||
apply_to_metrics:
|
||||
- system.processes.count
|
||||
- rename_metrics:
|
||||
system.processes.count: system.process.count
|
||||
system.processes.created: system.process.created
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/625
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
container.labels: container.label
|
||||
k8s.pod.labels: k8s.pod.label
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/330
|
||||
- rename_metrics:
|
||||
process.threads: process.thread.count
|
||||
process.open_file_descriptors: process.open_file_descriptor.count
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
state: process.cpu.state
|
||||
apply_to_metrics:
|
||||
- process.cpu.time
|
||||
- process.cpu.utilization
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
direction: disk.io.direction
|
||||
apply_to_metrics:
|
||||
- process.disk.io
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
type: process.context_switch_type
|
||||
apply_to_metrics:
|
||||
- process.context_switches
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
direction: network.io.direction
|
||||
apply_to_metrics:
|
||||
- process.network.io
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
type: process.paging.fault_type
|
||||
apply_to_metrics:
|
||||
- process.paging.faults
|
||||
all:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/854
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
message.type: rpc.message.type
|
||||
message.id: rpc.message.id
|
||||
message.compressed_size: rpc.message.compressed_size
|
||||
message.uncompressed_size: rpc.message.uncompressed_size
|
||||
|
||||
1.24.0:
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/536
|
||||
- rename_metrics:
|
||||
jvm.memory.usage: jvm.memory.used
|
||||
jvm.memory.usage_after_last_gc: jvm.memory.used_after_last_gc
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/530
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
system.network.io.direction: network.io.direction
|
||||
system.disk.io.direction: disk.io.direction
|
||||
1.23.1:
|
||||
1.23.0:
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/20
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
thread.daemon: jvm.thread.daemon
|
||||
apply_to_metrics:
|
||||
- jvm.thread.count
|
||||
1.22.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/229
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.message.payload_size_bytes: messaging.message.body.size
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/374
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
http.resend_count: http.request.resend_count
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/224
|
||||
- rename_metrics:
|
||||
http.client.duration: http.client.request.duration
|
||||
http.server.duration: http.server.request.duration
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/241
|
||||
- rename_metrics:
|
||||
process.runtime.jvm.memory.usage: jvm.memory.usage
|
||||
process.runtime.jvm.memory.committed: jvm.memory.committed
|
||||
process.runtime.jvm.memory.limit: jvm.memory.limit
|
||||
process.runtime.jvm.memory.usage_after_last_gc: jvm.memory.usage_after_last_gc
|
||||
process.runtime.jvm.gc.duration: jvm.gc.duration
|
||||
# also https://github.com/open-telemetry/semantic-conventions/pull/252
|
||||
process.runtime.jvm.threads.count: jvm.thread.count
|
||||
# also https://github.com/open-telemetry/semantic-conventions/pull/252
|
||||
process.runtime.jvm.classes.loaded: jvm.class.loaded
|
||||
# also https://github.com/open-telemetry/semantic-conventions/pull/252
|
||||
process.runtime.jvm.classes.unloaded: jvm.class.unloaded
|
||||
# also https://github.com/open-telemetry/semantic-conventions/pull/252
|
||||
# and https://github.com/open-telemetry/semantic-conventions/pull/60
|
||||
process.runtime.jvm.classes.current_loaded: jvm.class.count
|
||||
process.runtime.jvm.cpu.time: jvm.cpu.time
|
||||
process.runtime.jvm.cpu.recent_utilization: jvm.cpu.recent_utilization
|
||||
process.runtime.jvm.memory.init: jvm.memory.init
|
||||
process.runtime.jvm.system.cpu.utilization: jvm.system.cpu.utilization
|
||||
process.runtime.jvm.system.cpu.load_1m: jvm.system.cpu.load_1m
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/253
|
||||
process.runtime.jvm.buffer.usage: jvm.buffer.memory.usage
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/253
|
||||
process.runtime.jvm.buffer.limit: jvm.buffer.memory.limit
|
||||
process.runtime.jvm.buffer.count: jvm.buffer.count
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/20
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
type: jvm.memory.type
|
||||
pool: jvm.memory.pool.name
|
||||
apply_to_metrics:
|
||||
- jvm.memory.usage
|
||||
- jvm.memory.committed
|
||||
- jvm.memory.limit
|
||||
- jvm.memory.usage_after_last_gc
|
||||
- jvm.memory.init
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
name: jvm.gc.name
|
||||
action: jvm.gc.action
|
||||
apply_to_metrics:
|
||||
- jvm.gc.duration
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
daemon: thread.daemon
|
||||
apply_to_metrics:
|
||||
- jvm.threads.count
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
pool: jvm.buffer.pool.name
|
||||
apply_to_metrics:
|
||||
- jvm.buffer.memory.usage
|
||||
- jvm.buffer.memory.limit
|
||||
- jvm.buffer.count
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/89
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
state: system.cpu.state
|
||||
cpu: system.cpu.logical_number
|
||||
apply_to_metrics:
|
||||
- system.cpu.time
|
||||
- system.cpu.utilization
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
state: system.memory.state
|
||||
apply_to_metrics:
|
||||
- system.memory.usage
|
||||
- system.memory.utilization
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
state: system.paging.state
|
||||
apply_to_metrics:
|
||||
- system.paging.usage
|
||||
- system.paging.utilization
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
type: system.paging.type
|
||||
direction: system.paging.direction
|
||||
apply_to_metrics:
|
||||
- system.paging.faults
|
||||
- system.paging.operations
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
device: system.device
|
||||
direction: system.disk.direction
|
||||
apply_to_metrics:
|
||||
- system.disk.io
|
||||
- system.disk.operations
|
||||
- system.disk.io_time
|
||||
- system.disk.operation_time
|
||||
- system.disk.merged
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
device: system.device
|
||||
state: system.filesystem.state
|
||||
type: system.filesystem.type
|
||||
mode: system.filesystem.mode
|
||||
mountpoint: system.filesystem.mountpoint
|
||||
apply_to_metrics:
|
||||
- system.filesystem.usage
|
||||
- system.filesystem.utilization
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
device: system.device
|
||||
direction: system.network.direction
|
||||
protocol: network.protocol
|
||||
state: system.network.state
|
||||
apply_to_metrics:
|
||||
- system.network.dropped
|
||||
- system.network.packets
|
||||
- system.network.errors
|
||||
- system.network.io
|
||||
- system.network.connections
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
status: system.processes.status
|
||||
apply_to_metrics:
|
||||
- system.processes.count
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/247
|
||||
- rename_metrics:
|
||||
http.server.request.size: http.server.request.body.size
|
||||
http.server.response.size: http.server.response.body.size
|
||||
resources:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/178
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
telemetry.auto.version: telemetry.distro.version
|
||||
1.21.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3336
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.kafka.client_id: messaging.client_id
|
||||
messaging.rocketmq.client_id: messaging.client_id
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3402
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
# net.peer.(name|port) attributes were usually populated on client side
|
||||
# so they should be usually translated to server.(address|port)
|
||||
# net.host.* attributes were only populated on server side
|
||||
net.host.name: server.address
|
||||
net.host.port: server.port
|
||||
# was only populated on client side
|
||||
net.sock.peer.name: server.socket.domain
|
||||
# net.sock.peer.(addr|port) mapping is not possible
|
||||
# since they applied to both client and server side
|
||||
# were only populated on server side
|
||||
net.sock.host.addr: server.socket.address
|
||||
net.sock.host.port: server.socket.port
|
||||
http.client_ip: client.address
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3426
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
net.protocol.name: network.protocol.name
|
||||
net.protocol.version: network.protocol.version
|
||||
net.host.connection.type: network.connection.type
|
||||
net.host.connection.subtype: network.connection.subtype
|
||||
net.host.carrier.name: network.carrier.name
|
||||
net.host.carrier.mcc: network.carrier.mcc
|
||||
net.host.carrier.mnc: network.carrier.mnc
|
||||
net.host.carrier.icc: network.carrier.icc
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3355
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
http.method: http.request.method
|
||||
http.status_code: http.response.status_code
|
||||
http.scheme: url.scheme
|
||||
http.url: url.full
|
||||
http.request_content_length: http.request.body.size
|
||||
http.response_content_length: http.response.body.size
|
||||
metrics:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/semantic-conventions/pull/53
|
||||
- rename_metrics:
|
||||
process.runtime.jvm.cpu.utilization: process.runtime.jvm.cpu.recent_utilization
|
||||
1.20.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3272
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
net.app.protocol.name: net.protocol.name
|
||||
net.app.protocol.version: net.protocol.version
|
||||
1.19.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3209
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
faas.execution: faas.invocation_id
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3188
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
faas.id: cloud.resource_id
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3190
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
http.user_agent: user_agent.original
|
||||
resources:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/3190
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
browser.user_agent: user_agent.original
|
||||
1.18.0:
|
||||
1.17.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/2957
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
messaging.consumer_id: messaging.consumer.id
|
||||
messaging.protocol: net.app.protocol.name
|
||||
messaging.protocol_version: net.app.protocol.version
|
||||
messaging.destination: messaging.destination.name
|
||||
messaging.temp_destination: messaging.destination.temporary
|
||||
messaging.destination_kind: messaging.destination.kind
|
||||
messaging.message_id: messaging.message.id
|
||||
messaging.conversation_id: messaging.message.conversation_id
|
||||
messaging.message_payload_size_bytes: messaging.message.payload_size_bytes
|
||||
messaging.message_payload_compressed_size_bytes: messaging.message.payload_compressed_size_bytes
|
||||
messaging.rabbitmq.routing_key: messaging.rabbitmq.destination.routing_key
|
||||
messaging.kafka.message_key: messaging.kafka.message.key
|
||||
messaging.kafka.partition: messaging.kafka.destination.partition
|
||||
messaging.kafka.tombstone: messaging.kafka.message.tombstone
|
||||
messaging.rocketmq.message_type: messaging.rocketmq.message.type
|
||||
messaging.rocketmq.message_tag: messaging.rocketmq.message.tag
|
||||
messaging.rocketmq.message_keys: messaging.rocketmq.message.keys
|
||||
messaging.kafka.consumer_group: messaging.kafka.consumer.group
|
||||
1.16.0:
|
||||
1.15.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/2743
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
http.retry_count: http.resend_count
|
||||
1.14.0:
|
||||
1.13.0:
|
||||
spans:
|
||||
changes:
|
||||
# https://github.com/open-telemetry/opentelemetry-specification/pull/2614
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
net.peer.ip: net.sock.peer.addr
|
||||
net.host.ip: net.sock.host.addr
|
||||
1.12.0:
|
||||
1.11.0:
|
||||
1.10.0:
|
||||
1.9.0:
|
||||
1.8.0:
|
||||
spans:
|
||||
changes:
|
||||
- rename_attributes:
|
||||
attribute_map:
|
||||
db.cassandra.keyspace: db.name
|
||||
db.hbase.namespace: db.name
|
||||
1.7.0:
|
||||
1.6.1:
|
||||
1.5.0:
|
||||
1.4.0:
|
||||
157
tests/integration/tests/queriersemconv/02_cross_signal.py
Normal file
157
tests/integration/tests/queriersemconv/02_cross_signal.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""Phase 2 semantic-convention checks across logs and metrics."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.metrics import Metrics
|
||||
|
||||
DB_CURRENT = "db.system.name"
|
||||
DB_OLD = "db.system"
|
||||
METRIC_CURRENT = "container.cpu.usage"
|
||||
METRIC_OLD = "container.cpu.utilization"
|
||||
PREFIX = "semconv-phase2"
|
||||
|
||||
|
||||
def _raw_log_bodies(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
now: datetime,
|
||||
expression: str,
|
||||
) -> set[str]:
|
||||
response = querier.make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=querier.RequestType.RAW,
|
||||
queries=[querier.build_raw_query("A", "logs", limit=100, filter_expression=expression)],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
return {row["data"]["body"] for row in querier.get_rows(response)}
|
||||
|
||||
|
||||
def _field_values(signoz: types.SigNoz, token: str, signal: str, name: str, context: str) -> set[str]:
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": signal,
|
||||
"name": name,
|
||||
"fieldContext": context,
|
||||
"fieldDataType": "string",
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
return set(response.json()["data"]["values"].get("stringValues") or [])
|
||||
|
||||
|
||||
def test_logs_resolve_db_system_family(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(minutes=1)
|
||||
rows = [
|
||||
("old", {DB_OLD: "postgresql"}),
|
||||
("current", {DB_CURRENT: "postgresql"}),
|
||||
("conflict", {DB_OLD: "mysql", DB_CURRENT: "postgresql"}),
|
||||
("missing", {}),
|
||||
]
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now + timedelta(seconds=index),
|
||||
resources={"service.name": PREFIX, **attributes},
|
||||
attributes=attributes,
|
||||
body=f"{PREFIX}-{suffix}",
|
||||
)
|
||||
for index, (suffix, attributes) in enumerate(rows)
|
||||
]
|
||||
)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
present = {f"{PREFIX}-old", f"{PREFIX}-current", f"{PREFIX}-conflict"}
|
||||
for context in ("attribute", "resource"):
|
||||
for requested in (DB_CURRENT, DB_OLD):
|
||||
field = f"{context}.{requested}"
|
||||
assert _raw_log_bodies(signoz, token, now, f'{field} = "postgresql"') == present
|
||||
assert _raw_log_bodies(signoz, token, now, f"{field} EXISTS") == present
|
||||
assert _raw_log_bodies(signoz, token, now, f"{field} NOT EXISTS") == {f"{PREFIX}-missing"}
|
||||
assert _field_values(signoz, token, "logs", requested, context) == {"postgresql", "mysql"}
|
||||
|
||||
|
||||
def test_metrics_resolve_label_and_metric_name_families(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(
|
||||
[
|
||||
Metrics(
|
||||
metric_name=METRIC_CURRENT,
|
||||
labels={DB_CURRENT: "postgresql"},
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=10,
|
||||
),
|
||||
Metrics(
|
||||
metric_name=METRIC_OLD,
|
||||
labels={"db_system": "mysql"},
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=20,
|
||||
),
|
||||
Metrics(
|
||||
metric_name=METRIC_OLD,
|
||||
labels={DB_OLD: "mysql", DB_CURRENT: "postgresql", "series": "conflict"},
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=30,
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
for metric_name in (METRIC_CURRENT, METRIC_OLD):
|
||||
for requested_label in (DB_CURRENT, DB_OLD):
|
||||
response = querier.make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
querier.build_scalar_query(
|
||||
name="A",
|
||||
signal="metrics",
|
||||
aggregations=[
|
||||
querier.build_metrics_aggregation(
|
||||
metric_name,
|
||||
"latest",
|
||||
"sum",
|
||||
"unspecified",
|
||||
reduce_to="last",
|
||||
)
|
||||
],
|
||||
group_by=[querier.build_group_by_field(requested_label, "string", "attribute")],
|
||||
filter_expression=f"attribute.{requested_label} EXISTS",
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
data = {row[0]: row[-1] for row in querier.get_scalar_table_data(response.json())}
|
||||
assert data == {"postgresql": 40.0, "mysql": 20.0}, (metric_name, requested_label, data)
|
||||
239
tests/integration/tests/queriertraces/13_semconv_evolution.py
Normal file
239
tests/integration/tests/queriertraces/13_semconv_evolution.py
Normal file
@@ -0,0 +1,239 @@
|
||||
"""Phase 1 end-to-end checks for semantic-convention name evolution.
|
||||
|
||||
The fixture models a fleet split across SDK generations and deliberately includes
|
||||
a dual-emitting conflict. Both request spellings must address one logical field,
|
||||
with the current spelling winning when a row contains both.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Generator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import Aggregation, BuilderQuery, OrderBy, RequestType, TelemetryFieldKey, make_query_request
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
|
||||
|
||||
CURRENT = "deployment.environment.name"
|
||||
OLD = "deployment.environment"
|
||||
PREFIX = "semconv-phase1"
|
||||
|
||||
PRODUCTION_SPANS = {
|
||||
f"{PREFIX}-old",
|
||||
f"{PREFIX}-current",
|
||||
f"{PREFIX}-both",
|
||||
f"{PREFIX}-conflict",
|
||||
}
|
||||
STAGING_SPANS = {f"{PREFIX}-staging"}
|
||||
MISSING_SPANS = {f"{PREFIX}-missing"}
|
||||
|
||||
|
||||
def _span(timestamp: datetime, suffix: str, environment: dict[str, str]) -> Traces:
|
||||
service = f"{PREFIX}-{suffix}"
|
||||
return Traces(
|
||||
timestamp=timestamp,
|
||||
duration=timedelta(milliseconds=10),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
name=service,
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": service, **environment},
|
||||
attributes=dict(environment),
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="semconv_phase1_data")
|
||||
def semconv_phase1_data(
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
clickhouse: types.TestContainerClickhouse,
|
||||
) -> Generator[datetime]:
|
||||
now = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(minutes=2)
|
||||
insert_traces(
|
||||
[
|
||||
_span(now - timedelta(seconds=5), "old", {OLD: "production"}),
|
||||
_span(now - timedelta(seconds=4), "current", {CURRENT: "production"}),
|
||||
_span(now - timedelta(seconds=3), "both", {OLD: "production", CURRENT: "production"}),
|
||||
_span(now - timedelta(seconds=2), "conflict", {OLD: "staging", CURRENT: "production"}),
|
||||
_span(now - timedelta(seconds=1), "staging", {OLD: "staging"}),
|
||||
_span(now, "missing", {}),
|
||||
]
|
||||
)
|
||||
|
||||
# Service-map rows are derived by the collector in production. Seed the
|
||||
# derived table directly here so the backend alias allowlist is tested in
|
||||
# isolation; the collector repository owns its write-path integration test.
|
||||
for environment, suffix in (("production", "production"), ("staging", "staging")):
|
||||
clickhouse.conn.command(
|
||||
f"""
|
||||
INSERT INTO signoz_traces.distributed_dependency_graph_minutes_v2
|
||||
(src, dest, duration_quantiles_state, error_count, total_count, timestamp,
|
||||
deployment_environment, k8s_cluster_name, k8s_namespace_name)
|
||||
SELECT
|
||||
'{PREFIX}-map-{suffix}', '{PREFIX}-map-child',
|
||||
quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(1000000)),
|
||||
toUInt64(0), toUInt64(1), toDateTime({int(now.timestamp())}),
|
||||
'{environment}', '', ''
|
||||
"""
|
||||
)
|
||||
|
||||
yield now
|
||||
|
||||
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
|
||||
clickhouse.conn.command(
|
||||
f"ALTER TABLE signoz_traces.dependency_graph_minutes_v2 ON CLUSTER '{cluster}' "
|
||||
f"DELETE WHERE startsWith(src, '{PREFIX}-map-') SETTINGS mutations_sync = 1"
|
||||
)
|
||||
|
||||
|
||||
def _result(response: requests.Response) -> dict[str, Any]:
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
results = response.json()["data"]["data"]["results"]
|
||||
assert len(results) == 1
|
||||
return results[0]
|
||||
|
||||
|
||||
def _raw_names(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
now: datetime,
|
||||
expression: str,
|
||||
) -> set[str]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.RAW,
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="traces",
|
||||
name="A",
|
||||
limit=100,
|
||||
filter_expression=expression,
|
||||
select_fields=[TelemetryFieldKey("span.name")],
|
||||
order=[OrderBy(TelemetryFieldKey("timestamp"), "asc")],
|
||||
).to_dict()
|
||||
],
|
||||
)
|
||||
return {row["data"]["name"] for row in (_result(response).get("rows") or [])}
|
||||
|
||||
|
||||
def _metadata_values(signoz: types.SigNoz, token: str, name: str, context: str) -> set[str]:
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": "traces",
|
||||
"name": name,
|
||||
"fieldContext": context,
|
||||
"fieldDataType": "string",
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
return set(response.json()["data"]["values"].get("stringValues") or [])
|
||||
|
||||
|
||||
def test_semconv_phase1_mixed_sdk_generations( # pylint: disable=too-many-statements
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
semconv_phase1_data: datetime,
|
||||
) -> None:
|
||||
now = semconv_phase1_data
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Resource and span-attribute paths share the same matrix. Run every
|
||||
# operator with both the saved-query (old) and current request spellings.
|
||||
for context in ("resource", "attribute"):
|
||||
for requested in (CURRENT, OLD):
|
||||
field = f"{context}.{requested}"
|
||||
assert _raw_names(signoz, token, now, f"{field} = 'production'") == PRODUCTION_SPANS
|
||||
assert _raw_names(signoz, token, now, f"{field} = 'staging'") == STAGING_SPANS
|
||||
assert _raw_names(signoz, token, now, f"{field} != 'production'") == STAGING_SPANS
|
||||
assert _raw_names(signoz, token, now, f"{field} EXISTS") == PRODUCTION_SPANS | STAGING_SPANS
|
||||
assert _raw_names(signoz, token, now, f"{field} NOT EXISTS") == MISSING_SPANS
|
||||
|
||||
grouped = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.SCALAR,
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="traces",
|
||||
name="A",
|
||||
filter_expression=f"{field} EXISTS",
|
||||
aggregations=[Aggregation("count()")],
|
||||
group_by=[TelemetryFieldKey(requested, "string", context)],
|
||||
order=[OrderBy(TelemetryFieldKey(requested, "string", context), "asc")],
|
||||
).to_dict()
|
||||
],
|
||||
)
|
||||
result = _result(grouped)
|
||||
assert result["columns"][0]["name"] == requested, "response identity must match the request spelling"
|
||||
assert result["data"] == [["production", 4], ["staging", 1]]
|
||||
|
||||
assert _metadata_values(signoz, token, CURRENT, context) == {"production", "staging"}
|
||||
assert _metadata_values(signoz, token, OLD, context) == {"production", "staging"}
|
||||
|
||||
keys_response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": "traces", "searchText": OLD},
|
||||
)
|
||||
assert keys_response.status_code == HTTPStatus.OK, keys_response.text
|
||||
keys = keys_response.json()["data"]["keys"]
|
||||
assert CURRENT in keys
|
||||
assert OLD not in keys
|
||||
|
||||
start_ns = str(int((now - timedelta(minutes=2)).timestamp() * 1_000_000_000))
|
||||
end_ns = str(int((now + timedelta(minutes=1)).timestamp() * 1_000_000_000))
|
||||
for requested in (CURRENT, OLD):
|
||||
services_response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v2/services"),
|
||||
timeout=30,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"tags": [
|
||||
{
|
||||
"Key": requested,
|
||||
"Operator": "In",
|
||||
"StringValues": ["production"],
|
||||
"TagType": "ResourceAttribute",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert services_response.status_code == HTTPStatus.OK, services_response.text
|
||||
services = {item["serviceName"] for item in services_response.json()["data"]}
|
||||
assert services == PRODUCTION_SPANS
|
||||
|
||||
map_response = requests.post(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/dependency_graph"),
|
||||
timeout=30,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
json={
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"tags": [
|
||||
{
|
||||
"key": requested,
|
||||
"operator": "In",
|
||||
"stringValues": ["production"],
|
||||
"tagType": "ResourceAttribute",
|
||||
}
|
||||
],
|
||||
},
|
||||
)
|
||||
assert map_response.status_code == HTTPStatus.OK, map_response.text
|
||||
assert {edge["parent"] for edge in map_response.json()} == {f"{PREFIX}-map-production"}
|
||||
Reference in New Issue
Block a user