mirror of
https://github.com/SigNoz/signoz.git
synced 2026-09-09 04:50:40 +01:00
Compare commits
2 Commits
feat/story
...
test/expli
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
28bb4939c1 | ||
|
|
d5af6f6d6b |
@@ -34,6 +34,7 @@ export function mockFieldsValuesAPI(response: {
|
||||
relatedValues?: (string | null)[];
|
||||
stringValues?: (string | null)[];
|
||||
numberValues?: (number | null)[];
|
||||
boolValues?: (boolean | null)[];
|
||||
}): void {
|
||||
server.use(
|
||||
rest.get('http://localhost/api/v1/fields/values', (_, res, ctx) =>
|
||||
@@ -46,6 +47,7 @@ export function mockFieldsValuesAPI(response: {
|
||||
relatedValues: response.relatedValues ?? [],
|
||||
stringValues: response.stringValues ?? [],
|
||||
numberValues: response.numberValues ?? [],
|
||||
boolValues: response.boolValues ?? [],
|
||||
},
|
||||
},
|
||||
}),
|
||||
|
||||
@@ -92,8 +92,12 @@ export function useFieldValues({
|
||||
values.numberValues
|
||||
?.filter((value): value is number => value !== null && value !== undefined)
|
||||
.map((value) => value.toString()) || [];
|
||||
const boolValues =
|
||||
values.boolValues
|
||||
?.filter((value): value is boolean => value !== null && value !== undefined)
|
||||
.map((value) => value.toString()) || [];
|
||||
|
||||
return [...stringValues, ...numberValues];
|
||||
return [...stringValues, ...numberValues, ...boolValues];
|
||||
}, [data]);
|
||||
|
||||
return { relatedValues, allValues, isLoading, isFetching };
|
||||
|
||||
@@ -8,6 +8,7 @@ import { FiltersType, IQuickFiltersConfig, SignalType } from './types';
|
||||
const FILTER_TITLE_MAP: Record<string, string> = {
|
||||
duration_nano: 'Duration',
|
||||
hasError: 'Has Error (Status)',
|
||||
has_error: 'Has Error (Status)',
|
||||
};
|
||||
|
||||
const FILTER_TYPE_MAP: Record<string, FiltersType> = {
|
||||
|
||||
@@ -82,6 +82,7 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{
|
||||
StringValues: allValues.StringValues,
|
||||
BoolValues: allValues.BoolValues,
|
||||
NumberValues: allValues.NumberValues,
|
||||
RelatedValues: relatedValues,
|
||||
}
|
||||
|
||||
@@ -253,6 +253,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewAddQuickFilterTuplesFactory(sqlstore),
|
||||
sqlmigration.NewAddIngestionTuplesFactory(sqlstore),
|
||||
sqlmigration.NewAddSubscriptionTuplesFactory(sqlstore),
|
||||
sqlmigration.NewNormalizeQuickFilterFieldsFactory(sqlstore),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
161
pkg/sqlmigration/126_normalize_quick_filter_fields.go
Normal file
161
pkg/sqlmigration/126_normalize_quick_filter_fields.go
Normal file
@@ -0,0 +1,161 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
)
|
||||
|
||||
type quickFilterSourceRow struct {
|
||||
bun.BaseModel `bun:"table:quick_filter"`
|
||||
|
||||
ID string `bun:"id,pk"`
|
||||
Source string `bun:"source"`
|
||||
Filter string `bun:"filter"`
|
||||
}
|
||||
|
||||
type quickFilterStaticField struct {
|
||||
name string
|
||||
fieldContext string
|
||||
fieldDataType string
|
||||
}
|
||||
|
||||
// quickFilterSpanFields are the span-level fields the fields API serves with
|
||||
// the span context, keyed by every name a stored filter may carry for them.
|
||||
var quickFilterSpanFields = func() map[string]quickFilterStaticField {
|
||||
fields := map[string]quickFilterStaticField{}
|
||||
for name, dataType := range map[string]string{
|
||||
"trace_id": "string", "span_id": "string", "trace_state": "string", "parent_span_id": "string",
|
||||
"flags": "number", "name": "string", "kind": "number", "kind_string": "string",
|
||||
"duration_nano": "number", "status_code": "number", "status_message": "string", "status_code_string": "string",
|
||||
"response_status_code": "string", "external_http_url": "string", "http_url": "string",
|
||||
"external_http_method": "string", "http_method": "string", "http_host": "string",
|
||||
"db_name": "string", "db_operation": "string", "has_error": "bool", "is_remote": "string",
|
||||
} {
|
||||
fields[name] = quickFilterStaticField{name: name, fieldContext: "span", fieldDataType: dataType}
|
||||
}
|
||||
for deprecated, current := range map[string]string{
|
||||
"responseStatusCode": "response_status_code", "externalHttpUrl": "external_http_url", "httpUrl": "http_url",
|
||||
"externalHttpMethod": "external_http_method", "httpMethod": "http_method", "httpHost": "http_host",
|
||||
"dbName": "db_name", "dbOperation": "db_operation", "hasError": "has_error", "isRemote": "is_remote",
|
||||
} {
|
||||
fields[deprecated] = fields[current]
|
||||
}
|
||||
return fields
|
||||
}()
|
||||
|
||||
// quickFilterLogFields are the log-level fields the fields API serves with
|
||||
// the log context.
|
||||
var quickFilterLogFields = map[string]quickFilterStaticField{
|
||||
"body": {name: "body", fieldContext: "log", fieldDataType: "string"},
|
||||
"severity_text": {name: "severity_text", fieldContext: "log", fieldDataType: "string"},
|
||||
"severity_number": {name: "severity_number", fieldContext: "log", fieldDataType: "number"},
|
||||
"trace_id": {name: "trace_id", fieldContext: "log", fieldDataType: "string"},
|
||||
"span_id": {name: "span_id", fieldContext: "log", fieldDataType: "string"},
|
||||
"trace_flags": {name: "trace_flags", fieldContext: "log", fieldDataType: "number"},
|
||||
}
|
||||
|
||||
type normalizeQuickFilterFields struct {
|
||||
settings factory.ProviderSettings
|
||||
}
|
||||
|
||||
func NewNormalizeQuickFilterFieldsFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("normalize_quick_filter_fields"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return &normalizeQuickFilterFields{settings: ps}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (migration *normalizeQuickFilterFields) Register(migrations *migrate.Migrations) error {
|
||||
return migrations.Register(migration.Up, migration.Down)
|
||||
}
|
||||
|
||||
func (migration *normalizeQuickFilterFields) Up(ctx context.Context, db *bun.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
var rows []*quickFilterSourceRow
|
||||
if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var migrated, skipped int
|
||||
for _, row := range rows {
|
||||
normalized, changed, ok := normalizeQuickFilterEntries(row.Source, row.Filter)
|
||||
if !ok {
|
||||
migration.settings.Logger.WarnContext(ctx, "quick filter could not be parsed, leaving it untouched", slog.String("quick_filter_id", row.ID), slog.String("raw_filter", row.Filter))
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
|
||||
migrated++
|
||||
if _, err := tx.NewUpdate().Model((*quickFilterSourceRow)(nil)).Set("filter = ?", normalized).Where("id = ?", row.ID).Exec(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
migration.settings.Logger.InfoContext(ctx, "normalized quick filter static fields", slog.Int("total", len(rows)), slog.Int("migrated", migrated), slog.Int("skipped", skipped))
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (migration *normalizeQuickFilterFields) Down(context.Context, *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizeQuickFilterEntries rewrites the static fields of a stored filter
|
||||
// list to the name, context and data type the fields API serves them with:
|
||||
// span fields for the trace-based sources, log fields for logs, whatever
|
||||
// context the legacy seeds gave them. Other keys are left as they are;
|
||||
// ok=false means unparseable.
|
||||
func normalizeQuickFilterEntries(source string, filter string) (normalized string, changed bool, ok bool) {
|
||||
var staticFields map[string]quickFilterStaticField
|
||||
switch source {
|
||||
case "traces", "api_monitoring", "exceptions", "ai_observability":
|
||||
staticFields = quickFilterSpanFields
|
||||
case "logs":
|
||||
staticFields = quickFilterLogFields
|
||||
default:
|
||||
return "", false, true
|
||||
}
|
||||
|
||||
var entries []telemetryFieldKeyOutput
|
||||
if err := json.Unmarshal([]byte(filter), &entries); err != nil {
|
||||
return "", false, false
|
||||
}
|
||||
|
||||
for i, entry := range entries {
|
||||
field, static := staticFields[entry.Name]
|
||||
if !static {
|
||||
continue
|
||||
}
|
||||
if entry.Name == field.name && entry.FieldContext == field.fieldContext && entry.FieldDataType == field.fieldDataType {
|
||||
continue
|
||||
}
|
||||
entries[i].Name = field.name
|
||||
entries[i].FieldContext = field.fieldContext
|
||||
entries[i].FieldDataType = field.fieldDataType
|
||||
changed = true
|
||||
}
|
||||
|
||||
if !changed {
|
||||
return "", false, true
|
||||
}
|
||||
|
||||
normalizedJSON, err := marshalUnescaped(entries)
|
||||
if err != nil {
|
||||
return "", false, false
|
||||
}
|
||||
return string(normalizedJSON), true, true
|
||||
}
|
||||
61
pkg/telemetrymetadata/bool_values.go
Normal file
61
pkg/telemetrymetadata/bool_values.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package telemetrymetadata
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/tracestelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// boolFieldValues is the suggestion set for a bool field, optionally narrowed
|
||||
// by the search text.
|
||||
func boolFieldValues(searchText string) *telemetrytypes.TelemetryFieldValues {
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
needle := strings.ToLower(searchText)
|
||||
for _, v := range []bool{true, false} {
|
||||
if needle == "" || strings.Contains(strconv.FormatBool(v), needle) {
|
||||
values.BoolValues = append(values.BoolValues, v)
|
||||
}
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// spanSearchScopeFieldValues is the suggestion set for a search-scope selector
|
||||
// (isRoot, isEntryPoint), which only filters with true. ok is false for any
|
||||
// other name.
|
||||
func spanSearchScopeFieldValues(name, searchText string) (*telemetrytypes.TelemetryFieldValues, bool) {
|
||||
for scopeName := range tracestelemetryschema.SpanSearchScopeFields {
|
||||
if !strings.EqualFold(scopeName, name) {
|
||||
continue
|
||||
}
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
if needle := strings.ToLower(searchText); needle == "" || strings.Contains("true", needle) {
|
||||
values.BoolValues = []bool{true}
|
||||
}
|
||||
return values, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// isKnownBoolField is true when the caller asked for the bool data type, or
|
||||
// when the name is one of the signal's static bool fields and the requested
|
||||
// context does not rule that static field out.
|
||||
func isKnownBoolField(selector *telemetrytypes.FieldValueSelector, staticFields ...map[string]telemetrytypes.TelemetryFieldKey) bool {
|
||||
if selector.FieldDataType == telemetrytypes.FieldDataTypeBool {
|
||||
return true
|
||||
}
|
||||
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
|
||||
return false
|
||||
}
|
||||
for _, fields := range staticFields {
|
||||
field, ok := fields[selector.Name]
|
||||
if !ok || field.FieldDataType != telemetrytypes.FieldDataTypeBool {
|
||||
continue
|
||||
}
|
||||
if selector.FieldContext == telemetrytypes.FieldContextUnspecified || selector.FieldContext == field.FieldContext {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -187,8 +187,6 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
).From(t.tracesDBName + "." + t.spanAttributesKeysTblName)
|
||||
var limit int
|
||||
|
||||
searchTexts := []string{}
|
||||
|
||||
conds := []string{}
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
|
||||
@@ -208,14 +206,12 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
|
||||
}
|
||||
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
// now look at the field context
|
||||
// we don't write most of intrinsic fields to keys table
|
||||
// for this reason we don't want to apply tagType if the field context
|
||||
// is not attribute or resource attribute
|
||||
if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified &&
|
||||
(fieldKeySelector.FieldContext == telemetrytypes.FieldContextAttribute ||
|
||||
fieldKeySelector.FieldContext == telemetrytypes.FieldContextResource) {
|
||||
// is not attribute, resource attribute or scope
|
||||
switch fieldKeySelector.FieldContext {
|
||||
case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope:
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("tagType", fieldKeySelector.FieldContext.TagType()))
|
||||
}
|
||||
|
||||
@@ -288,41 +284,20 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{"isRoot", "isEntryPoint"}
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
|
||||
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
|
||||
// Add the matching static fields: the span scope selectors, the intrinsic
|
||||
// columns and the calculated columns. These don't count towards the limit
|
||||
staticFields := maps.Values(tracestelemetryschema.SpanSearchScopeFields)
|
||||
staticFields = append(staticFields, maps.Values(tracestelemetryschema.IntrinsicFields)...)
|
||||
staticFields = append(staticFields, maps.Values(tracestelemetryschema.CalculatedFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
if v == "" || strings.Contains(key, v) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
for _, field := range staticFields {
|
||||
if !staticFieldMatchesAny(field, fieldKeySelectors) {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
})
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
|
||||
if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
|
||||
@@ -542,12 +517,6 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
allArgs = append(allArgs, args...)
|
||||
}
|
||||
|
||||
if len(queries) == 0 {
|
||||
// No matching contexts, return empty result
|
||||
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
|
||||
}
|
||||
|
||||
// Combine queries with UNION ALL
|
||||
var limit int
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
limit += fieldKeySelector.Limit
|
||||
@@ -556,7 +525,15 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
mainQuery := fmt.Sprintf(`
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
|
||||
rowCount := 0
|
||||
|
||||
// the log and scope contexts have no keys table; they are served by the
|
||||
// static fields appended below
|
||||
if len(queries) > 0 {
|
||||
// Combine queries with UNION ALL
|
||||
mainQuery := fmt.Sprintf(`
|
||||
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
|
||||
FROM (
|
||||
%s
|
||||
@@ -566,103 +543,75 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
LIMIT %d
|
||||
`, strings.Join(queries, " UNION ALL "), limit+1)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
|
||||
rowCount := 0
|
||||
searchTexts := []string{}
|
||||
|
||||
// Collect search texts for static field matching
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
|
||||
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
|
||||
// if the user did not also directly request this name — a field like "education" can be
|
||||
// both a parent of "education[].name" and an explicitly queried field in its own right.
|
||||
switch fieldDataType {
|
||||
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
|
||||
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
|
||||
parentTypes[name] = append(parentTypes[name], fieldDataType)
|
||||
if !mapOfRequestedSelectors[name] {
|
||||
continue // skip; don't register the key.
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
|
||||
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
|
||||
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
|
||||
// if the user did not also directly request this name — a field like "education" can be
|
||||
// both a parent of "education[].name" and an explicitly queried field in its own right.
|
||||
switch fieldDataType {
|
||||
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
|
||||
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
|
||||
parentTypes[name] = append(parentTypes[name], fieldDataType)
|
||||
if !mapOfRequestedSelectors[name] {
|
||||
continue // skip; don't register the key.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
|
||||
// if there is no materialised column, create a key with the field context and data type
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
// if there is no materialised column, create a key with the field context and data type
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
}
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
}
|
||||
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
|
||||
staticKeys := []string{}
|
||||
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
|
||||
|
||||
// Add matching intrinsic and matching calculated fields
|
||||
// These don't count towards the limit
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
if v == "" || strings.Contains(key, v) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
// Add the matching intrinsic columns. These don't count towards the limit
|
||||
for _, field := range maps.Values(logstelemetryschema.IntrinsicFields) {
|
||||
if !staticFieldMatchesAny(field, fieldKeySelectors) {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key,
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
})
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
|
||||
// enrich body keys with promoted paths, indexes, and JSON access plans
|
||||
@@ -806,10 +755,6 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
allArgs = append(allArgs, args...)
|
||||
}
|
||||
|
||||
if len(queries) == 0 {
|
||||
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
|
||||
}
|
||||
|
||||
var limit int
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
limit += fieldKeySelector.Limit
|
||||
@@ -818,7 +763,13 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
mainQuery := fmt.Sprintf(`
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
rowCount := 0
|
||||
|
||||
// the log and scope contexts have no keys table; they are served by the
|
||||
// static fields appended below
|
||||
if len(queries) > 0 {
|
||||
mainQuery := fmt.Sprintf(`
|
||||
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
|
||||
FROM (
|
||||
%s
|
||||
@@ -828,73 +779,57 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
|
||||
LIMIT %d
|
||||
`, strings.Join(queries, " UNION ALL "), limit+1)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
keys := []*telemetrytypes.TelemetryFieldKey{}
|
||||
rowCount := 0
|
||||
searchTexts := []string{}
|
||||
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
searchTexts = append(searchTexts, fieldKeySelector.Name)
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
defer rows.Close()
|
||||
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
if rowCount > limit {
|
||||
break
|
||||
}
|
||||
|
||||
var name string
|
||||
var fieldContext telemetrytypes.FieldContext
|
||||
var fieldDataType telemetrytypes.FieldDataType
|
||||
var priority uint8
|
||||
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
|
||||
|
||||
if !ok {
|
||||
key = &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
}
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
keys = append(keys, key)
|
||||
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
if rows.Err() != nil {
|
||||
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
|
||||
}
|
||||
}
|
||||
|
||||
complete := rowCount <= limit
|
||||
|
||||
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
|
||||
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
|
||||
for _, key := range staticKeys {
|
||||
found := false
|
||||
for _, v := range searchTexts {
|
||||
if v == "" || strings.Contains(key, v) {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
// Add the matching intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
|
||||
for _, field := range maps.Values(audittelemetryschema.IntrinsicFields) {
|
||||
if !staticFieldMatchesAny(field, fieldKeySelectors) {
|
||||
continue
|
||||
}
|
||||
|
||||
if found {
|
||||
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
}
|
||||
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, &field)
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
@@ -1091,9 +1026,12 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
|
||||
}
|
||||
// meter labels are stored as strings in the labels JSON and have no
|
||||
// attribute context, so only the data type is known
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1506,88 +1444,13 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "getSpanFieldValues",
|
||||
})
|
||||
// build the query to get the keys from the spans that match the field selection criteria
|
||||
limit := fieldValueSelector.Limit
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
|
||||
if values, ok := spanSearchScopeFieldValues(fieldValueSelector.Name, fieldValueSelector.Value); ok {
|
||||
return values, true, nil
|
||||
}
|
||||
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
}
|
||||
|
||||
// now look at the field context
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
|
||||
}
|
||||
|
||||
// now look at the field data type
|
||||
if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
|
||||
sb.Where(sb.E("tag_data_type", fieldValueSelector.FieldDataType.TagDataType()))
|
||||
}
|
||||
|
||||
if fieldValueSelector.Value != "" {
|
||||
switch fieldValueSelector.FieldDataType {
|
||||
case telemetrytypes.FieldDataTypeString:
|
||||
sb.Where(sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
|
||||
case telemetrytypes.FieldDataTypeNumber:
|
||||
sb.Where(sb.IsNotNull("number_value"))
|
||||
sb.Where(sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
|
||||
case telemetrytypes.FieldDataTypeUnspecified:
|
||||
// or b/w string and number
|
||||
sb.Where(sb.Or(
|
||||
sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
|
||||
sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// query one extra to check if we hit the limit
|
||||
sb.Limit(limit + 1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
seen := make(map[string]bool)
|
||||
rowCount := 0
|
||||
totalCount := 0 // Track total unique values
|
||||
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
|
||||
var stringValue string
|
||||
var numberValue float64
|
||||
if err := rows.Scan(&stringValue, &numberValue); err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
|
||||
// Only add values if we haven't hit the limit yet
|
||||
if totalCount < limit {
|
||||
if _, ok := seen[stringValue]; !ok && stringValue != "" {
|
||||
values.StringValues = append(values.StringValues, stringValue)
|
||||
seen[stringValue] = true
|
||||
totalCount++
|
||||
}
|
||||
if _, ok := seen[fmt.Sprintf("%f", numberValue)]; !ok && numberValue != 0 && totalCount < limit {
|
||||
values.NumberValues = append(values.NumberValues, numberValue)
|
||||
seen[fmt.Sprintf("%f", numberValue)] = true
|
||||
totalCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
|
||||
return values, complete, nil
|
||||
knownBool := isKnownBoolField(fieldValueSelector, tracestelemetryschema.IntrinsicFields, tracestelemetryschema.CalculatedFields)
|
||||
// unix_milli is the hour of the span start
|
||||
return t.getTagTableValues(ctx, t.tracesDBName+"."+t.tracesFieldsTblName, fieldValueSelector, knownBool)
|
||||
}
|
||||
|
||||
func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
|
||||
@@ -1596,17 +1459,77 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
|
||||
instrumentationtypes.CodeNamespace: "metadata",
|
||||
instrumentationtypes.CodeFunctionName: "getLogFieldValues",
|
||||
})
|
||||
// build the query to get the keys from the spans that match the field selection criteria
|
||||
|
||||
knownBool := isKnownBoolField(fieldValueSelector, logstelemetryschema.IntrinsicFields)
|
||||
// unix_milli is the hour the log was ingested, not the log's own timestamp
|
||||
return t.getTagTableValues(ctx, t.logsDBName+"."+t.logsFieldsTblName, fieldValueSelector, knownBool)
|
||||
}
|
||||
|
||||
// tagTableSinceDay restricts rows to the tag table's day partitions from the
|
||||
// start's day on. Partitions are toDate(unix_milli / 1000) in the server's
|
||||
// timezone, and a value's surviving row within a day carries whichever hour
|
||||
// was inserted last, so the day is the finest safe unit.
|
||||
func tagTableSinceDay(sb *sqlbuilder.SelectBuilder, startUnixMilli int64) {
|
||||
if startUnixMilli != 0 {
|
||||
sb.Where(fmt.Sprintf("toDate(unix_milli / 1000) >= toDate(%d)", startUnixMilli/1000))
|
||||
}
|
||||
}
|
||||
|
||||
// tagTableHasBoolRows reports whether the tag table holds a bool row for the
|
||||
// key. Bool rows carry no value, so one row is enough to know the key takes
|
||||
// the values true and false.
|
||||
func (t *telemetryMetaStore) tagTableHasBoolRows(ctx context.Context, table string, selector *telemetrytypes.FieldValueSelector) (bool, error) {
|
||||
sb := sqlbuilder.Select("1").From(table)
|
||||
sb.Where(sb.E("tag_key", selector.Name))
|
||||
sb.Where(sb.E("tag_data_type", telemetrytypes.FieldDataTypeBool.TagDataType()))
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
sb.Where(sb.E("tag_type", selector.FieldContext.TagType()))
|
||||
}
|
||||
tagTableSinceDay(sb, selector.StartUnixMilli)
|
||||
sb.Limit(1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
|
||||
}
|
||||
defer rows.Close()
|
||||
return rows.Next(), rows.Err()
|
||||
}
|
||||
|
||||
// getTagTableValues returns the string and number values of the key from a
|
||||
// tag table, and true and false when the key is a known bool field or the
|
||||
// table holds bool rows for it. Bool rows do not count towards the limit.
|
||||
func (t *telemetryMetaStore) getTagTableValues(ctx context.Context, table string, fieldValueSelector *telemetrytypes.FieldValueSelector, knownBool bool) (*telemetrytypes.TelemetryFieldValues, bool, error) {
|
||||
limit := fieldValueSelector.Limit
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
}
|
||||
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
if knownBool {
|
||||
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
|
||||
if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeBool {
|
||||
return values, true, nil
|
||||
}
|
||||
} else if fieldValueSelector.FieldDataType == telemetrytypes.FieldDataTypeUnspecified {
|
||||
hasBoolRows, err := t.tagTableHasBoolRows(ctx, table, fieldValueSelector)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if hasBoolRows {
|
||||
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
|
||||
}
|
||||
}
|
||||
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(table)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
}
|
||||
sb.Where(sb.NE("tag_data_type", telemetrytypes.FieldDataTypeBool.TagDataType()))
|
||||
|
||||
tagTableSinceDay(sb, fieldValueSelector.StartUnixMilli)
|
||||
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
|
||||
@@ -1643,7 +1566,6 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
seen := make(map[string]bool)
|
||||
rowCount := 0
|
||||
totalCount := 0 // Track total unique values
|
||||
@@ -2097,6 +2019,18 @@ func populateAllUnspecifiedValues(allUnspecifiedValues *telemetrytypes.Telemetry
|
||||
}
|
||||
}
|
||||
|
||||
for _, value := range values.BoolValues {
|
||||
if totalCount >= limit {
|
||||
complete = false
|
||||
break
|
||||
}
|
||||
if _, ok := mapOfValues[value]; !ok {
|
||||
mapOfValues[value] = true
|
||||
allUnspecifiedValues.BoolValues = append(allUnspecifiedValues.BoolValues, value)
|
||||
totalCount++
|
||||
}
|
||||
}
|
||||
|
||||
for _, value := range values.RelatedValues {
|
||||
if totalCount >= limit {
|
||||
complete = false
|
||||
@@ -2467,6 +2401,10 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
|
||||
|
||||
// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
|
||||
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
|
||||
// an empty selector list would run the evolution query without a filter
|
||||
if len(keysToUpdate) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
|
||||
for _, keySelector := range keysToUpdate {
|
||||
|
||||
53
pkg/telemetrymetadata/static_fields.go
Normal file
53
pkg/telemetrymetadata/static_fields.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package telemetrymetadata
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
func staticFieldMatchesAny(field telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) bool {
|
||||
for _, selector := range selectors {
|
||||
if staticFieldMatches(field, selector) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// staticFieldMatches mirrors the keys-table lookup for a static field: the
|
||||
// requested context and data type, when given, must agree with the field's,
|
||||
// and the name matches case-insensitively, as a substring for fuzzy selectors
|
||||
// and as the whole name for exact ones.
|
||||
func staticFieldMatches(field telemetrytypes.TelemetryFieldKey, selector *telemetrytypes.FieldKeySelector) bool {
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != field.FieldContext {
|
||||
return false
|
||||
}
|
||||
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && !sameDataTypeFamily(selector.FieldDataType, field.FieldDataType) {
|
||||
return false
|
||||
}
|
||||
if selector.Name == "" {
|
||||
return true
|
||||
}
|
||||
if selector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
return strings.EqualFold(field.Name, selector.Name)
|
||||
}
|
||||
return strings.Contains(strings.ToLower(field.Name), strings.ToLower(selector.Name))
|
||||
}
|
||||
|
||||
// sameDataTypeFamily treats the numeric types as one family: static fields
|
||||
// declare "number" while callers may ask for int64 or float64.
|
||||
func sameDataTypeFamily(requested, actual telemetrytypes.FieldDataType) bool {
|
||||
if requested == actual {
|
||||
return true
|
||||
}
|
||||
return isNumericDataType(requested) && isNumericDataType(actual)
|
||||
}
|
||||
|
||||
func isNumericDataType(dataType telemetrytypes.FieldDataType) bool {
|
||||
switch dataType {
|
||||
case telemetrytypes.FieldDataTypeNumber, telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -392,6 +392,24 @@ var (
|
||||
SpanSearchScopeRoot = "isroot"
|
||||
SpanSearchScopeEntryPoint = "isentrypoint"
|
||||
|
||||
// SpanSearchScopeFields are the search-scope selectors (isRoot, isEntryPoint),
|
||||
// not columns and unrelated to the instrumentation scope: they only filter
|
||||
// with the value true.
|
||||
SpanSearchScopeFields = map[string]telemetrytypes.TelemetryFieldKey{
|
||||
"isRoot": {
|
||||
Name: "isRoot",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
"isEntryPoint": {
|
||||
Name: "isEntryPoint",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
}
|
||||
|
||||
// IntrinsicSpanFields lists the intrinsic span columns, in the order they
|
||||
// should appear when a raw query expands its SelectFields.
|
||||
IntrinsicSpanFields = []telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
@@ -173,18 +173,18 @@ func NewSourceFilterFromStorableQuickFilter(storableQuickFilter *StorableQuickFi
|
||||
// NewDefaultQuickFilter generates default filters for all supported sources.
|
||||
func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
|
||||
tracesFilters := []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
{Name: "duration_nano", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
{Name: "deployment.environment", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "hasError", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeBool},
|
||||
{Name: "has_error", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeBool},
|
||||
{Name: "service.name", FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "name", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "name", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "rpc.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "response_status_code", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_host", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "response_status_code", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_host", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http.method", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http.route", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_url", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "trace_id", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "http_url", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "trace_id", FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
}
|
||||
|
||||
logsFilters := []telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
98
tests/fixtures/queriercommon.py
vendored
98
tests/fixtures/queriercommon.py
vendored
@@ -1,4 +1,4 @@
|
||||
"""Seed data for the queriercommon keyless-semantics tests.
|
||||
"""Seed data for the queriercommon keyless-semantics and explicit-context tests.
|
||||
|
||||
Three identities exist in every signal. GOLD and SILVER carry the test keys.
|
||||
NONE carries no key at all. The tests assert which identities a filter
|
||||
@@ -8,6 +8,7 @@ The attribute names are outside every semantic-convention family, so the
|
||||
seeded data pins base behavior with any semconv overlay state.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Callable, Generator
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
@@ -122,3 +123,98 @@ def keyless_series(insert_metrics: Callable[[list[Metrics]], None]) -> Generator
|
||||
]
|
||||
)
|
||||
yield start, start + points * 60
|
||||
|
||||
|
||||
EXPLICIT_PREFIX = "explicit-ctx"
|
||||
# Unambiguous string attribute that names the row. Every assertion reads it back.
|
||||
IDENTITY_KEY = "probe.id"
|
||||
# Attribute-only key with no same-named column, for the own-context miss. On
|
||||
# logs the rows that lack the attribute carry it nested in the body JSON.
|
||||
ATTRIBUTE_ONLY_KEY = "route.tag"
|
||||
CONTESTED_VALUE = "checkout"
|
||||
|
||||
# Row identities, by where the contested name carries the contested value:
|
||||
# the intrinsic column (`name` on spans, `severity_text` on logs), the
|
||||
# same-named string attribute, both, neither, or a same-named number
|
||||
# attribute (a data type that contradicts the column).
|
||||
COLUMN_ONLY = f"{EXPLICIT_PREFIX}-column"
|
||||
ATTRIBUTE_ONLY = f"{EXPLICIT_PREFIX}-attribute"
|
||||
BOTH = f"{EXPLICIT_PREFIX}-both"
|
||||
NEITHER = f"{EXPLICIT_PREFIX}-neither"
|
||||
NUMBER_ATTRIBUTE = f"{EXPLICIT_PREFIX}-number"
|
||||
NUMBER_VALUE = 42
|
||||
|
||||
# (identity, column carries the value, attribute carries the value,
|
||||
# attribute carries the number, resource service.name, attribute service.name,
|
||||
# carries route.tag, insert offset in seconds)
|
||||
ROWS = [
|
||||
(COLUMN_ONLY, True, False, False, "svc-a", None, True, 1),
|
||||
(ATTRIBUTE_ONLY, False, True, False, "svc-b", "svc-a", False, 2),
|
||||
(BOTH, True, True, False, "svc-a", "svc-a", True, 3),
|
||||
(NEITHER, False, False, False, "svc-b", "svc-b", False, 4),
|
||||
(NUMBER_ATTRIBUTE, False, False, True, "svc-b", None, False, 5),
|
||||
]
|
||||
|
||||
# Logs only: the declared scope path `scope.name` next to a scope attribute
|
||||
# that is also named `name`, and a plain scope attribute.
|
||||
SCOPE_NAME = "scope-a"
|
||||
SCOPE_ATTRIBUTE_KEY = "env"
|
||||
SCOPE_ATTRIBUTE_VALUE = "prod"
|
||||
|
||||
|
||||
@pytest.fixture(name="ambiguous_rows", scope="function")
|
||||
def ambiguous_rows(
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> Generator[datetime]:
|
||||
"""One span and one log per identity. `service.name` exists as a resource
|
||||
attribute on every row and as a span or log attribute on some, with
|
||||
different values, so a bare `service.name` is ambiguous. Logs that lack
|
||||
the `route.tag` attribute carry it in the body JSON instead. Logs with
|
||||
the column value carry the scope name; logs with the attribute value
|
||||
carry the scope attributes. Yields the base timestamp."""
|
||||
now = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(minutes=1)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=offset),
|
||||
duration=timedelta(milliseconds=10),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
name=CONTESTED_VALUE if column else "other",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
resources={"service.name": resource_service},
|
||||
attributes={
|
||||
IDENTITY_KEY: identity,
|
||||
**({"name": CONTESTED_VALUE} if attribute else {}),
|
||||
**({"name": NUMBER_VALUE} if number else {}),
|
||||
**({"service.name": attribute_service} if attribute_service else {}),
|
||||
**({ATTRIBUTE_ONLY_KEY: CONTESTED_VALUE} if tagged else {}),
|
||||
},
|
||||
)
|
||||
for identity, column, attribute, number, resource_service, attribute_service, tagged, offset in ROWS
|
||||
]
|
||||
)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=offset),
|
||||
body=json.dumps({} if tagged else {"route": {"tag": CONTESTED_VALUE}}),
|
||||
severity_text="ERROR" if column else "INFO",
|
||||
scope_name=SCOPE_NAME if column else "",
|
||||
scope_attributes={"name": CONTESTED_VALUE, SCOPE_ATTRIBUTE_KEY: SCOPE_ATTRIBUTE_VALUE} if attribute else {},
|
||||
resources={"service.name": resource_service},
|
||||
attributes={
|
||||
IDENTITY_KEY: identity,
|
||||
**({"severity_text": "ERROR"} if attribute else {}),
|
||||
**({"severity_text": NUMBER_VALUE} if number else {}),
|
||||
**({"service.name": attribute_service} if attribute_service else {}),
|
||||
**({ATTRIBUTE_ONLY_KEY: CONTESTED_VALUE} if tagged else {}),
|
||||
},
|
||||
)
|
||||
for identity, column, attribute, number, resource_service, attribute_service, tagged, offset in ROWS
|
||||
]
|
||||
)
|
||||
yield now
|
||||
|
||||
371
tests/integration/tests/queriercommon/07_explicit_context.py
Normal file
371
tests/integration/tests/queriercommon/07_explicit_context.py
Normal file
@@ -0,0 +1,371 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import (
|
||||
RequestType,
|
||||
assert_scalar_value,
|
||||
build_aggregation,
|
||||
build_group_by_field,
|
||||
build_order_by,
|
||||
build_raw_query,
|
||||
build_scalar_query,
|
||||
get_all_warnings,
|
||||
get_column_data_from_response,
|
||||
get_scalar_table_data,
|
||||
make_query_request,
|
||||
)
|
||||
from fixtures.queriercommon import (
|
||||
ATTRIBUTE_ONLY,
|
||||
BOTH,
|
||||
COLUMN_ONLY,
|
||||
EXPLICIT_PREFIX,
|
||||
IDENTITY_KEY,
|
||||
NEITHER,
|
||||
NUMBER_ATTRIBUTE,
|
||||
)
|
||||
|
||||
# Which rows a filter returns when the same name exists as an intrinsic
|
||||
# column and as an attribute (`name` on spans, `severity_text` on logs), or
|
||||
# as a resource attribute and a span or log attribute (`service.name`).
|
||||
# An explicit context is honored as written. A bare name that is both a
|
||||
# column and an attribute reads both, with an ambiguity warning. A bare name
|
||||
# that is both a resource and an attribute reads the resource, with a
|
||||
# warning. The warning also fires for an explicit attribute context when the
|
||||
# attribute exists in two data types, and a string operand reaches the
|
||||
# number attribute through a text cast. A key under the signal's own context
|
||||
# that exists only as an attribute corrects to the attribute; on logs the
|
||||
# correction also reads the body JSON path.
|
||||
FILTER_MATRIX = [
|
||||
pytest.param("{contested} = '{value}'", {COLUMN_ONLY, ATTRIBUTE_ONLY, BOTH}, True, id="bare_column_and_attribute"),
|
||||
pytest.param("{own}.{contested} = '{value}'", {COLUMN_ONLY, BOTH}, False, id="own_context_column_only"),
|
||||
pytest.param("attribute.{contested} = '{value}'", {ATTRIBUTE_ONLY, BOTH}, True, id="attribute_context_warns_about_two_types"),
|
||||
pytest.param("{contested} != '{value}'", {NEITHER, NUMBER_ATTRIBUTE}, True, id="bare_negative_excludes_every_carrier"),
|
||||
pytest.param("{contested} EXISTS", {COLUMN_ONLY, ATTRIBUTE_ONLY, BOTH, NEITHER, NUMBER_ATTRIBUTE}, True, id="bare_exists_is_the_column"),
|
||||
pytest.param("{contested} NOT EXISTS", set(), True, id="bare_not_exists_is_never"),
|
||||
pytest.param("attribute.{contested} EXISTS", {ATTRIBUTE_ONLY, BOTH, NUMBER_ATTRIBUTE}, True, id="attribute_exists_spans_both_types"),
|
||||
pytest.param("attribute.{contested} NOT EXISTS", {COLUMN_ONLY, NEITHER}, True, id="attribute_not_exists"),
|
||||
pytest.param("{contested} = '42'", {NUMBER_ATTRIBUTE}, True, id="bare_string_operand_reaches_the_number_attribute"),
|
||||
pytest.param("attribute.{contested}:string = '{value}'", {ATTRIBUTE_ONLY, BOTH}, False, id="type_suffix_selects_the_string_attribute"),
|
||||
pytest.param("attribute.{contested}:float64 = 42", {NUMBER_ATTRIBUTE}, False, id="type_suffix_selects_the_number_attribute"),
|
||||
pytest.param("service.name = 'svc-a'", {COLUMN_ONLY, BOTH}, True, id="bare_resource_wins_with_warning"),
|
||||
pytest.param("service.name != 'svc-a'", {ATTRIBUTE_ONLY, NEITHER, NUMBER_ATTRIBUTE}, True, id="bare_resource_negative"),
|
||||
pytest.param("resource.service.name = 'svc-a'", {COLUMN_ONLY, BOTH}, False, id="resource_context_no_warning"),
|
||||
pytest.param("resource.service.name != 'svc-a'", {ATTRIBUTE_ONLY, NEITHER, NUMBER_ATTRIBUTE}, False, id="resource_context_negative"),
|
||||
pytest.param("attribute.service.name = 'svc-a'", {ATTRIBUTE_ONLY, BOTH}, False, id="attribute_context_no_warning"),
|
||||
pytest.param(
|
||||
"{own}.route.tag = 'checkout'",
|
||||
{"traces": {COLUMN_ONLY, BOTH}, "logs": {COLUMN_ONLY, ATTRIBUTE_ONLY, BOTH, NEITHER, NUMBER_ATTRIBUTE}},
|
||||
False,
|
||||
id="own_context_miss_corrects_to_attribute_and_on_logs_to_body",
|
||||
),
|
||||
pytest.param("route.tag = 'checkout'", {COLUMN_ONLY, BOTH}, False, id="bare_attribute_only_key"),
|
||||
]
|
||||
|
||||
SIGNALS = [
|
||||
pytest.param("traces", "span", "name", "checkout", "other", id="traces"),
|
||||
pytest.param("logs", "log", "severity_text", "ERROR", "INFO", id="logs"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expression_template,expected,expects_ambiguity_warning", FILTER_MATRIX)
|
||||
@pytest.mark.parametrize("signal,own_context,contested,value,other_value", SIGNALS)
|
||||
def test_filter_resolution(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
ambiguous_rows: datetime,
|
||||
signal: str,
|
||||
own_context: str,
|
||||
contested: str,
|
||||
value: str,
|
||||
other_value: str, # pylint: disable=unused-argument
|
||||
expression_template: str,
|
||||
expected: set[str] | dict[str, set[str]],
|
||||
expects_ambiguity_warning: bool,
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
expression = expression_template.format(own=own_context, contested=contested, value=value)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((ambiguous_rows - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((ambiguous_rows + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.RAW,
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
signal,
|
||||
limit=100,
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "asc")],
|
||||
select_fields=[{"name": IDENTITY_KEY}],
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
|
||||
matched = {row for row in get_column_data_from_response(response.json(), IDENTITY_KEY) if row.startswith(EXPLICIT_PREFIX)}
|
||||
assert matched == (expected[signal] if isinstance(expected, dict) else expected), expression
|
||||
|
||||
warnings = [w["message"] for w in get_all_warnings(response.json())]
|
||||
assert any("ambiguous" in w for w in warnings) == expects_ambiguity_warning, warnings
|
||||
|
||||
|
||||
# Group by resolves the contested name in the column stage: a bare name that
|
||||
# is both a column and an attribute groups by the column alone, an explicit
|
||||
# context groups by that context alone.
|
||||
GROUP_BY_MATRIX = [
|
||||
pytest.param(None, {"{value}": 2, "{other}": 3}, id="bare_groups_by_the_column"),
|
||||
pytest.param("own", {"{value}": 2, "{other}": 3}, id="own_context_groups_by_the_column"),
|
||||
pytest.param("attribute", {"{value}": 2}, id="attribute_context_groups_by_the_attribute"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context,expected_template", GROUP_BY_MATRIX)
|
||||
@pytest.mark.parametrize("signal,own_context,contested,value,other_value", SIGNALS)
|
||||
def test_group_by_resolution(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
ambiguous_rows: datetime,
|
||||
signal: str,
|
||||
own_context: str,
|
||||
contested: str,
|
||||
value: str,
|
||||
other_value: str,
|
||||
context: str | None,
|
||||
expected_template: dict[str, int],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
field_context = own_context if context == "own" else context
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((ambiguous_rows - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((ambiguous_rows + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.SCALAR,
|
||||
queries=[
|
||||
build_scalar_query(
|
||||
"A",
|
||||
signal,
|
||||
[build_aggregation("count()", "rows")],
|
||||
group_by=[build_group_by_field(contested, "string", field_context) if field_context else {"name": contested}],
|
||||
filter_expression=f"{IDENTITY_KEY} LIKE '{EXPLICIT_PREFIX}%'",
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
|
||||
expected = {key.format(value=value, other=other_value): count for key, count in expected_template.items()}
|
||||
groups = {row[0]: row[1] for row in get_scalar_table_data(response.json()) if row[0] in expected}
|
||||
assert groups == expected, get_scalar_table_data(response.json())
|
||||
|
||||
|
||||
# A raw select of a bare name that is both a resource and an attribute reads
|
||||
# one value per row: the resource value, also on the row whose attribute
|
||||
# carries a different value.
|
||||
@pytest.mark.parametrize("signal", ["traces", "logs"])
|
||||
def test_select_of_ambiguous_name(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
ambiguous_rows: datetime,
|
||||
signal: str,
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((ambiguous_rows - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((ambiguous_rows + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.RAW,
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
signal,
|
||||
limit=100,
|
||||
filter_expression=f"{IDENTITY_KEY} LIKE '{EXPLICIT_PREFIX}%'",
|
||||
order=[build_order_by("timestamp", "asc")],
|
||||
select_fields=[{"name": IDENTITY_KEY}, {"name": "service.name"}],
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
|
||||
rows = response.json()["data"]["data"]["results"][0]["rows"] or []
|
||||
by_identity = {row["data"][IDENTITY_KEY]: row["data"]["service.name"] for row in rows if row["data"].get(IDENTITY_KEY, "").startswith(EXPLICIT_PREFIX)}
|
||||
assert by_identity == {
|
||||
COLUMN_ONLY: "svc-a",
|
||||
ATTRIBUTE_ONLY: "svc-b",
|
||||
BOTH: "svc-a",
|
||||
NEITHER: "svc-b",
|
||||
NUMBER_ATTRIBUTE: "svc-b",
|
||||
}
|
||||
|
||||
|
||||
# Order by resolves the contested name in the column stage, descending, with
|
||||
# the timestamp descending as the tie breaker. A bare or own-context name
|
||||
# sorts by the column alone. An explicit attribute context sorts by the
|
||||
# attribute on traces, where the number attribute reads as text and rows
|
||||
# without the attribute come last; on logs it still sorts by the column.
|
||||
BY_COLUMN = [ATTRIBUTE_ONLY, NEITHER, NUMBER_ATTRIBUTE, COLUMN_ONLY, BOTH]
|
||||
ORDER_BY_MATRIX = [
|
||||
pytest.param(None, {"traces": BY_COLUMN, "logs": BY_COLUMN}, id="bare_orders_by_the_column"),
|
||||
pytest.param("own", {"traces": BY_COLUMN, "logs": BY_COLUMN}, id="own_context_orders_by_the_column"),
|
||||
pytest.param(
|
||||
"attribute",
|
||||
{"traces": [ATTRIBUTE_ONLY, BOTH, NUMBER_ATTRIBUTE, COLUMN_ONLY, NEITHER], "logs": BY_COLUMN},
|
||||
id="attribute_context_orders_by_the_attribute_on_traces_only",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context,expected", ORDER_BY_MATRIX)
|
||||
@pytest.mark.parametrize("signal,own_context,contested,value,other_value", SIGNALS)
|
||||
def test_order_by_resolution(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
ambiguous_rows: datetime,
|
||||
signal: str,
|
||||
own_context: str,
|
||||
contested: str,
|
||||
value: str, # pylint: disable=unused-argument
|
||||
other_value: str, # pylint: disable=unused-argument
|
||||
context: str | None,
|
||||
expected: dict[str, list[str]],
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
prefix = f"{own_context}." if context == "own" else f"{context}." if context else ""
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((ambiguous_rows - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((ambiguous_rows + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.RAW,
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
signal,
|
||||
limit=100,
|
||||
filter_expression=f"{IDENTITY_KEY} LIKE '{EXPLICIT_PREFIX}%'",
|
||||
order=[build_order_by(f"{prefix}{contested}", "desc"), build_order_by("timestamp", "desc")],
|
||||
select_fields=[{"name": IDENTITY_KEY}],
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
|
||||
ordered = [row for row in get_column_data_from_response(response.json(), IDENTITY_KEY) if row.startswith(EXPLICIT_PREFIX)]
|
||||
assert ordered == expected[signal]
|
||||
|
||||
|
||||
# An aggregation argument resolves the contested name in the column stage: a
|
||||
# bare name counts the column's values alone; an attribute context counts the
|
||||
# attribute in both of its data types, so the number attribute adds a
|
||||
# distinct value.
|
||||
AGGREGATION_MATRIX = [
|
||||
pytest.param(None, 2, id="bare_counts_the_column"),
|
||||
pytest.param("own", 2, id="own_context_counts_the_column"),
|
||||
pytest.param("attribute", 2, id="attribute_context_counts_both_attribute_types"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("context,expected", AGGREGATION_MATRIX)
|
||||
@pytest.mark.parametrize("signal,own_context,contested,value,other_value", SIGNALS)
|
||||
def test_aggregation_argument_resolution(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
ambiguous_rows: datetime,
|
||||
signal: str,
|
||||
own_context: str,
|
||||
contested: str,
|
||||
value: str, # pylint: disable=unused-argument
|
||||
other_value: str, # pylint: disable=unused-argument
|
||||
context: str | None,
|
||||
expected: int,
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
prefix = f"{own_context}." if context == "own" else f"{context}." if context else ""
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((ambiguous_rows - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((ambiguous_rows + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.SCALAR,
|
||||
queries=[
|
||||
build_scalar_query(
|
||||
"A",
|
||||
signal,
|
||||
[build_aggregation(f"count_distinct({prefix}{contested})", "distinct")],
|
||||
filter_expression=f"{IDENTITY_KEY} LIKE '{EXPLICIT_PREFIX}%'",
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert_scalar_value(response, "A", expected)
|
||||
|
||||
|
||||
# Logs only. `body.x` addresses the body JSON and never the same-named
|
||||
# attribute; `log.x` reads the attribute and the body JSON path together,
|
||||
# even when the attribute exists in metadata. A `scope.` key is a strict
|
||||
# context resolved through metadata alone: the declared scope path
|
||||
# `scope.name` and a scope attribute both answer "key not found" when
|
||||
# metadata does not report them, even when the rows carry them.
|
||||
LOGS_ONLY_MATRIX = [
|
||||
pytest.param("body.route.tag = 'checkout'", {ATTRIBUTE_ONLY, NEITHER, NUMBER_ATTRIBUTE}, id="body_context_reads_the_body_json"),
|
||||
pytest.param("log.route.tag = 'checkout'", {COLUMN_ONLY, ATTRIBUTE_ONLY, BOTH, NEITHER, NUMBER_ATTRIBUTE}, id="log_context_reads_attribute_and_body"),
|
||||
pytest.param("scope.name = 'scope-a'", "key `name` not found", id="scope_name_needs_metadata"),
|
||||
pytest.param("scope.env = 'prod'", "key `env` not found", id="scope_attribute_needs_metadata"),
|
||||
pytest.param("scope.env EXISTS", "key `env` not found", id="scope_attribute_exists_needs_metadata"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expression,expected", LOGS_ONLY_MATRIX)
|
||||
def test_logs_body_and_scope_contexts(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
ambiguous_rows: datetime,
|
||||
expression: str,
|
||||
expected: set[str] | str,
|
||||
) -> None:
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((ambiguous_rows - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((ambiguous_rows + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.RAW,
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
limit=100,
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "asc")],
|
||||
select_fields=[{"name": IDENTITY_KEY}],
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
if isinstance(expected, str):
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
assert expected in response.text, response.text
|
||||
return
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
matched = {row for row in get_column_data_from_response(response.json(), IDENTITY_KEY) if row.startswith(EXPLICIT_PREFIX)}
|
||||
assert matched == expected, expression
|
||||
235
tests/integration/tests/queriercommon/07_fields_keys_values.py
Normal file
235
tests/integration/tests/queriercommon/07_fields_keys_values.py
Normal file
@@ -0,0 +1,235 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.traces import Traces
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,field_context,present,absent",
|
||||
[
|
||||
pytest.param("logs", "log", {"severity_text": "log", "body": "log", "trace_id": "log"}, ["code.file", "scope_name"], id="log_context_lists_log_intrinsics"),
|
||||
pytest.param("logs", "scope", {"scope_name": "scope", "scope_version": "scope"}, ["severity_text", "body", "code.file"], id="scope_context_lists_scope_intrinsics_for_logs"),
|
||||
pytest.param("logs", "attribute", {"code.file": "attribute"}, ["body", "scope_name"], id="attribute_context_excludes_log_intrinsics"),
|
||||
pytest.param("traces", "span", {"name": "span", "has_error": "span", "isRoot": "span", "http.method": "attribute"}, ["scope.name"], id="span_context_lists_span_intrinsics_and_attributes"),
|
||||
pytest.param("traces", "scope", {"scope.name": "scope", "scope.version": "scope"}, ["name", "has_error", "isRoot", "http.method", "host.name"], id="scope_context_lists_scope_intrinsics_for_traces"),
|
||||
pytest.param("traces", "resource", {"host.name": "resource"}, ["name", "has_error", "isRoot", "http.method"], id="resource_context_excludes_span_intrinsics"),
|
||||
pytest.param("traces", "attribute", {"http.method": "attribute"}, ["name", "has_error", "isRoot", "host.name"], id="attribute_context_excludes_span_intrinsics"),
|
||||
],
|
||||
)
|
||||
def test_fields_keys_by_context(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
signal: str,
|
||||
field_context: str,
|
||||
present: dict[str, str],
|
||||
absent: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a log with a code.file attribute and a span with an http.method attribute and a host.name resource.
|
||||
|
||||
Tests:
|
||||
1. Keys for a context list that context's intrinsic columns and the stored keys the context maps to,
|
||||
each with its context; intrinsics of other contexts are not listed. The span context also keeps
|
||||
listing attributes because `span.<attribute>` resolves attributes in queries.
|
||||
"""
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs([Logs(timestamp=now, attributes={"code.file": "/opt/integration.go"}, body="a log line")])
|
||||
insert_traces([Traces(timestamp=now, resources={"host.name": "linux-001"}, attributes={"http.method": "GET"})])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "fieldContext": field_context},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
keys = response.json()["data"]["keys"]
|
||||
listed = {name: [key["fieldContext"] for key in keys.get(name, [])] for name in present}
|
||||
assert listed == {name: [context] for name, context in present.items()}, f"keys for the {field_context} context"
|
||||
assert [name for name in absent if name in keys] == [], f"keys that do not belong to the {field_context} context"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,field_context,field_data_type,present,absent",
|
||||
[
|
||||
pytest.param("traces", "span", "float64", ["duration_nano", "status_code"], ["name", "has_error"], id="float64_matches_number_span_intrinsics"),
|
||||
pytest.param("traces", "span", "int64", ["duration_nano", "status_code"], ["name", "has_error"], id="int64_matches_number_span_intrinsics"),
|
||||
pytest.param("traces", "span", "bool", ["has_error", "isRoot", "isEntryPoint"], ["name", "duration_nano"], id="bool_matches_bool_span_intrinsics"),
|
||||
pytest.param("traces", "span", "string", ["name", "http_method"], ["duration_nano", "has_error"], id="string_matches_string_span_intrinsics"),
|
||||
pytest.param("logs", "log", "number", ["severity_number", "trace_flags"], ["severity_text", "body"], id="number_matches_number_log_intrinsics"),
|
||||
],
|
||||
)
|
||||
def test_fields_keys_by_data_type(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
signal: str,
|
||||
field_context: str,
|
||||
field_data_type: str,
|
||||
present: list[str],
|
||||
absent: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Tests:
|
||||
1. A data type filter keeps the intrinsic columns of that type; number, int64 and float64 are one family.
|
||||
"""
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "fieldContext": field_context, "fieldDataType": field_data_type},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert [name for name in present if name not in keys] == [], f"intrinsics of type {field_data_type}"
|
||||
assert [name for name in absent if name in keys] == [], f"intrinsics not of type {field_data_type}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,search_text,present",
|
||||
[
|
||||
pytest.param("logs", "SEVERITY", ["severity_text", "severity_number"], id="upper_case_search_logs"),
|
||||
pytest.param("traces", "HTTP_", ["http_method", "http_host", "http_url"], id="upper_case_search_traces"),
|
||||
pytest.param("traces", "Duration", ["duration_nano"], id="mixed_case_search_traces"),
|
||||
pytest.param("traces", "span.HAS_ERR", ["has_error"], id="context_prefix_with_upper_case_search"),
|
||||
],
|
||||
)
|
||||
def test_fields_keys_search_matches_intrinsics_case_insensitively(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
signal: str,
|
||||
search_text: str,
|
||||
present: list[str],
|
||||
) -> None:
|
||||
"""
|
||||
Tests:
|
||||
1. The search text matches intrinsic columns case-insensitively, as it does for stored keys,
|
||||
with or without a context prefix.
|
||||
"""
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "searchText": search_text},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
keys = response.json()["data"]["keys"]
|
||||
assert [name for name in present if name not in keys] == [], f"intrinsics matching {search_text!r}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"signal,params,expected",
|
||||
[
|
||||
pytest.param("traces", {"name": "has_error"}, [True, False], id="calculated_bool_span_field"),
|
||||
pytest.param("traces", {"name": "has_error", "fieldContext": "span"}, [True, False], id="calculated_bool_span_field_with_context"),
|
||||
pytest.param("traces", {"name": "has_error", "searchText": "tr"}, [True], id="search_text_narrows_bool_values"),
|
||||
pytest.param("traces", {"name": "isRoot"}, [True], id="span_scope_field_is_true_only"),
|
||||
pytest.param("logs", {"name": "retry"}, [True, False], id="bool_attribute_from_tag_rows"),
|
||||
pytest.param("logs", {"name": "retry", "fieldContext": "attribute"}, [True, False], id="bool_attribute_with_context"),
|
||||
pytest.param("logs", {"name": "retry", "searchText": "tr"}, [True], id="search_text_narrows_stored_bool_values"),
|
||||
pytest.param("logs", {"name": "never_seen", "fieldDataType": "bool"}, [True, False], id="declared_bool_type_needs_no_rows"),
|
||||
],
|
||||
)
|
||||
def test_fields_values_bool_fields(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
signal: str,
|
||||
params: dict[str, str],
|
||||
expected: list[bool],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a log with a bool attribute.
|
||||
|
||||
Tests:
|
||||
1. Values for a bool field are true and false (narrowed by the search text): for the calculated span
|
||||
field, for a stored bool attribute whose tag rows carry no value, and for a key the caller
|
||||
declares bool.
|
||||
2. A span scope selector (isRoot) only takes true.
|
||||
"""
|
||||
insert_logs([Logs(timestamp=datetime.now(tz=UTC), attributes={"retry": True}, body="retrying")])
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, **params},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["values"]["boolValues"] == expected
|
||||
assert response.json()["data"]["complete"] is True
|
||||
|
||||
|
||||
def test_fields_values_start_excludes_span_values_not_seen_since_the_day(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert a span three days old and a span now, with different service names.
|
||||
|
||||
Tests:
|
||||
1. Values with startUnixMilli an hour ago contain only the service seen today: the start is
|
||||
floored to the day, the tag table's deduplication unit.
|
||||
2. Values without a start contain both services.
|
||||
|
||||
Logs are not covered: the logs collector stamps tag rows with the ingestion hour, not the
|
||||
log's timestamp, and the fixture writes the log's timestamp.
|
||||
"""
|
||||
signal = "traces"
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_traces(
|
||||
[
|
||||
Traces(timestamp=now - timedelta(days=3), resources={"service.name": "archived-service"}),
|
||||
Traces(timestamp=now, resources={"service.name": "live-service"}),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": signal,
|
||||
"name": "service.name",
|
||||
"startUnixMilli": int((now - timedelta(hours=1)).timestamp() * 1000),
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json()["data"]["values"]["stringValues"] == ["live-service"], "values last seen before the start must be dropped"
|
||||
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=2,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={"signal": signal, "name": "service.name"},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert set(response.json()["data"]["values"]["stringValues"]) == {"archived-service", "live-service"}
|
||||
@@ -71,7 +71,7 @@ def test_v1_get_serves_legacy_shape(
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
filters = response.json()["data"]["filters"]
|
||||
assert filters[0]["key"] == "duration_nano"
|
||||
assert filters[0]["type"] == "tag"
|
||||
assert filters[0]["type"] == "", "span fields have no v3 attribute type"
|
||||
assert filters[0]["dataType"] == "float64"
|
||||
assert all("name" not in legacy_filter for legacy_filter in filters)
|
||||
|
||||
@@ -274,3 +274,36 @@ def test_update_quick_filters_rejects_invalid_input(
|
||||
timeout=2,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
|
||||
|
||||
def test_default_traces_filters_are_served_as_the_fields_api_serves_them(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: types.Operation, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
):
|
||||
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v2/quick_filters/traces"),
|
||||
headers={"Authorization": f"Bearer {admin_token}"},
|
||||
timeout=2,
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
filters = {field_key["name"]: field_key for field_key in response.json()["data"]["filters"]}
|
||||
|
||||
assert "hasError" not in filters
|
||||
assert (filters["has_error"]["fieldContext"], filters["has_error"]["fieldDataType"]) == ("span", "bool")
|
||||
assert (filters["name"]["fieldContext"], filters["name"]["fieldDataType"]) == ("span", "string")
|
||||
assert (filters["duration_nano"]["fieldContext"], filters["duration_nano"]["fieldDataType"]) == ("span", "number")
|
||||
assert (filters["http.route"]["fieldContext"], filters["http.route"]["fieldDataType"]) == ("attribute", "string")
|
||||
|
||||
for name in ("has_error", "name"):
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
|
||||
headers={"Authorization": f"Bearer {admin_token}"},
|
||||
timeout=2,
|
||||
params={"signal": "traces", "searchText": name, "fieldContext": filters[name]["fieldContext"]},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
served = response.json()["data"]["keys"][name]
|
||||
assert (filters[name]["fieldContext"], filters[name]["fieldDataType"]) in [(key["fieldContext"], key["fieldDataType"]) for key in served]
|
||||
|
||||
Reference in New Issue
Block a user