Compare commits

..

3 Commits

Author SHA1 Message Date
srikanthccv
8369b4c11e test(fields): parametrize the fields keys and values integration cases
- keys by context for every log and span context, including scope and
  attribute, and the span context keeping attributes
- keys by data type, with number, int64 and float64 as one family
- case-insensitive search with and without a context prefix
- bool values for the calculated span field, a stored bool attribute and a
  caller-declared bool type; start filter for logs and traces

Assisted-by: Claude Fable 5.1
Claude-Session: https://claude.ai/code/session_01NNQrizqt5XQ1N2NNAcuAi8
2026-09-02 15:25:05 +05:30
srikanthccv
bd3c2cd5ba fix(telemetrymetadata): keep span-context key lookups lenient and return bool values
- the span context must keep returning attribute keys from the keys table:
  queries spell span attributes as span.<name> and resolve them through the
  same lookup; only the static fields are filtered by context now
- the fields/values handler dropped boolValues when assembling the response
- replace the ClickHouse-mock unit tests with integration tests at the HTTP
  boundary (tests/integration/tests/queriercommon/07_fields_keys_values.py)

Assisted-by: Claude Fable 5.1
Claude-Session: https://claude.ai/code/session_01NNQrizqt5XQ1N2NNAcuAi8
2026-09-02 14:13:12 +05:30
srikanthccv
fd30597641 feat(telemetrymetadata): bool values, start filter, and context-aware static keys
- fields/values returns boolValues for bool fields: when the caller asks for
  the bool type, when the name is a static bool field of the signal, or when
  the tag table holds bool rows for the key
- span and log values honor startUnixMilli against the tag tables
- static (intrinsic and calculated) keys honor fieldContext and fieldDataType
  and match case-insensitively; span and log contexts are served by the
  static fields instead of returning attribute keys or nothing
- isRoot/isEntryPoint are typed bool and meter keys carry a string data type
- skip the column evolution lookup when there are no keys to enrich

Assisted-by: Claude Fable 5.1
Claude-Session: https://claude.ai/code/session_01NNQrizqt5XQ1N2NNAcuAi8
2026-09-02 13:45:54 +05:30
16 changed files with 530 additions and 166 deletions

View File

@@ -57,8 +57,8 @@ function MenuItemGenerator({
handleExplorerTabChange(currentPanelType, {
query,
viewName: name,
viewKey: id,
name,
id,
});
},
[viewData, handleExplorerTabChange],

View File

@@ -452,15 +452,15 @@ function ExplorerOptions({
if (handleChangeSelectedView) {
handleChangeSelectedView(panelTypeToExplorerView[currentPanelType], {
query,
viewName: name,
viewKey: id,
name,
id,
});
} else {
// to remove this after traces cleanup
handleExplorerTabChange(currentPanelType, {
query,
viewName: name,
viewKey: id,
name,
id,
});
}
},

View File

@@ -115,8 +115,8 @@ export default function SavedViews({
currentPanelType,
{
query,
viewName: name,
viewKey: id,
name,
id,
},
SOURCEPAGE_VS_ROUTES[selectedEntity],
);

View File

@@ -16,6 +16,7 @@ import {
} from 'constants/antlrQueryConstants';
import { FeatureKeys } from 'constants/features';
import { useActiveLog } from 'hooks/logs/useActiveLog';
import { useGetSavedViewParams } from 'hooks/saveViews/useGetSavedViewParams';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { ICurrentQueryData } from 'hooks/useHandleExplorerTabChange';
import { useNotifications } from 'hooks/useNotifications';
@@ -48,6 +49,7 @@ function BodyTitleRenderer({
const { featureFlags } = useAppContext();
const [, setCopy] = useCopyToClipboard();
const { notifications } = useNotifications();
const { viewName } = useGetSavedViewParams();
const cleanedNodeKey = removeObjectFromString(nodeKey);
const isBodyJsonQueryEnabled =
@@ -121,6 +123,8 @@ function BodyTitleRenderer({
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
@@ -133,6 +137,7 @@ function BodyTitleRenderer({
stagedQuery,
updateQueriesData,
value,
viewName,
]);
const onClickHandler = (key: string): void => {

View File

@@ -12,6 +12,7 @@ import ROUTES from 'constants/routes';
import { ChangeViewFunctionType } from 'container/ExplorerOptions/types';
import { RESTRICTED_SELECTED_FIELDS } from 'container/LogsFilters/config';
import { MetricsType } from 'container/MetricsApplication/constant';
import { useGetSavedViewParams } from 'hooks/saveViews/useGetSavedViewParams';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { ICurrentQueryData } from 'hooks/useHandleExplorerTabChange';
import {
@@ -139,6 +140,7 @@ export default function TableViewActions(
const { pathname } = useLocation();
const { stagedQuery, updateQueriesData } = useQueryBuilder();
const { viewName } = useGetSavedViewParams();
const { dataType, logType: fieldType } = getFieldAttributes(record.field);
// there is no option for where clause in old logs explorer and live logs page or infra monitoring
@@ -199,6 +201,8 @@ export default function TableViewActions(
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
@@ -210,6 +214,7 @@ export default function TableViewActions(
fieldType,
dataType,
handleChangeSelectedView,
viewName,
]);
const handleReplaceFilter = useCallback((): void => {
@@ -259,6 +264,8 @@ export default function TableViewActions(
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
@@ -271,6 +278,7 @@ export default function TableViewActions(
dataType,
fieldData,
handleChangeSelectedView,
viewName,
]);
// Memoize textToCopy computation

View File

@@ -272,6 +272,8 @@ describe('TableViewActions', () => {
expect(defaultProps.handleChangeSelectedView).toHaveBeenCalledWith(
ExplorerViews.TIMESERIES,
expect.objectContaining({
name: '',
id: 'test-query-id',
query: expect.objectContaining({
builder: expect.objectContaining({
queryData: expect.arrayContaining([

View File

@@ -5,6 +5,7 @@ import { convertFiltersToExpression } from 'components/QueryBuilderV2/utils';
import { FeatureKeys } from 'constants/features';
import ROUTES from 'constants/routes';
import { ChangeViewFunctionType } from 'container/ExplorerOptions/types';
import { useGetSavedViewParams } from 'hooks/saveViews/useGetSavedViewParams';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { ICurrentQueryData } from 'hooks/useHandleExplorerTabChange';
import { ExplorerViews } from 'pages/LogsExplorer/utils';
@@ -56,6 +57,7 @@ export function useLogAttributeActions({
const { pathname } = useLocation();
const { stagedQuery, updateQueriesData } = useQueryBuilder();
const { featureFlags } = useAppContext();
const { viewName } = useGetSavedViewParams();
const isBodyJsonQueryEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.USE_JSON_BODY)
@@ -108,6 +110,8 @@ export function useLogAttributeActions({
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
handleChangeSelectedView?.(ExplorerViews.LIST, queryData);
@@ -116,6 +120,7 @@ export function useLogAttributeActions({
stagedQuery,
isBodyJsonQueryEnabled,
updateQueriesData,
viewName,
handleChangeSelectedView,
onApplyLogFilter,
],
@@ -142,6 +147,8 @@ export function useLogAttributeActions({
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
handleChangeSelectedView?.(ExplorerViews.TIMESERIES, queryData);
@@ -150,6 +157,7 @@ export function useLogAttributeActions({
stagedQuery,
isBodyJsonQueryEnabled,
updateQueriesData,
viewName,
handleChangeSelectedView,
],
);
@@ -175,6 +183,8 @@ export function useLogAttributeActions({
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
handleChangeSelectedView?.(ExplorerViews.LIST, queryData);
@@ -183,6 +193,7 @@ export function useLogAttributeActions({
stagedQuery,
isBodyJsonQueryEnabled,
updateQueriesData,
viewName,
handleChangeSelectedView,
],
);

View File

@@ -78,6 +78,8 @@ function AllAttributes({
PANEL_TYPES.TIME_SERIES,
{
query: compositeQuery,
name: metricName,
id: metricName,
},
ROUTES.METRICS_EXPLORER_EXPLORER,
true,
@@ -107,6 +109,8 @@ function AllAttributes({
PANEL_TYPES.TIME_SERIES,
{
query: compositeQuery,
name: metricName,
id: metricName,
},
ROUTES.METRICS_EXPLORER_EXPLORER,
true,

View File

@@ -92,6 +92,8 @@ function MetricDetails({
PANEL_TYPES.TIME_SERIES,
{
query: compositeQuery,
name: metricName,
id: metricName,
},
ROUTES.METRICS_EXPLORER_EXPLORER,
true,

View File

@@ -10,8 +10,8 @@ import { useGetSavedViewParams } from './saveViews/useGetSavedViewParams';
import { useQueryBuilder } from './queryBuilder/useQueryBuilder';
export interface ICurrentQueryData {
viewName?: string;
viewKey?: string;
name: string;
id: string;
query: Query;
}
@@ -57,8 +57,6 @@ export const useHandleExplorerTabChange = (): {
[currentQuery, updateAllQueriesOperators, updateQueriesData],
);
//TODO: this util is used not just to change explorer tab but also
// for changing just the query or saved view. consider renaming this.
const handleExplorerTabChange = useCallback(
(
type: string,
@@ -79,8 +77,8 @@ export const useHandleExplorerTabChange = (): {
query,
{
[QueryParams.panelTypes]: newPanelType,
[QueryParams.viewName]: currentQueryData?.viewName || viewName,
[QueryParams.viewKey]: currentQueryData?.viewKey || viewKey,
[QueryParams.viewName]: currentQueryData?.name || viewName,
[QueryParams.viewKey]: currentQueryData?.id || viewKey,
},
redirectToUrl,
undefined,
@@ -91,8 +89,8 @@ export const useHandleExplorerTabChange = (): {
query,
{
[QueryParams.panelTypes]: newPanelType,
[QueryParams.viewName]: currentQueryData?.viewName || viewName,
[QueryParams.viewKey]: currentQueryData?.viewKey || viewKey,
[QueryParams.viewName]: currentQueryData?.name || viewName,
[QueryParams.viewKey]: currentQueryData?.id || viewKey,
},
undefined,
undefined,

View File

@@ -209,8 +209,8 @@ function SaveView(): JSX.Element {
currentPanelType,
{
query,
viewName: name,
viewKey: id,
name,
id,
},
SOURCEPAGE_VS_ROUTES[sourcepage],
);

View File

@@ -82,6 +82,7 @@ func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Reques
values := &telemetrytypes.TelemetryFieldValues{
StringValues: allValues.StringValues,
BoolValues: allValues.BoolValues,
NumberValues: allValues.NumberValues,
RelatedValues: relatedValues,
}

View File

@@ -0,0 +1,43 @@
package telemetrymetadata
import (
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// boolFieldValues is the suggestion set for a bool field, optionally narrowed
// by the search text.
func boolFieldValues(searchText string) *telemetrytypes.TelemetryFieldValues {
values := &telemetrytypes.TelemetryFieldValues{}
needle := strings.ToLower(searchText)
for _, v := range []bool{true, false} {
if needle == "" || strings.Contains(strconv.FormatBool(v), needle) {
values.BoolValues = append(values.BoolValues, v)
}
}
return values
}
// isKnownBoolField is true when the caller asked for the bool data type, or
// when the name is one of the signal's static bool fields and the requested
// context does not rule that static field out.
func isKnownBoolField(selector *telemetrytypes.FieldValueSelector, staticFields ...map[string]telemetrytypes.TelemetryFieldKey) bool {
if selector.FieldDataType == telemetrytypes.FieldDataTypeBool {
return true
}
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
return false
}
for _, fields := range staticFields {
field, ok := fields[selector.Name]
if !ok || field.FieldDataType != telemetrytypes.FieldDataTypeBool {
continue
}
if selector.FieldContext == telemetrytypes.FieldContextUnspecified || selector.FieldContext == field.FieldContext {
return true
}
}
return false
}

View File

@@ -187,8 +187,6 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
).From(t.tracesDBName + "." + t.spanAttributesKeysTblName)
var limit int
searchTexts := []string{}
conds := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
@@ -208,7 +206,6 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
fieldKeyConds = append(fieldKeyConds, sb.ILike("tagKey", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
}
searchTexts = append(searchTexts, fieldKeySelector.Name)
// now look at the field context
// we don't write most of intrinsic fields to keys table
// for this reason we don't want to apply tagType if the field context
@@ -288,41 +285,23 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
// hit the limit? (only counting DB results)
complete := rowCount <= limit
staticKeys := []string{"isRoot", "isEntryPoint"}
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.IntrinsicFields)...)
staticKeys = append(staticKeys, maps.Keys(tracestelemetryschema.CalculatedFields)...)
// Add the matching static fields: the span scope selectors, the intrinsic
// columns and the calculated columns. These don't count towards the limit
staticFields := []telemetrytypes.TelemetryFieldKey{
{Name: "isRoot", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeBool},
{Name: "isEntryPoint", Signal: telemetrytypes.SignalTraces, FieldContext: telemetrytypes.FieldContextSpan, FieldDataType: telemetrytypes.FieldDataTypeBool},
}
staticFields = append(staticFields, maps.Values(tracestelemetryschema.IntrinsicFields)...)
staticFields = append(staticFields, maps.Values(tracestelemetryschema.CalculatedFields)...)
// Add matching intrinsic and matching calculated fields
// These don't count towards the limit
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
for _, field := range staticFields {
if !staticFieldMatchesAny(field, fieldKeySelectors) {
continue
}
if found {
if field, exists := tracestelemetryschema.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
if field, exists := tracestelemetryschema.CalculatedFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: key,
FieldContext: telemetrytypes.FieldContextSpan,
Signal: telemetrytypes.SignalTraces,
})
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
continue
}
keys = append(keys, &field)
}
if err = t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
@@ -542,12 +521,6 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
allArgs = append(allArgs, args...)
}
if len(queries) == 0 {
// No matching contexts, return empty result
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
}
// Combine queries with UNION ALL
var limit int
for _, fieldKeySelector := range fieldKeySelectors {
limit += fieldKeySelector.Limit
@@ -556,7 +529,15 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
limit = 1000
}
mainQuery := fmt.Sprintf(`
keys := []*telemetrytypes.TelemetryFieldKey{}
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
rowCount := 0
// the log and scope contexts have no keys table; they are served by the
// static fields appended below
if len(queries) > 0 {
// Combine queries with UNION ALL
mainQuery := fmt.Sprintf(`
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
FROM (
%s
@@ -566,103 +547,75 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
LIMIT %d
`, strings.Join(queries, " UNION ALL "), limit+1)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
defer rows.Close()
keys := []*telemetrytypes.TelemetryFieldKey{}
parentTypes := make(map[string][]telemetrytypes.FieldDataType)
rowCount := 0
searchTexts := []string{}
// Collect search texts for static field matching
for _, fieldKeySelector := range fieldKeySelectors {
searchTexts = append(searchTexts, fieldKeySelector.Name)
}
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
defer rows.Close()
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
// if the user did not also directly request this name — a field like "education" can be
// both a parent of "education[].name" and an explicitly queried field in its own right.
switch fieldDataType {
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
parentTypes[name] = append(parentTypes[name], fieldDataType)
if !mapOfRequestedSelectors[name] {
continue // skip; don't register the key.
for rows.Next() {
rowCount++
// reached the limit, we know there are more results
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
// ArrayJSON/ArrayDynamic body rows for parent paths are needed by the JSON access plan
// builder (enrichJSONKeys). Always record them in parentTypes. Only skip adding to keys
// if the user did not also directly request this name — a field like "education" can be
// both a parent of "education[].name" and an explicitly queried field in its own right.
switch fieldDataType {
case telemetrytypes.FieldDataTypeArrayJSON, telemetrytypes.FieldDataTypeArrayDynamic:
if fieldContext == telemetrytypes.FieldContextBody && parentPaths[name] {
parentTypes[name] = append(parentTypes[name], fieldDataType)
if !mapOfRequestedSelectors[name] {
continue // skip; don't register the key.
}
}
}
}
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
// if there is no materialised column, create a key with the field context and data type
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
// if there is no materialised column, create a key with the field context and data type
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
}
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
}
// hit the limit? (only counting DB results)
complete := rowCount <= limit
staticKeys := []string{}
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
// Add matching intrinsic and matching calculated fields
// These don't count towards the limit
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
// Add the matching intrinsic columns. These don't count towards the limit
for _, field := range maps.Values(logstelemetryschema.IntrinsicFields) {
if !staticFieldMatchesAny(field, fieldKeySelectors) {
continue
}
if found {
if field, exists := logstelemetryschema.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
continue
}
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: key,
FieldContext: telemetrytypes.FieldContextLog,
Signal: telemetrytypes.SignalLogs,
})
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
continue
}
keys = append(keys, &field)
}
// enrich body keys with promoted paths, indexes, and JSON access plans
@@ -836,11 +789,6 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
keys := []*telemetrytypes.TelemetryFieldKey{}
rowCount := 0
searchTexts := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
searchTexts = append(searchTexts, fieldKeySelector.Name)
}
for rows.Next() {
rowCount++
@@ -877,24 +825,15 @@ func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors
complete := rowCount <= limit
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
staticKeys := maps.Keys(audittelemetryschema.IntrinsicFields)
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
// Add the matching intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
for _, field := range maps.Values(audittelemetryschema.IntrinsicFields) {
if !staticFieldMatchesAny(field, fieldKeySelectors) {
continue
}
if found {
if field, exists := audittelemetryschema.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
}
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; added {
continue
}
keys = append(keys, &field)
}
return keys, complete, nil
@@ -1091,9 +1030,12 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
}
// meter labels are stored as strings in the labels JSON and have no
// attribute context, so only the data type is known
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalMetrics,
Name: name,
Signal: telemetrytypes.SignalMetrics,
FieldDataType: telemetrytypes.FieldDataTypeString,
})
}
@@ -1512,12 +1454,24 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
limit = 50
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.tracesDBName + "." + t.tracesFieldsTblName)
// bool rows in the tag table carry no value; the two possible values are
// known without a query
if isKnownBoolField(fieldValueSelector, tracestelemetryschema.IntrinsicFields, tracestelemetryschema.CalculatedFields) {
return boolFieldValues(fieldValueSelector.Value), true, nil
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value, tag_data_type").From(t.tracesDBName + "." + t.tracesFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
}
// unix_milli is the hour bucket a value was written in and rows are
// deduplicated per day, so this is a day-granular "seen since" filter
if fieldValueSelector.StartUnixMilli != 0 {
sb.Where(sb.GE("unix_milli", fieldValueSelector.StartUnixMilli))
}
// now look at the field context
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
@@ -1565,10 +1519,20 @@ func (t *telemetryMetaStore) getSpanFieldValues(ctx context.Context, fieldValueS
var stringValue string
var numberValue float64
if err := rows.Scan(&stringValue, &numberValue); err != nil {
var tagDataType string
if err := rows.Scan(&stringValue, &numberValue, &tagDataType); err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
// bool rows carry no value; the key's presence is enough to know the
// two values it can take
if tagDataType == telemetrytypes.FieldDataTypeBool.TagDataType() {
if len(values.BoolValues) == 0 {
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
}
continue
}
// Only add values if we haven't hit the limit yet
if totalCount < limit {
if _, ok := seen[stringValue]; !ok && stringValue != "" {
@@ -1602,12 +1566,24 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
limit = 50
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
// bool rows in the tag table carry no value; the two possible values are
// known without a query
if isKnownBoolField(fieldValueSelector, logstelemetryschema.IntrinsicFields) {
return boolFieldValues(fieldValueSelector.Value), true, nil
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value, tag_data_type").From(t.logsDBName + "." + t.logsFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
}
// unix_milli is the hour bucket a value was written in and rows are
// deduplicated per day, so this is a day-granular "seen since" filter
if fieldValueSelector.StartUnixMilli != 0 {
sb.Where(sb.GE("unix_milli", fieldValueSelector.StartUnixMilli))
}
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
}
@@ -1653,10 +1629,20 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
var stringValue string
var numberValue float64
if err := rows.Scan(&stringValue, &numberValue); err != nil {
var tagDataType string
if err := rows.Scan(&stringValue, &numberValue, &tagDataType); err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetLogsKeys.Error())
}
// bool rows carry no value; the key's presence is enough to know the
// two values it can take
if tagDataType == telemetrytypes.FieldDataTypeBool.TagDataType() {
if len(values.BoolValues) == 0 {
values.BoolValues = boolFieldValues(fieldValueSelector.Value).BoolValues
}
continue
}
// Only add values if we haven't hit the limit yet
if totalCount < limit {
if _, ok := seen[stringValue]; !ok && stringValue != "" {
@@ -2097,6 +2083,18 @@ func populateAllUnspecifiedValues(allUnspecifiedValues *telemetrytypes.Telemetry
}
}
for _, value := range values.BoolValues {
if totalCount >= limit {
complete = false
break
}
if _, ok := mapOfValues[value]; !ok {
mapOfValues[value] = true
allUnspecifiedValues.BoolValues = append(allUnspecifiedValues.BoolValues, value)
totalCount++
}
}
for _, value := range values.RelatedValues {
if totalCount >= limit {
complete = false
@@ -2467,6 +2465,10 @@ func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Conte
// updateColumnEvolutionMetadataForKeys updates the evolution field for keys.
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) error {
// an empty selector list would run the evolution query without a filter
if len(keysToUpdate) == 0 {
return nil
}
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
for _, keySelector := range keysToUpdate {

View File

@@ -0,0 +1,53 @@
package telemetrymetadata
import (
"strings"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
func staticFieldMatchesAny(field telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) bool {
for _, selector := range selectors {
if staticFieldMatches(field, selector) {
return true
}
}
return false
}
// staticFieldMatches mirrors the keys-table lookup for a static field: the
// requested context and data type, when given, must agree with the field's,
// and the name matches case-insensitively, as a substring for fuzzy selectors
// and as the whole name for exact ones.
func staticFieldMatches(field telemetrytypes.TelemetryFieldKey, selector *telemetrytypes.FieldKeySelector) bool {
if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != field.FieldContext {
return false
}
if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && !sameDataTypeFamily(selector.FieldDataType, field.FieldDataType) {
return false
}
if selector.Name == "" {
return true
}
if selector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
return strings.EqualFold(field.Name, selector.Name)
}
return strings.Contains(strings.ToLower(field.Name), strings.ToLower(selector.Name))
}
// sameDataTypeFamily treats the numeric types as one family: static fields
// declare "number" while callers may ask for int64 or float64.
func sameDataTypeFamily(requested, actual telemetrytypes.FieldDataType) bool {
if requested == actual {
return true
}
return isNumericDataType(requested) && isNumericDataType(actual)
}
func isNumericDataType(dataType telemetrytypes.FieldDataType) bool {
switch dataType {
case telemetrytypes.FieldDataTypeNumber, telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64:
return true
}
return false
}

View File

@@ -0,0 +1,235 @@
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.traces import Traces
@pytest.mark.parametrize(
"signal,field_context,present,absent",
[
pytest.param("logs", "log", {"severity_text": "log", "body": "log", "trace_id": "log"}, ["code.file", "scope_name"], id="log_context_lists_log_intrinsics"),
pytest.param("logs", "scope", {"scope_name": "scope", "scope_version": "scope"}, ["severity_text", "body", "code.file"], id="scope_context_lists_scope_intrinsics_for_logs"),
pytest.param("logs", "attribute", {"code.file": "attribute"}, ["body", "scope_name"], id="attribute_context_excludes_log_intrinsics"),
pytest.param("traces", "span", {"name": "span", "has_error": "span", "isRoot": "span", "http.method": "attribute"}, ["scope.name"], id="span_context_lists_span_intrinsics_and_attributes"),
pytest.param("traces", "scope", {"scope.name": "scope", "scope.version": "scope"}, ["name", "has_error", "isRoot"], id="scope_context_lists_scope_intrinsics_for_traces"),
pytest.param("traces", "resource", {"host.name": "resource"}, ["name", "has_error", "isRoot", "http.method"], id="resource_context_excludes_span_intrinsics"),
pytest.param("traces", "attribute", {"http.method": "attribute"}, ["name", "has_error", "isRoot", "host.name"], id="attribute_context_excludes_span_intrinsics"),
],
)
def test_fields_keys_by_context(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
insert_traces: Callable[[list[Traces]], None],
signal: str,
field_context: str,
present: dict[str, str],
absent: list[str],
) -> None:
"""
Setup:
Insert a log with a code.file attribute and a span with an http.method attribute and a host.name resource.
Tests:
1. Keys for a context list that context's intrinsic columns and the stored keys the context maps to,
each with its context; intrinsics of other contexts are not listed. The span context also keeps
listing attributes because `span.<attribute>` resolves attributes in queries.
"""
now = datetime.now(tz=UTC)
insert_logs([Logs(timestamp=now, attributes={"code.file": "/opt/integration.go"}, body="a log line")])
insert_traces([Traces(timestamp=now, resources={"host.name": "linux-001"}, attributes={"http.method": "GET"})])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "fieldContext": field_context},
)
assert response.status_code == HTTPStatus.OK
keys = response.json()["data"]["keys"]
listed = {name: [key["fieldContext"] for key in keys.get(name, [])] for name in present}
assert listed == {name: [context] for name, context in present.items()}, f"keys for the {field_context} context"
assert [name for name in absent if name in keys] == [], f"keys that do not belong to the {field_context} context"
@pytest.mark.parametrize(
"signal,field_context,field_data_type,present,absent",
[
pytest.param("traces", "span", "float64", ["duration_nano", "status_code"], ["name", "has_error"], id="float64_matches_number_span_intrinsics"),
pytest.param("traces", "span", "int64", ["duration_nano", "status_code"], ["name", "has_error"], id="int64_matches_number_span_intrinsics"),
pytest.param("traces", "span", "bool", ["has_error", "isRoot", "isEntryPoint"], ["name", "duration_nano"], id="bool_matches_bool_span_intrinsics"),
pytest.param("traces", "span", "string", ["name", "http_method"], ["duration_nano", "has_error"], id="string_matches_string_span_intrinsics"),
pytest.param("logs", "log", "number", ["severity_number", "trace_flags"], ["severity_text", "body"], id="number_matches_number_log_intrinsics"),
],
)
def test_fields_keys_by_data_type(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
signal: str,
field_context: str,
field_data_type: str,
present: list[str],
absent: list[str],
) -> None:
"""
Tests:
1. A data type filter keeps the intrinsic columns of that type; number, int64 and float64 are one family.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "fieldContext": field_context, "fieldDataType": field_data_type},
)
assert response.status_code == HTTPStatus.OK
keys = response.json()["data"]["keys"]
assert [name for name in present if name not in keys] == [], f"intrinsics of type {field_data_type}"
assert [name for name in absent if name in keys] == [], f"intrinsics not of type {field_data_type}"
@pytest.mark.parametrize(
"signal,search_text,present",
[
pytest.param("logs", "SEVERITY", ["severity_text", "severity_number"], id="upper_case_search_logs"),
pytest.param("traces", "HTTP_", ["http_method", "http_host", "http_url"], id="upper_case_search_traces"),
pytest.param("traces", "Duration", ["duration_nano"], id="mixed_case_search_traces"),
pytest.param("traces", "span.HAS_ERR", ["has_error"], id="context_prefix_with_upper_case_search"),
],
)
def test_fields_keys_search_matches_intrinsics_case_insensitively(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
signal: str,
search_text: str,
present: list[str],
) -> None:
"""
Tests:
1. The search text matches intrinsic columns case-insensitively, as it does for stored keys,
with or without a context prefix.
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/keys"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "searchText": search_text},
)
assert response.status_code == HTTPStatus.OK
keys = response.json()["data"]["keys"]
assert [name for name in present if name not in keys] == [], f"intrinsics matching {search_text!r}"
@pytest.mark.parametrize(
"signal,params,expected",
[
pytest.param("traces", {"name": "has_error"}, [True, False], id="calculated_bool_span_field"),
pytest.param("traces", {"name": "has_error", "fieldContext": "span"}, [True, False], id="calculated_bool_span_field_with_context"),
pytest.param("traces", {"name": "has_error", "searchText": "tr"}, [True], id="search_text_narrows_bool_values"),
pytest.param("logs", {"name": "retry"}, [True, False], id="bool_attribute_from_tag_rows"),
pytest.param("logs", {"name": "retry", "fieldContext": "attribute"}, [True, False], id="bool_attribute_with_context"),
pytest.param("logs", {"name": "never_seen", "fieldDataType": "bool"}, [True, False], id="declared_bool_type_needs_no_rows"),
],
)
def test_fields_values_bool_fields(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
signal: str,
params: dict[str, str],
expected: list[bool],
) -> None:
"""
Setup:
Insert a log with a bool attribute.
Tests:
1. Values for a bool field are true and false (narrowed by the search text): for the calculated span
field, for a stored bool attribute whose tag rows carry no value, and for a key the caller declares bool.
"""
insert_logs([Logs(timestamp=datetime.now(tz=UTC), attributes={"retry": True}, body="retrying")])
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, **params},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["data"]["values"]["boolValues"] == expected
assert response.json()["data"]["complete"] is True
@pytest.mark.parametrize("signal", [pytest.param("logs", id="logs"), pytest.param("traces", id="traces")])
def test_fields_values_start_excludes_values_not_seen_since(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[list[Logs]], None],
insert_traces: Callable[[list[Traces]], None],
signal: str,
) -> None:
"""
Setup:
Insert a log and a span three days old and a log and a span now, with different service names.
Tests:
1. Values with startUnixMilli an hour ago contain only the service seen now.
2. Values without a start contain both services.
"""
now = datetime.now(tz=UTC)
insert_logs(
[
Logs(timestamp=now - timedelta(days=3), resources={"service.name": "archived-service"}, body="old"),
Logs(timestamp=now, resources={"service.name": "live-service"}, body="new"),
]
)
insert_traces(
[
Traces(timestamp=now - timedelta(days=3), resources={"service.name": "archived-service"}),
Traces(timestamp=now, resources={"service.name": "live-service"}),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={
"signal": signal,
"name": "service.name",
"startUnixMilli": int((now - timedelta(hours=1)).timestamp() * 1000),
},
)
assert response.status_code == HTTPStatus.OK
assert response.json()["data"]["values"]["stringValues"] == ["live-service"], "values last seen before the start must be dropped"
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
timeout=2,
headers={"authorization": f"Bearer {token}"},
params={"signal": signal, "name": "service.name"},
)
assert response.status_code == HTTPStatus.OK
assert set(response.json()["data"]["values"]["stringValues"]) == {"archived-service", "live-service"}