Compare commits

..

2 Commits

Author SHA1 Message Date
swapnil-signoz
0666bb4bfa feat: adding migration for dashboard change 2026-08-18 20:03:39 +05:30
swapnil-signoz
56027c4f34 refactor: adding FunctionName variable in Lambda dashboard 2026-08-18 13:30:01 +05:30
37 changed files with 1183 additions and 1120 deletions

View File

@@ -1,5 +1,9 @@
// temporary flag to be removed with old log details code.
export const isLogDetailsV2 = true;
import getLocalStorage from 'api/browser/localstorage/get';
import { LOCALSTORAGE } from 'constants/localStorage';
// Temp feature flag before actual roll-out
export const isLogDetailsV2 =
getLocalStorage(LOCALSTORAGE.LOG_DETAILS_V2) === 'true';
export const VIEW_TYPES = {
OVERVIEW: 'OVERVIEW',

View File

@@ -13,6 +13,7 @@ export enum LOCALSTORAGE {
TRACES_LIST_COLUMNS = 'TRACES_LIST_COLUMNS',
LOGS_LIST_COLUMNS = 'LOGS_LIST_COLUMNS',
LOGS_LIST_COLUMN_SIZING = 'LOGS_LIST_COLUMN_SIZING',
LOG_DETAILS_V2 = 'LOG_DETAILS_V2',
LOGGED_IN_USER_NAME = 'LOGGED_IN_USER_NAME',
LOGGED_IN_USER_EMAIL = 'LOGGED_IN_USER_EMAIL',
CHAT_SUPPORT = 'CHAT_SUPPORT',

View File

@@ -23,9 +23,9 @@ import { useLogAttributeActions } from './hooks/useLogAttributeActions';
import TableView from './TableView';
import {
aggregateAttributesResourcesToObject,
buildPrettyViewData,
getBodyDisplayString,
getSanitizedLogBody,
parseJsonStringBody,
removeEscapeCharacters,
} from './utils';
@@ -71,7 +71,11 @@ function Overview({
if (isLogDetailsV2) {
const raw = aggregateAttributesResourcesToObject(logData);
const prettyData = buildPrettyViewData(raw);
const prettyData = Object.fromEntries(
Object.entries({ ...raw, body: parseJsonStringBody(raw.body) }).filter(
([, value]) => value !== undefined,
),
);
return (
<div className="overview-container">
<DataViewer

View File

@@ -1,6 +1,6 @@
export enum LogAttributeBucket {
ATTRIBUTES = 'attributes',
RESOURCES = 'resource',
RESOURCES = 'resources',
SCOPE = 'scope',
}

View File

@@ -33,7 +33,7 @@ describe('buildLogFilterTarget', () => {
it('maps `resources` with Resource type', () => {
expect(
buildLogFilterTarget(['resource', 'service.name'], 'api', true),
buildLogFilterTarget(['resources', 'service.name'], 'api', true),
).toMatchObject({
fieldKey: 'service.name',
metricsType: MetricsType.Resource,
@@ -53,30 +53,6 @@ describe('buildLogFilterTarget', () => {
});
});
describe('nested attribute values (parsed JSON)', () => {
it('marks a sub-field of a parsed attribute copy-only (restricted, no group-by)', () => {
const t = buildLogFilterTarget(['attributes', 'payload', 'x'], 1, true);
expect(t.isRestricted).toBe(true);
expect(t.groupBySupported).toBe(false);
expect(t.groupByKey).toBeUndefined();
});
it('leaves a top-level attribute (depth 2) filterable', () => {
const t = buildLogFilterTarget(['attributes', 'payload'], 'v', true);
expect(t.isRestricted).toBe(false);
expect(t.groupBySupported).toBe(true);
});
it('does not restrict nested resource/scope values', () => {
expect(
buildLogFilterTarget(['resource', 'k8s', 'pod'], 'p', true).isRestricted,
).toBe(false);
expect(
buildLogFilterTarget(['scope', 'a', 'b'], 'v', true).isRestricted,
).toBe(false);
});
});
describe('restricted fields (timestamp / id)', () => {
it.each(['timestamp', 'id'])(
'marks %s restricted with no group-by',
@@ -89,30 +65,6 @@ describe('buildLogFilterTarget', () => {
);
});
describe('group-by-restricted fields (trace_id)', () => {
it('allows filtering but not group-by on top-level trace_id', () => {
const t = buildLogFilterTarget(['trace_id'], 'abc123', true);
expect(t.isRestricted).toBe(false);
expect(t.filterInOperator).toBe('=');
expect(t.groupBySupported).toBe(false);
expect(t.groupByKey).toBeUndefined();
});
it.each([
['resource', ['resource', 'trace_id']],
['attributes', ['attributes', 'trace_id']],
])(
'blocks group-by on a %s field named trace_id, keeping filter',
(_bucket, path) => {
const t = buildLogFilterTarget(path as string[], 'abc123', true);
expect(t.isRestricted).toBe(false);
expect(t.filterInOperator).toBe('=');
expect(t.groupBySupported).toBe(false);
expect(t.groupByKey).toBeUndefined();
},
);
});
describe('body scalars', () => {
it('maps a top-level body scalar to body.<key> with =/!=, groupable when json body on', () => {
const t = buildLogFilterTarget(['body', 'message'], 'hello', true);

View File

@@ -5,10 +5,7 @@ import {
QUERY_BUILDER_FUNCTIONS,
} from 'constants/antlrQueryConstants';
import { OPERATORS as QUERY_BUILDER_OPERATORS } from 'constants/queryBuilder';
import {
RESTRICTED_GROUP_BY_FIELDS,
RESTRICTED_SELECTED_FIELDS,
} from 'container/LogsFilters/config';
import { RESTRICTED_SELECTED_FIELDS } from 'container/LogsFilters/config';
import { MetricsType } from 'container/MetricsApplication/constant';
import { getOperatorValue } from 'container/QueryBuilder/filters/QueryBuilderSearch/utils';
import { chooseAutocompleteFromCustomValue } from 'lib/newQueryBuilder/chooseAutocompleteFromCustomValue';
@@ -86,24 +83,15 @@ export const buildLogFilterTarget = (
if (root !== 'body') {
const fieldKey =
fieldKeyPath.length > 1 ? fieldKeyPath.slice(1).join('.') : String(root);
// Temporarily removing filter/group-by support for nested attributes.
// This will be removed once backend starts to support these actions.
const isNestedAttributeValue =
root === LogAttributeBucket.ATTRIBUTES && fieldKeyPath.length > 2;
const isRestricted =
RESTRICTED_SELECTED_FIELDS.includes(fieldKey) || isNestedAttributeValue;
const groupBySupported =
!isRestricted && !RESTRICTED_GROUP_BY_FIELDS.includes(fieldKey);
const isRestricted = RESTRICTED_SELECTED_FIELDS.includes(fieldKey);
return {
fieldKey,
filterInOperator: OPERATORS['='],
filterOutOperator: OPERATORS['!='],
dataType: getDataTypes(value),
metricsType: metricsTypeForRoot(root),
groupBySupported,
groupByKey: groupBySupported ? fieldKey : undefined,
groupBySupported: !isRestricted,
groupByKey: isRestricted ? undefined : fieldKey,
isRestricted,
};
}

View File

@@ -3,79 +3,45 @@ import { DataTypes } from 'types/api/queryBuilder/queryAutocompleteResponse';
import {
aggregateAttributesResourcesToObject,
buildPrettyViewData,
flattenObject,
getDataTypes,
getSanitizedLogBody,
parseJsonStringValue,
parseJsonStringBody,
recursiveParseJSON,
} from './utils';
describe('parseJsonStringValue', () => {
describe('parseJsonStringBody', () => {
it('parses a JSON-object string into an object', () => {
expect(parseJsonStringValue('{"a":1,"b":{"c":2}}')).toStrictEqual({
expect(parseJsonStringBody('{"a":1,"b":{"c":2}}')).toStrictEqual({
a: 1,
b: { c: 2 },
});
});
it('parses a JSON-array string into an array', () => {
expect(parseJsonStringValue('[1,2,3]')).toStrictEqual([1, 2, 3]);
expect(parseJsonStringBody('[1,2,3]')).toStrictEqual([1, 2, 3]);
});
it('returns a plain (non-JSON) string unchanged', () => {
expect(parseJsonStringValue('plain log line')).toBe('plain log line');
expect(parseJsonStringBody('plain log line')).toBe('plain log line');
});
it('returns a string that is not object/array-looking unchanged', () => {
expect(parseJsonStringValue('42')).toBe('42');
expect(parseJsonStringBody('42')).toBe('42');
});
it('returns an invalid JSON string unchanged', () => {
expect(parseJsonStringValue('{not valid}')).toBe('{not valid}');
expect(parseJsonStringBody('{not valid}')).toBe('{not valid}');
});
it('returns an already-object value unchanged (same reference)', () => {
const value = { message: 'hi', a: 1 };
expect(parseJsonStringValue(value)).toBe(value);
it('returns an already-object body unchanged (same reference)', () => {
const body = { message: 'hi', a: 1 };
expect(parseJsonStringBody(body)).toBe(body);
});
it('leaves a value larger than the 128KB parse guard as a string', () => {
it('leaves a body larger than the 128KB parse guard as a string', () => {
const huge = `{"x":"${'a'.repeat(130 * 1024)}"}`;
expect(parseJsonStringValue(huge)).toBe(huge);
});
});
describe('buildPrettyViewData', () => {
const baseRaw = {
id: 'log-1',
timestamp: 1234,
body: 'hello',
attributes: {},
resource: {},
scope: {},
} as any;
it('parses a JSON-string body into a tree', () => {
const result = buildPrettyViewData({ ...baseRaw, body: '{"a":1}' });
expect(result.body).toStrictEqual({ a: 1 });
});
it('parses attribute values that are JSON strings, leaves others as-is', () => {
const result = buildPrettyViewData({
...baseRaw,
attributes: { payload: '{"x":1}', name: 'cart', count: 3 },
});
expect(result.attributes).toStrictEqual({
payload: { x: 1 },
name: 'cart',
count: 3,
});
});
it('drops undefined fields so they do not render as empty rows', () => {
const result = buildPrettyViewData({ ...baseRaw, trace_id: undefined });
expect('trace_id' in result).toBe(false);
expect(parseJsonStringBody(huge)).toBe(huge);
});
});
@@ -99,7 +65,7 @@ describe('aggregateAttributesResourcesToObject', () => {
'http.method': 'GET',
retries: 3,
});
expect(result.resource).toStrictEqual({ 'service.name': 'cart' });
expect(result.resources).toStrictEqual({ 'service.name': 'cart' });
expect(result.scope).toStrictEqual({ lib: 'otel' });
expect(result.body).toBe('hello');
expect(result.id).toBe('log-1');

View File

@@ -276,7 +276,7 @@ export const aggregateAttributesResourcesToObject = (
traceFlags: logData.traceFlags,
traceId: logData.traceId,
attributes: {},
resource: {},
resources: {},
scope: {},
severity_text: logData.severity_text,
severity_number: logData.severity_number,
@@ -290,8 +290,8 @@ export const aggregateAttributesResourcesToObject = (
outputJson.attributes = outputJson.attributes || {};
Object.assign(outputJson.attributes, logData[key as keyof ILog]);
} else if (key.startsWith('resources_')) {
outputJson.resource = outputJson.resource || {};
Object.assign(outputJson.resource, logData[key as keyof ILog]);
outputJson.resources = outputJson.resources || {};
Object.assign(outputJson.resources, logData[key as keyof ILog]);
} else if (key.startsWith('scope_string')) {
outputJson.scope = outputJson.scope || {};
Object.assign(outputJson.scope, logData[key as keyof ILog]);
@@ -315,57 +315,30 @@ export const aggregateAttributesResourcesToString = (logData: ILog): string => {
}
};
const MAX_JSON_PARSE_BYTES = 128 * 1024;
const MAX_JSON_BODY_PARSE_BYTES = 128 * 1024;
// A JSON-encoded object/array string is parsed so DataViewer renders it as a tree
// instead of one escaped string; non-JSON / plain-text values are returned unchanged.
// A JSON-encoded object/array `body` is parsed so DataViewer renders it as a
// tree instead of one escaped string; plain-text bodies are returned unchanged.
// Guarded against very large payloads.
export const parseJsonStringValue = (value: unknown): unknown => {
if (typeof value !== 'string') {
return value;
export const parseJsonStringBody = (body: ILog['body']): ILog['body'] => {
if (typeof body !== 'string') {
return body;
}
const trimmed = value.trim();
const trimmed = body.trim();
const looksLikeJson = trimmed.startsWith('{') || trimmed.startsWith('[');
if (!looksLikeJson || trimmed.length > MAX_JSON_PARSE_BYTES) {
return value;
if (!looksLikeJson || trimmed.length > MAX_JSON_BODY_PARSE_BYTES) {
return body;
}
try {
const parsed = JSON.parse(trimmed);
return parsed !== null && typeof parsed === 'object' ? parsed : value;
return parsed !== null && typeof parsed === 'object'
? (parsed as ILogBody)
: body;
} catch {
return value;
return body;
}
};
// Parse each attribute value that's a stringified JSON string into an object
// Non-JSON values are left unchanged.
const parseAttributeJsonValues = (
attributes: Record<string, unknown>,
): Record<string, unknown> => {
const parsed: Record<string, unknown> = {};
Object.keys(attributes).forEach((key) => {
parsed[key] = parseJsonStringValue(attributes[key]);
});
return parsed;
};
export const buildPrettyViewData = (
raw: ILogAggregateAttributesResources,
): Record<string, unknown> => {
const prettyData: Record<string, unknown> = { ...raw };
prettyData.body = parseJsonStringValue(raw.body);
prettyData.attributes = parseAttributeJsonValues(raw.attributes);
// drop undefined fields so they don't render as empty rows
Object.keys(prettyData).forEach((key) => {
if (prettyData[key] === undefined) {
delete prettyData[key];
}
});
return prettyData;
};
const isFloat = (num: number): boolean => num % 1 !== 0;
const isBooleanString = (str: string): boolean =>

View File

@@ -2,9 +2,6 @@ import { blue, red } from '@ant-design/colors';
export const RESTRICTED_SELECTED_FIELDS = ['timestamp', 'id'];
// Fields that can be filtered on but not grouped by in the log details view.
export const RESTRICTED_GROUP_BY_FIELDS = ['body', 'trace_id'];
export const ICON_STYLE = {
PLUS: { color: blue[5] },
CLOSE: { color: red[5] },

View File

@@ -2,17 +2,14 @@ import { useTranslation } from 'react-i18next';
import { Input } from '@signozhq/ui/input';
import { Switch } from '@signozhq/ui/switch';
import { Form, Select, Space } from 'antd';
import { FeatureKeys } from 'constants/features';
import { ModalFooterTitle } from 'container/PipelinePage/styles';
import { useAppContext } from 'providers/App/App';
import { ProcessorData } from 'types/api/pipeline/def';
import { formValidationRules } from '../config';
import { ProcessorFormField } from './config';
import { processorFields, ProcessorFormField } from './config';
import CSVInput from './FormFields/CSVInput';
import JsonFlattening from './FormFields/JsonFlattening';
import { FormWrapper, PipelineIndexIcon, StyledSelect } from './styles';
import { resolveProcessorFields } from './utils';
import './styles.scss';
@@ -136,23 +133,16 @@ function ProcessorForm({
selectedProcessorData,
isAdd,
}: ProcessorFormProps): JSX.Element {
const { featureFlags } = useAppContext();
const isBodyJsonEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.USE_JSON_BODY)
?.active || false;
return (
<div className="processor-form-container">
{resolveProcessorFields(processorType, isBodyJsonEnabled).map(
(fieldData: ProcessorFormField) => (
<ProcessorFieldInput
key={fieldData.name + String(fieldData.initialValue)}
fieldData={fieldData}
selectedProcessorData={selectedProcessorData}
isAdd={isAdd}
/>
),
)}
{processorFields[processorType]?.map((fieldData: ProcessorFormField) => (
<ProcessorFieldInput
key={fieldData.name + String(fieldData.initialValue)}
fieldData={fieldData}
selectedProcessorData={selectedProcessorData}
isAdd={isAdd}
/>
))}
</div>
);
}

View File

@@ -1,24 +0,0 @@
import { processorFields, ProcessorFormField } from './config';
const BODY_PARSE_FROM = 'body';
const JSON_BODY_PARSE_FROM = 'body.message';
// With use_json_body the collector normalizes every body into a map before user
// operators run, so a parser pointed at `body` gets a map it cannot read and
// silently extracts nothing. The log text lives at body.message.
export function resolveProcessorFields(
processorType: string,
isBodyJsonEnabled: boolean,
): Array<ProcessorFormField> {
const fields = processorFields[processorType] ?? [];
if (!isBodyJsonEnabled) {
return fields;
}
return fields.map((field) =>
field.name === 'parse_from' && field.initialValue === BODY_PARSE_FROM
? { ...field, initialValue: JSON_BODY_PARSE_FROM }
: field,
);
}

View File

@@ -1,45 +0,0 @@
import { processorFields } from '../PipelineListsView/AddNewProcessor/config';
import { resolveProcessorFields } from '../PipelineListsView/AddNewProcessor/utils';
const parseFromDefault = (
fields: ReturnType<typeof resolveProcessorFields>,
): unknown => fields.find((field) => field.name === 'parse_from')?.initialValue;
describe('resolveProcessorFields', () => {
it.each(['grok_parser', 'regex_parser', 'json_parser'])(
'defaults %s parse_from to body.message when use_json_body is on',
(processorType) => {
expect(parseFromDefault(resolveProcessorFields(processorType, true))).toBe(
'body.message',
);
},
);
it.each(['grok_parser', 'regex_parser', 'json_parser'])(
'keeps %s parse_from as body when use_json_body is off',
(processorType) => {
expect(parseFromDefault(resolveProcessorFields(processorType, false))).toBe(
'body',
);
},
);
it('leaves parse_from defaults that do not point at the body alone', () => {
expect(parseFromDefault(resolveProcessorFields('time_parser', true))).toBe(
'attributes.timestamp',
);
expect(
parseFromDefault(resolveProcessorFields('severity_parser', true)),
).toBe('attributes.logLevel');
});
it('does not mutate the shared config', () => {
resolveProcessorFields('grok_parser', true);
expect(parseFromDefault(processorFields.grok_parser)).toBe('body');
});
it('returns an empty list for an unknown processor type', () => {
expect(resolveProcessorFields('does_not_exist', true)).toStrictEqual([]);
});
});

View File

@@ -41,6 +41,6 @@ type OmitAttributesResources = Pick<
export type ILogAggregateAttributesResources = OmitAttributesResources & {
attributes: Record<string, never>;
resource: Record<string, never>;
resources: Record<string, never>;
scope: Record<string, never>;
};

View File

@@ -51,6 +51,28 @@
},
"name": "Region"
}
},
{
"kind": "ListVariable",
"spec": {
"display": {
"name": "FunctionName",
"description": "Name of the Lambda function"
},
"allowAllValue": true,
"allowMultiple": true,
"customAllValue": "",
"capturingRegexp": "",
"sort": "none",
"plugin": {
"kind": "signoz/DynamicVariable",
"spec": {
"name": "FunctionName",
"signal": "metrics"
}
},
"name": "FunctionName"
}
}
],
"panels": {
@@ -118,7 +140,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -218,7 +240,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -318,7 +340,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -418,7 +440,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -518,7 +540,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -618,7 +640,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -718,7 +740,7 @@
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS)"
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
@@ -831,4 +853,4 @@
"refreshInterval": "",
"links": []
}
}
}

View File

@@ -29,7 +29,6 @@ type builderQuery[T any] struct {
telemetryStore telemetrystore.TelemetryStore
orgID valuer.UUID
stmtBuilder qbtypes.StatementBuilder[T]
queryType qbtypes.QueryType
spec qbtypes.QueryBuilderQuery[T]
variables map[string]qbtypes.VariableItem
@@ -52,7 +51,6 @@ func newBuilderQuery[T any](
telemetryStore telemetrystore.TelemetryStore,
orgID valuer.UUID,
stmtBuilder qbtypes.StatementBuilder[T],
queryType qbtypes.QueryType,
spec qbtypes.QueryBuilderQuery[T],
tr qbtypes.TimeRange,
kind qbtypes.RequestType,
@@ -64,7 +62,6 @@ func newBuilderQuery[T any](
telemetryStore: telemetryStore,
orgID: orgID,
stmtBuilder: stmtBuilder,
queryType: queryType,
spec: spec,
variables: variables,
fromMS: tr.From,
@@ -84,7 +81,7 @@ func (q *builderQuery[T]) Fingerprint() string {
// Create a deterministic fingerprint for builder queries
// This needs to include all fields that affect the query results
parts := []string{q.queryType.StringValue()}
parts := []string{"builder"}
// Add signal type
parts = append(parts, fmt.Sprintf("signal=%s", q.spec.Signal.StringValue()))

View File

@@ -3,7 +3,6 @@ package querier
import (
"strings"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -21,8 +20,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "fingerprint includes shiftby when ShiftBy field is set",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 3600,
@@ -42,8 +40,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "fingerprint includes shiftby but not other functions",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 3600,
@@ -66,8 +63,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "no shiftby in fingerprint when ShiftBy is zero",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 0,
@@ -98,29 +94,6 @@ func TestBuilderQueryFingerprint(t *testing.T) {
}
}
func TestBuilderQueryFingerprintQueryType(t *testing.T) {
spec := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "count()"}},
Filter: &qbtypes.Filter{Expression: "gen_ai.request.model EXISTS"},
}
regular := &builderQuery[qbtypes.TraceAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
spec: spec,
}
ai := &builderQuery[qbtypes.TraceAggregation]{
queryType: qbtypes.QueryTypeBuilderAI,
kind: qbtypes.RequestTypeTimeSeries,
spec: spec,
}
assert.True(t, strings.HasPrefix(regular.Fingerprint(), qbtypes.QueryTypeBuilder.StringValue()+"&"))
assert.True(t, strings.HasPrefix(ai.Fingerprint(), qbtypes.QueryTypeBuilderAI.StringValue()+"&"))
assert.NotEqual(t, regular.Fingerprint(), ai.Fingerprint())
}
func TestMakeBucketsOrder(t *testing.T) {
// Test that makeBuckets returns buckets in reverse chronological order by default
// Using milliseconds as input - need > 1 hour range to get multiple buckets

View File

@@ -305,7 +305,7 @@ func (q *querier) buildQueries(
}
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryTypeBuilder:
@@ -313,7 +313,7 @@ func (q *querier) buildQueries(
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
@@ -323,7 +323,7 @@ func (q *querier) buildQueries(
if spec.Source == telemetrytypes.SourceAudit {
stmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
@@ -340,9 +340,9 @@ func (q *querier) buildQueries(
if spec.Source == telemetrytypes.SourceMeter {
event.Source = telemetrytypes.SourceMeter.StringValue()
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
} else {
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
}
queries[spec.Name] = bq
@@ -618,7 +618,7 @@ func (q *querier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qb
if spec.Source == telemetrytypes.SourceAudit {
liveTailStmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, liveTailStmtBuilder, query.Type, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, liveTailStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
"id": {
Value: updatedLogID,
},
@@ -941,9 +941,8 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
// reuse the original query's statement builder and type so an AI query
// keeps its AI builder and cache key
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, qt.builderConfig)
// reuse the original query's statement builder so an AI query keeps its AI builder
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
case *builderQuery[qbtypes.LogAggregation]:
specCopy := qt.spec.Copy()
@@ -953,16 +952,16 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
if qt.spec.Source == telemetrytypes.SourceAudit {
shiftStmtBuilder = q.auditStmtBuilder
}
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, q.builderConfig)
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, q.builderConfig)
case *builderQuery[qbtypes.MetricAggregation]:
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
if qt.spec.Source == telemetrytypes.SourceMeter {
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.meterStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.meterStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
}
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.metricStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.metricStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
case *traceOperatorQuery:
specCopy := qt.spec.Copy()
return &traceOperatorQuery{

View File

@@ -56,17 +56,6 @@ func QueryStringToKeysSelectors(query string) []*telemetrytypes.FieldKeySelector
FieldDataType: key.FieldDataType,
})
}
// todo(tushar): consider reverting changes done to this method in below PR to avoid scope specific checks
// https://github.com/SigNoz/signoz/issues/11374
if key.FieldContext == telemetrytypes.FieldContextScope {
keys = append(keys, &telemetrytypes.FieldKeySelector{
Name: key.FieldContext.StringValue() + "." + key.Name,
Signal: key.Signal,
FieldContext: telemetrytypes.FieldContextUnspecified, // this allows 'scope.' prefix for keys with other context as well
FieldDataType: key.FieldDataType,
})
}
}
}

View File

@@ -72,23 +72,6 @@ func TestQueryToKeys(t *testing.T) {
},
},
},
{
query: `scope.version = '1.0.0'`,
expectedKeys: []telemetrytypes.FieldKeySelector{
{
Name: "version",
Signal: telemetrytypes.SignalUnspecified,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
{
Name: "scope.version",
Signal: telemetrytypes.SignalUnspecified,
FieldContext: telemetrytypes.FieldContextUnspecified,
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
},
},
},
}
for _, testCase := range testCases {

View File

@@ -242,6 +242,7 @@ func NewSQLMigrationProviderFactories(
sqlmigration.NewRestructureAuthDomainConfigFactory(sqlstore),
sqlmigration.NewFixSavedViewSelectFieldsFactory(sqlstore),
sqlmigration.NewDeleteOrphanUserRolesFactory(),
sqlmigration.NewMigrateLambdaDashboardsFactory(),
)
}

View File

@@ -0,0 +1,160 @@
package sqlmigration
import (
"bytes"
"context"
"embed"
"encoding/json"
"time"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
)
//go:embed 116_migrate_lambda_dashboards
var lambdaDashboardFiles embed.FS
// These values mirror the cloud integration and dashboard packages but are duplicated
// here so this migration keeps targeting and writing the same rows even if those
// constants are later renamed or changed.
const (
lambdaDashboardFile = "116_migrate_lambda_dashboards/aws/lambda/overview.json"
lambdaDashboardSlug = "aws-lambda-overview"
cloudIntegrationDashboardProvider = "cloud_integration"
integrationDashboardSource = "integration"
dashboardSchemaVersion = "v6"
)
type migrateLambdaDashboards struct{}
type lambdaDashboardRow struct {
bun.BaseModel `bun:"table:dashboard,alias:dashboard"`
ID string `bun:"id"`
Data string `bun:"data"`
}
// lambdaDashboardDefinition is the part of the embedded dashboard this migration reads:
// its spec, which is what the cloud integration stores under data.spec.
type lambdaDashboardDefinition struct {
Spec map[string]any `json:"spec"`
}
func NewMigrateLambdaDashboardsFactory() factory.ProviderFactory[SQLMigration, Config] {
return factory.NewProviderFactory(
factory.MustNewName("migrate_lambda_dashboards"),
func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
return &migrateLambdaDashboards{}, nil
},
)
}
func (m *migrateLambdaDashboards) Register(migrations *migrate.Migrations) error {
return migrations.Register(m.Up, m.Down)
}
// Up rewrites the spec of every provisioned AWS Lambda overview dashboard to the
// embedded revision that added the FunctionName variable. Cloud integration dashboards
// are provisioned once and never updated afterwards, so existing installs only pick up
// this change through a migration. Only the spec is replaced; the row keeps its id, name,
// tags and metadata, so the dashboard is updated in place rather than recreated.
func (m *migrateLambdaDashboards) Up(ctx context.Context, db *bun.DB) error {
spec, err := m.loadSpec()
if err != nil {
return err
}
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
var rows []*lambdaDashboardRow
if err := tx.NewSelect().
Model(&rows).
Join("JOIN integration_dashboard AS id ON id.dashboard_id = dashboard.id").
Where("id.provider = ?", cloudIntegrationDashboardProvider).
Where("id.slug = ?", lambdaDashboardSlug).
Where("dashboard.source = ?", integrationDashboardSource).
Scan(ctx); err != nil {
return err
}
for _, row := range rows {
data := map[string]any{}
if err := json.Unmarshal([]byte(row.Data), &data); err != nil {
return err
}
// The embedded spec is v6-shaped, so only rewrite a row already carrying a v6 spec;
// anything else is left alone rather than turned into a broken mix of versions.
if !m.hasV6Spec(data) {
continue
}
data["spec"] = spec
encoded, err := m.marshalUnescaped(data)
if err != nil {
return err
}
// Skip rows already carrying this spec so a re-run does not needlessly rewrite them.
if string(encoded) == row.Data {
continue
}
if _, err := tx.NewUpdate().
Model((*lambdaDashboardRow)(nil)).
Set("data = ?", string(encoded)).
Set("updated_at = ?", time.Now()).
Where("id = ?", row.ID).
Exec(ctx); err != nil {
return err
}
}
return tx.Commit()
}
func (m *migrateLambdaDashboards) Down(context.Context, *bun.DB) error {
return nil
}
// hasV6Spec reports whether the stored data is a v6 dashboard with a spec object, which
// is the shape whose spec this migration replaces.
func (m *migrateLambdaDashboards) hasV6Spec(data map[string]any) bool {
metadata, _ := data["metadata"].(map[string]any)
version, _ := metadata["schemaVersion"].(string)
if version != dashboardSchemaVersion {
return false
}
_, ok := data["spec"].(map[string]any)
return ok
}
func (m *migrateLambdaDashboards) marshalUnescaped(v any) ([]byte, error) {
var buf bytes.Buffer
encoder := json.NewEncoder(&buf)
encoder.SetEscapeHTML(false)
if err := encoder.Encode(v); err != nil {
return nil, err
}
return bytes.TrimRight(buf.Bytes(), "\n"), nil
}
func (m *migrateLambdaDashboards) loadSpec() (map[string]any, error) {
raw, err := lambdaDashboardFiles.ReadFile(lambdaDashboardFile)
if err != nil {
return nil, err
}
var dashboard lambdaDashboardDefinition
if err := json.Unmarshal(raw, &dashboard); err != nil {
return nil, err
}
return dashboard.Spec, nil
}

View File

@@ -0,0 +1,856 @@
{
"schemaVersion": "v6",
"image": "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iODAwcHgiIGhlaWdodD0iODAwcHgiIHZpZXdCb3g9IjAgMCAxNiAxNiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBmaWxsPSJub25lIj48cGF0aCBmaWxsPSIjRkE3RTE0IiBkPSJNNy45ODMgOC4zN2MtLjA1My4wNzMtLjA5OC4xMzMtLjE0MS4xOTRMNS43NzUgMTEuNWMtLjY0LjkxLTEuMjgyIDEuODItMS45MjQgMi43M2EuMTI4LjEyOCAwIDAxLS4wOTIuMDUxYy0uOTA2LS4wMDctMS44MTMtLjAxNy0yLjcxOS0uMDI4LS4wMSAwLS4wMi0uMDAzLS4wNC0uMDA2YS40NTUuNDU1IDAgMDEuMDI1LS4wNTMgMTM5NzcuNDk2IDEzOTc3LjQ5NiAwIDAxNS40NDYtOC4xNDZjLjA5Mi0uMTM4LjE4OC0uMjczLjI3NS0uNDEzYS4xNjUuMTY1IDAgMDAuMDE4LS4xMjRjLS4xNjctLjUxNS0uMzM4LTEuMDMtLjUwOC0xLjU0My0uMDczLS4yMi0uMTUtLjQ0LS4yMTgtLjY2LS4wMjItLjA3Mi0uMDU5LS4wOTQtLjEzNC0uMDkzLS41Ny4wMDItMS4xMzYuMDAxLTEuNzA0LjAwMS0uMTA4IDAtLjEwOCAwLS4xMDgtLjEwMyAwLS42NzQgMC0xLjM0Ny0uMDAyLTIuMDIxIDAtLjA3NS4wMjYtLjA5Mi4wOTktLjA5MiAxLjE0My4wMDIgMi4yODYuMDAyIDMuNDMgMGEuMTEzLjExMyAwIDAxLjA3Ni4wMTcuMTA3LjEwNyAwIDAxLjA0NS4wNjEgMTgyNjYuMTg0IDE4MjY2LjE4NCAwIDAwMy45MiA5LjUxYy4yMTguNTMuNDM4IDEuMDU5LjY1NCAxLjU5LjAyNi4wNjQuMDUzLjA3Ni4xMi4wNTYuNi0uMTc4IDEuMi0uMzUyIDEuOC0uNTMxLjA3NS0uMDIzLjEwMi0uMDA4LjEyNi4wNjQuMjA0LjYyLjQxMiAxLjIzOS42MiAxLjg1OGwuMDIuMDczYy0uMDQzLjAxNS0uMDgzLjAzMi0uMTI0LjA0M2wtNC4wODUgMS4yNWMtLjA2NS4wMi0uMDg1IDAtLjEwNi0uMDU0bC0xLjI1LTMuMDQ4LTEuMjI2LTIuOTg0LS4xODMtLjQ0OWMtLjAxLS4wMjYtLjAyMy0uMDQ4LS4wNDMtLjA4N3oiLz48L3N2Zz4=",
"name": "",
"generateName": true,
"tags": [],
"spec": {
"display": {
"name": "AWS Lambda Overview",
"description": "Overview of AWS Lambda functions"
},
"variables": [
{
"kind": "ListVariable",
"spec": {
"display": {
"name": "Account",
"description": "AWS Account"
},
"allowAllValue": false,
"allowMultiple": false,
"customAllValue": "",
"capturingRegexp": "",
"sort": "none",
"plugin": {
"kind": "signoz/QueryVariable",
"spec": {
"queryValue": "SELECT JSONExtractString(labels, 'cloud.account.id') as `cloud.account.id`\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE \n metric_name like 'aws_Lambda_Invocations_sum'\nGROUP BY `cloud.account.id`\n\n"
}
},
"name": "Account"
}
},
{
"kind": "ListVariable",
"spec": {
"display": {
"name": "Region",
"description": "AWS Region"
},
"allowAllValue": false,
"allowMultiple": false,
"customAllValue": "",
"capturingRegexp": "",
"sort": "none",
"plugin": {
"kind": "signoz/QueryVariable",
"spec": {
"queryValue": "SELECT JSONExtractString(labels, 'cloud.region') as `cloud.region`\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE \n metric_name like 'aws_Lambda_Invocations_sum'\n and JSONExtractString(labels, 'cloud.account.id') IN {{.Account}}\nGROUP BY `cloud.region`\n"
}
},
"name": "Region"
}
},
{
"kind": "ListVariable",
"spec": {
"display": {
"name": "FunctionName",
"description": "Name of the Lambda function"
},
"allowAllValue": true,
"allowMultiple": true,
"customAllValue": "",
"capturingRegexp": "",
"sort": "none",
"plugin": {
"kind": "signoz/DynamicVariable",
"spec": {
"name": "FunctionName",
"signal": "metrics"
}
},
"name": "FunctionName"
}
}
],
"panels": {
"2516c785-b025-49b3-aeb4-a4735ccb2709": {
"kind": "Panel",
"spec": {
"display": {
"name": "Errors",
"description": "The number of invocations that result in a function error. Function errors include exceptions that your code throws and exceptions that the Lambda runtime throws. The runtime returns errors for issues such as timeouts and configuration errors. To calculate the error rate, divide the value of Errors by the value of Invocations. Note that the timestamp on an error metric reflects when the function was invoked, not when the error occurred.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "none",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_Errors_sum",
"temporality": "",
"timeAggregation": "sum",
"spaceAggregation": "sum",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
},
"4119a1e5-32a8-4859-96e9-a5451114782b": {
"kind": "Panel",
"spec": {
"display": {
"name": "Async events dropped",
"description": "The number of events that are dropped without successfully executing the function. If you configure a dead-letter queue (DLQ) or OnFailure destination, then events are sent there before they're dropped. Events are dropped for various reasons. For example, events can exceed the maximum event age or exhaust the maximum retry attempts, or reserved concurrency might be set to 0. To troubleshoot why events are dropped, look at the Errors metric to identify function errors and the Throttles metric to identify concurrency issues.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "none",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_AsyncEventsDropped_sum",
"temporality": "",
"timeAggregation": "sum",
"spaceAggregation": "sum",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
},
"6354ea62-e82b-4323-a33d-eef92519e843": {
"kind": "Panel",
"spec": {
"display": {
"name": "Throttles",
"description": "The number of invocation requests that are throttled. When all function instances are processing requests and no concurrency is available to scale up, Lambda rejects additional requests with a TooManyRequestsException error. Throttled requests and other invocation errors don't count as either Invocations or Errors.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "none",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_Throttles_sum",
"temporality": "",
"timeAggregation": "sum",
"spaceAggregation": "sum",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
},
"853d3a92-b396-4064-8762-18d7487989e0": {
"kind": "Panel",
"spec": {
"display": {
"name": "Async events received",
"description": "The number of events that Lambda successfully queues for processing. This metric provides insight into the number of events that a Lambda function receives. Monitor this metric and set alarms for thresholds to check for issues. For example, to detect an undesirable number of events sent to Lambda, and to quickly diagnose issues resulting from incorrect trigger or function configurations. Mismatches between AsyncEventsReceived and Invocations can indicate a disparity in processing, events being dropped, or a potential queue backlog.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "none",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_AsyncEventsReceived_sum",
"temporality": "",
"timeAggregation": "sum",
"spaceAggregation": "sum",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
},
"877bb5c8-331c-492f-b666-2054c2ae39bd": {
"kind": "Panel",
"spec": {
"display": {
"name": "Invocations",
"description": "The number of times that your function code is invoked, including successful invocations and invocations that result in a function error. Invocations aren't recorded if the invocation request is throttled or otherwise results in an invocation error. The value of Invocations equals the number of requests billed.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "none",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_Invocations_sum",
"temporality": "",
"timeAggregation": "sum",
"spaceAggregation": "sum",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
},
"ae6d7c81-d921-4d4c-95ec-6b42d900ea45": {
"kind": "Panel",
"spec": {
"display": {
"name": "Max Async Event Age",
"description": "The time between when Lambda successfully queues the event and when the function is invoked. The value of this metric increases when events are being retried due to invocation failures or throttling. Monitor this metric and set alarms for thresholds on different statistics for when a queue buildup occurs. To troubleshoot an increase in this metric, look at the Errors metric to identify function errors and the Throttles metric to identify concurrency issues.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "ms",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_AsyncEventAge_max",
"temporality": "",
"timeAggregation": "max",
"spaceAggregation": "max",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
},
"b038520d-0756-4e46-a915-12a2f19a0254": {
"kind": "Panel",
"spec": {
"display": {
"name": "Max Duration",
"description": "The amount of time that your function code spends processing an event. The billed duration for an invocation is the value of Duration rounded up to the nearest millisecond. Duration does not include cold start time.\n\nSee more at https://docs.aws.amazon.com/lambda/latest/dg/monitoring-metrics-types.html"
},
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {
"visualization": {
"timePreference": "global_time",
"fillSpans": false
},
"formatting": {
"unit": "ms",
"decimalPrecision": "2"
},
"chartAppearance": {
"lineInterpolation": "spline",
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {
"fillOnlyBelow": false,
"fillLessThan": ""
}
},
"axes": {
"softMin": 0,
"softMax": 0,
"isLogScale": false
},
"legend": {
"position": "bottom",
"mode": "list",
"customColors": null
},
"thresholds": null
}
},
"queries": [
{
"kind": "time_series",
"spec": {
"name": "A",
"plugin": {
"kind": "signoz/BuilderQuery",
"spec": {
"name": "A",
"stepInterval": 60,
"signal": "metrics",
"source": "",
"aggregations": [
{
"metricName": "aws_Lambda_Duration_max",
"temporality": "",
"timeAggregation": "max",
"spaceAggregation": "max",
"reduceTo": "avg"
}
],
"disabled": false,
"filter": {
"expression": "(cloud.account.id = $Account AND cloud.region = $Region AND FunctionName EXISTS AND Resource NOT EXISTS) AND FunctionName IN $FunctionName"
},
"groupBy": [
{
"name": "cloud.account.id",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "cloud.region",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
},
{
"name": "FunctionName",
"signal": "",
"fieldContext": "attribute",
"fieldDataType": "string"
}
],
"order": [],
"having": {
"expression": ""
},
"functions": [],
"legend": "{{FunctionName}}"
}
}
}
}
],
"links": []
}
}
},
"layouts": [
{
"kind": "Grid",
"spec": {
"items": [
{
"x": 0,
"y": 0,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/877bb5c8-331c-492f-b666-2054c2ae39bd"
}
},
{
"x": 6,
"y": 0,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/b038520d-0756-4e46-a915-12a2f19a0254"
}
},
{
"x": 0,
"y": 6,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/2516c785-b025-49b3-aeb4-a4735ccb2709"
}
},
{
"x": 6,
"y": 6,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/6354ea62-e82b-4323-a33d-eef92519e843"
}
},
{
"x": 0,
"y": 12,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/853d3a92-b396-4064-8762-18d7487989e0"
}
},
{
"x": 6,
"y": 12,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/ae6d7c81-d921-4d4c-95ec-6b42d900ea45"
}
},
{
"x": 0,
"y": 18,
"width": 6,
"height": 6,
"content": {
"$ref": "#/spec/panels/4119a1e5-32a8-4859-96e9-a5451114782b"
}
}
]
}
}
],
"duration": "",
"refreshInterval": "",
"links": []
}
}

View File

@@ -271,25 +271,18 @@ func adjustTraceKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*te
*/
var isIntrinsicOrCalculatedField bool
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
// A scope-context key addresses the scope JSON column and must not bind to a non-scope
// intrinsic/calculated field that only shares its name (e.g. `{name, scope}` is the scope's
// name, not the span `name` column). The span<->attribute remapping of legacy fields is
// intentionally context-blind and left untouched.
boundToScopeMismatch := func(f telemetrytypes.TelemetryFieldKey) bool {
return key.FieldContext == telemetrytypes.FieldContextScope && f.FieldContext != telemetrytypes.FieldContextScope
}
if f, ok := tracestelemetryschema.IntrinsicFields[key.Name]; ok && !boundToScopeMismatch(f) {
if _, ok := tracestelemetryschema.IntrinsicFields[key.Name]; ok {
isIntrinsicOrCalculatedField = true
intrinsicOrCalculatedField = f
} else if f, ok := tracestelemetryschema.CalculatedFields[key.Name]; ok && !boundToScopeMismatch(f) {
intrinsicOrCalculatedField = tracestelemetryschema.IntrinsicFields[key.Name]
} else if _, ok := tracestelemetryschema.CalculatedFields[key.Name]; ok {
isIntrinsicOrCalculatedField = true
intrinsicOrCalculatedField = f
} else if f, ok := tracestelemetryschema.IntrinsicFieldsDeprecated[key.Name]; ok && !boundToScopeMismatch(f) {
intrinsicOrCalculatedField = tracestelemetryschema.CalculatedFields[key.Name]
} else if _, ok := tracestelemetryschema.IntrinsicFieldsDeprecated[key.Name]; ok {
isIntrinsicOrCalculatedField = true
intrinsicOrCalculatedField = f
} else if f, ok := tracestelemetryschema.CalculatedFieldsDeprecated[key.Name]; ok && !boundToScopeMismatch(f) {
intrinsicOrCalculatedField = tracestelemetryschema.IntrinsicFieldsDeprecated[key.Name]
} else if _, ok := tracestelemetryschema.CalculatedFieldsDeprecated[key.Name]; ok {
isIntrinsicOrCalculatedField = true
intrinsicOrCalculatedField = f
intrinsicOrCalculatedField = tracestelemetryschema.CalculatedFieldsDeprecated[key.Name]
}
if isIntrinsicOrCalculatedField {

View File

@@ -369,99 +369,11 @@ func TestStatementBuilder(t *testing.T) {
},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> 0, toFloat64(duration_nano), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> 0, toFloat64(duration_nano), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_responseStatusCode`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_responseStatusCode` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_responseStatusCode`",
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), __limit_cte AS (SELECT toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> 0, accurateCastOrNull(duration_nano, 'Float64'), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_responseStatusCode` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(response_status_code <> '', response_status_code, NULL)) AS `__GROUP_BY_KEY_0_responseStatusCode`, quantile(0.90)(multiIf(duration_nano <> 0, accurateCastOrNull(duration_nano, 'Float64'), NULL)) AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_responseStatusCode`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_responseStatusCode` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_responseStatusCode`",
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.name filter and group by",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.name = 'opentelemetry-io'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(scope.name::String <> '', scope.name::String, NULL)) AS `__GROUP_BY_KEY_0_scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.name::String = ? AND scope.name::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_scope.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(scope.name::String <> '', scope.name::String, NULL)) AS `__GROUP_BY_KEY_0_scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.name::String = ? AND scope.name::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_scope.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_scope.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_scope.name`",
Args: []any{"opentelemetry-io", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "opentelemetry-io", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.version filter with scope.name group by",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(scope.name::String <> '', scope.name::String, NULL)) AS `__GROUP_BY_KEY_0_scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_scope.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(scope.name::String <> '', scope.name::String, NULL)) AS `__GROUP_BY_KEY_0_scope.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_scope.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_scope.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_scope.name`",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
expectedErr: nil,
},
{
name: "scope.version filter only (no scope field in group by)",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Aggregations: []qbtypes.TraceAggregation{
{
Expression: "count()",
},
},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
GroupBy: []qbtypes.GroupByKey{
{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
},
},
},
},
expected: qbtypes.Statement{
Query: "WITH __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? GROUP BY `__GROUP_BY_KEY_0_service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(timestamp, INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `__GROUP_BY_KEY_0_service.name`, count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND (`__GROUP_BY_KEY_0_service.name`) GLOBAL IN (SELECT `__GROUP_BY_KEY_0_service.name` FROM __limit_cte) GROUP BY ts, `__GROUP_BY_KEY_0_service.name`",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10, "1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
},
},
}
fl := flaggertest.New(t)
@@ -758,7 +670,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
Limit: 10,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, resource_string_service$$name AS `__SELECT_KEY_4_serviceName`, duration_nano AS `__SELECT_KEY_5_durationNano`, http_method AS `__SELECT_KEY_6_httpMethod`, multiIf(mapContains(attributes_string, 'non-existent.key'), toString(attributes_string['non-existent.key']), mapContains(attributes_number, 'non-existent.key'), toString(attributes_number['non-existent.key']), mapContains(attributes_bool, 'non-existent.key'), toString(attributes_bool['non-existent.key']), scope.attributes.`non-existent.key` IS NOT NULL, toString(scope.attributes.`non-existent.key`::String), NULL) AS `__SELECT_KEY_7_non-existent.key` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, name AS `__SELECT_KEY_3_name`, resource_string_service$$name AS `__SELECT_KEY_4_serviceName`, duration_nano AS `__SELECT_KEY_5_durationNano`, http_method AS `__SELECT_KEY_6_httpMethod`, multiIf(mapContains(attributes_string, 'non-existent.key'), toString(attributes_string['non-existent.key']), mapContains(attributes_number, 'non-existent.key'), toString(attributes_number['non-existent.key']), mapContains(attributes_bool, 'non-existent.key'), toString(attributes_bool['non-existent.key']), NULL) AS `__SELECT_KEY_7_non-existent.key` FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
expectedErr: nil,
@@ -888,110 +800,6 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
},
expectedErr: nil,
},
{
name: "List query with scope filter only (no scope in select or group by)",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{
"scope.version": {
{
Name: "scope.version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{
Expression: "scope.version = '1.0.0'",
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, trace_state AS `__SELECT_KEY_3_trace_state`, parent_span_id AS `__SELECT_KEY_4_parent_span_id`, flags AS `__SELECT_KEY_5_flags`, name AS `__SELECT_KEY_6_name`, kind AS `__SELECT_KEY_7_kind`, kind_string AS `__SELECT_KEY_8_kind_string`, duration_nano AS `__SELECT_KEY_9_duration_nano`, status_code AS `__SELECT_KEY_10_status_code`, status_message AS `__SELECT_KEY_11_status_message`, status_code_string AS `__SELECT_KEY_12_status_code_string`, events AS `__SELECT_KEY_13_events`, links AS `__SELECT_KEY_14_links`, response_status_code AS `__SELECT_KEY_15_response_status_code`, external_http_url AS `__SELECT_KEY_16_external_http_url`, http_url AS `__SELECT_KEY_17_http_url`, external_http_method AS `__SELECT_KEY_18_external_http_method`, http_method AS `__SELECT_KEY_19_http_method`, http_host AS `__SELECT_KEY_20_http_host`, db_name AS `__SELECT_KEY_21_db_name`, db_operation AS `__SELECT_KEY_22_db_operation`, has_error AS `__SELECT_KEY_23_has_error`, is_remote AS `__SELECT_KEY_24_is_remote`, attributes_string, attributes_number, attributes_bool, resources_string FROM signoz_traces.distributed_signoz_index_v3 WHERE (scope.version::String = ? AND scope.version::String <> '') AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1.0.0", "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
{
// Regression test: scope.version in selectFields with no metadata (isColumn=true filters it out)
// must still produce scope.version::String, not scope.attributes.version::String
name: "scope.version in selectFields only, no metadata (intrinsic field fallback)",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{},
SelectFields: []telemetrytypes.TelemetryFieldKey{
{Name: "scope.version", FieldContext: telemetrytypes.FieldContextUnspecified},
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, multiIf(scope.version::String <> '', scope.version::String, NULL) AS `__SELECT_KEY_3_scope.version` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
{
// A scope-context key that shares its name with a span intrinsic must resolve
// against the scope column, not the span `name` column, even with no metadata.
name: "scope-context name with no metadata resolves to the scope attribute, not the span column",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{},
SelectFields: []telemetrytypes.TelemetryFieldKey{
{Name: "name", FieldContext: telemetrytypes.FieldContextScope},
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, multiIf(scope.attributes.`name` IS NOT NULL, scope.attributes.`name`::String, NULL) AS `__SELECT_KEY_3_name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
{
// A scope name that collides with a declared path: with both the declared
// scope.version and a scope attribute literally named `version` in metadata, a
// select on `{version, scope}` unions both (attribute first, declared fallback).
name: "scope select field unions a same-named scope attribute and the declared path",
requestType: qbtypes.RequestTypeRaw,
keysMap: map[string][]*telemetrytypes.TelemetryFieldKey{
"scope.version": {
{
Name: "scope.version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"version": {
{
Name: "version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
query: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
Filter: &qbtypes.Filter{},
SelectFields: []telemetrytypes.TelemetryFieldKey{
{Name: "version", FieldContext: telemetrytypes.FieldContextScope},
},
Limit: 10,
},
expected: qbtypes.Statement{
Query: "SELECT timestamp AS `__SELECT_KEY_0_timestamp`, trace_id AS `__SELECT_KEY_1_trace_id`, span_id AS `__SELECT_KEY_2_span_id`, multiIf(scope.attributes.`version` IS NOT NULL, toString(scope.attributes.`version`::String), scope.version::String <> '', toString(scope.version::String), NULL) AS `__SELECT_KEY_3_version` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), 10},
},
},
}
for _, c := range cases {

View File

@@ -344,7 +344,7 @@ func TestTraceOperatorStatementBuilder(t *testing.T) {
},
},
expected: qbtypes.Statement{
Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND toFloat64(response_status_code) < ?), A_AND_B AS (SELECT l.* FROM A AS l INNER JOIN B AS r ON l.trace_id = r.trace_id) SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, avg(multiIf(duration_nano <> 0, toFloat64(duration_nano), mapContains(attributes_number, 'duration_nano'), toFloat64(attributes_number['duration_nano']), NULL)) AS __result_0 FROM A_AND_B GROUP BY `service.name` ORDER BY __result_0 desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000",
Query: "WITH toDateTime64(1747947419000000000, 9) AS t_from, toDateTime64(1747983448000000000, 9) AS t_to, 1747945619 AS bucket_from, 1747983448 AS bucket_to, all_spans AS (SELECT *, resource_string_service$$name AS `service.name` FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), __resource_filter_A AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint), A AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter_A) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ?), B AS (SELECT * FROM signoz_traces.distributed_signoz_index_v3 WHERE timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND toFloat64(response_status_code) < ?), A_AND_B AS (SELECT l.* FROM A AS l INNER JOIN B AS r ON l.trace_id = r.trace_id) SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, avg(multiIf(duration_nano <> 0, accurateCastOrNull(duration_nano, 'Float64'), mapContains(attributes_number, 'duration_nano'), toFloat64(attributes_number['duration_nano']), NULL)) AS __result_0 FROM A_AND_B GROUP BY `service.name` ORDER BY __result_0 desc SETTINGS distributed_product_mode='allow', max_memory_usage=10000000000",
Args: []any{"1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "frontend", "%service.name%", "%service.name\":\"frontend%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448), float64(400)},
},
expectedErr: nil,

View File

@@ -180,7 +180,7 @@ func (t *telemetryMetaStore) getTracesKeys(ctx context.Context, fieldKeySelector
`CASE
// WHEN tagType = 'spanfield' THEN 1
WHEN tagType = 'resource' THEN 2
WHEN tagType = 'scope' THEN 3
// WHEN tagType = 'scope' THEN 3
WHEN tagType = 'tag' THEN 4
ELSE 5
END as priority`,

View File

@@ -391,83 +391,6 @@ func TestConditionForResourceWithEvolution(t *testing.T) {
}
}
// TestConditionForScopeIntrinsicFields covers the scope.name/scope.version intrinsic
// fields against the "scope" JSON column. These are *declared* String paths on that
// column, so a row without a scope reads as ” and never NULL: presence must be an
// empty-string check, since "IS NOT NULL" would hold for every row. That also rules
// out treating them as nested attribute keys under scope.attributes, which are
// undeclared (Dynamic) paths and genuinely NULL when absent.
func TestConditionForScopeIntrinsicFields(t *testing.T) {
ctx := context.Background()
fm := NewFieldMapper(flaggertest.New(t))
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
testCases := []struct {
name string
key telemetrytypes.TelemetryFieldKey
operator qbtypes.FilterOperator
value any
expectedSQL string
}{
{
name: "Equal - scope.name",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
operator: qbtypes.FilterOperatorEqual,
value: "io.signoz.payment",
expectedSQL: "(scope.name::String = ? AND scope.name::String <> '')",
},
{
name: "Equal - scope.version",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
operator: qbtypes.FilterOperatorEqual,
value: "2.3.1",
expectedSQL: "(scope.version::String = ? AND scope.version::String <> '')",
},
{
name: "Exists - scope.name",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
operator: qbtypes.FilterOperatorExists,
value: nil,
expectedSQL: "scope.name::String <> ''",
},
{
name: "NotExists - scope.version",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
operator: qbtypes.FilterOperatorNotExists,
value: nil,
expectedSQL: "scope.version::String = ''",
},
}
for _, tc := range testCases {
sb := sqlbuilder.NewSelectBuilder()
t.Run(tc.name, func(t *testing.T) {
conds, _, err := conditionBuilder.ConditionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, map[string][]*telemetrytypes.TelemetryFieldKey{tc.key.Name: {&tc.key}}, qbtypes.ConditionBuilderOptions{}, tc.operator, tc.value, sb)
require.NoError(t, err)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, tc.expectedSQL)
assert.NotContains(t, sql, "scope.`scope.", "must not double-prefix the scope JSON path")
})
}
}
// TestConditionForSynthesizedKeys covers the KeyNotFound fallback: when a
// referenced attribute key has no metadata match, the builder synthesizes key(s) from
// user input and queries anyway, emitting a warning instead of failing.
@@ -518,13 +441,12 @@ func TestConditionForSynthesizedKeys(t *testing.T) {
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorExists, nil, sb)
assert.NoError(t, err)
assert.NotEmpty(t, warnings)
assert.Len(t, conds, 4, "exists should fan out to string/number/bool, plus scope attribute")
assert.Len(t, conds, 3, "exists should fan out to string/number/bool")
sb.Where(sb.Or(conds...))
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, "mapContains(attributes_string, 'exception.type')")
assert.Contains(t, sql, "mapContains(attributes_number, 'exception.type')")
assert.Contains(t, sql, "mapContains(attributes_bool, 'exception.type')")
assert.Contains(t, sql, "scope.attributes.`exception.type` IS NOT NULL")
})
t.Run("qualified data type honored without fanout", func(t *testing.T) {
@@ -532,11 +454,10 @@ func TestConditionForSynthesizedKeys(t *testing.T) {
key := telemetrytypes.TelemetryFieldKey{Name: "custom.key", FieldDataType: telemetrytypes.FieldDataTypeString}
conds, _, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
assert.NoError(t, err)
assert.Len(t, conds, 2, "qualified data type skips attribute-type fanout, but the scope attribute candidate still applies")
sb.Where(sb.Or(conds...))
assert.Len(t, conds, 1)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, "attributes_string['custom.key']")
assert.Contains(t, sql, "scope.attributes.`custom.key`")
})
t.Run("bare intrinsic column resolves to the column, not synthesized attributes", func(t *testing.T) {
@@ -598,11 +519,10 @@ func TestConditionForSynthesizedKeys(t *testing.T) {
conds, warnings, err := cb.ConditionFor(ctx, valuer.UUID{}, 0, 0, &key, noMatches, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "v", sb)
require.NoError(t, err)
assert.NotEmpty(t, warnings)
require.Len(t, conds, 2, "stripped attribute candidate, plus the scope attribute candidate")
sb.Where(sb.Or(conds...))
require.Len(t, conds, 1)
sb.Where(conds...)
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
assert.Contains(t, sql, "attributes_string['custom.attr']")
assert.Contains(t, sql, "scope.attributes.`custom.attr`")
assert.NotContains(t, sql, "span.custom.attr")
})

View File

@@ -121,20 +121,6 @@ var (
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"scope.name": {
Name: "scope.name",
Description: "Instrumentation scope name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"scope.version": {
Name: "scope.version",
Description: "Instrumentation scope version",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}
IntrinsicFieldsDeprecated = map[string]telemetrytypes.TelemetryFieldKey{
"traceID": {

View File

@@ -53,7 +53,6 @@ var (
ValueType: schema.ColumnTypeString,
}},
"resource": {Name: "resource", Type: schema.JSONColumnType{}},
"scope": {Name: "scope", Type: schema.JSONColumnType{}},
"events": {Name: "events", Type: schema.ArrayColumnType{
ElementType: schema.ColumnTypeString,
@@ -182,7 +181,7 @@ func (m *fieldMapper) getColumn(
case telemetrytypes.FieldContextResource:
return []*schema.Column{indexV3Columns["resource"], indexV3Columns["resources_string"]}, nil
case telemetrytypes.FieldContextScope:
return []*schema.Column{indexV3Columns["scope"]}, nil
return []*schema.Column{}, qbtypes.ErrColumnNotFound
case telemetrytypes.FieldContextAttribute:
switch key.FieldDataType {
case telemetrytypes.FieldDataTypeString:
@@ -293,24 +292,14 @@ func (m *fieldMapper) resolveColumnExprs(
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
// json is only supported for resource context as of now
if key.FieldContext != telemetrytypes.FieldContextResource {
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
}
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
case telemetrytypes.FieldContextScope:
if isDeclaredScopePath(key.Name) {
// declared String paths on the scope column read '' for the missing case
exprs = append(exprs, fmt.Sprintf("%s::String", key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s <> ''", key.Name))
} else {
exprs = append(exprs, fmt.Sprintf("%s.attributes.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.attributes.`%s` IS NOT NULL", columnName, key.Name))
}
default:
return nil, nil, nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource and scope context fields are supported for json columns, got %s", key.FieldContext.String)
}
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,
@@ -352,58 +341,18 @@ func (m *fieldMapper) resolveColumnExprs(
return exprs, existExprs, columns, nil
}
// resolveReferencedField resolves a referenced field to the candidate member key(s) a select /
// group by / order by queries for it, unioning every home the name maps to (a scope field and a
// same-named scope attribute, an attribute and a resource attribute, ...) flattened to members.
// Unlike the filter path it does not narrow an attribute+resource collision to resource — select
// surfaces every home. It returns empty when the name is absent from metadata; the caller
// synthesizes and upgrades the result back to families.
func (m *fieldMapper) resolveReferencedField(
ctx context.Context,
orgID valuer.UUID,
startNs, endNs uint64,
field *telemetrytypes.TelemetryFieldKey,
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
) []*telemetrytypes.TelemetryFieldKey {
var resolved []*telemetrytypes.TelemetryFieldKey
for _, logical := range querybuilder.MatchingLogicalFields(ctx, orgID, m.fl, field, fieldKeys) {
resolved = append(resolved, logical.Members...)
}
// A bare key that names a real column resolves to the column first, keeping only same-named
// metadata keys whose type is consistent with it so a corrupt entry can't shadow the column.
if field.FieldContext == telemetrytypes.FieldContextUnspecified && len(resolved) > 0 {
var column *schema.Column
var columnKey *telemetrytypes.TelemetryFieldKey
for _, k := range resolved {
if k.FieldContext == telemetrytypes.FieldContextSpan {
if cols, err := m.ColumnFor(ctx, orgID, startNs, endNs, k); err == nil && len(cols) > 0 {
column, columnKey = cols[0], k
}
break
}
}
if column == nil {
probe := telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextSpan, field.FieldDataType)
if cols, err := m.ColumnFor(ctx, orgID, startNs, endNs, probe); err == nil && len(cols) > 0 {
column, columnKey = cols[0], probe
}
}
if column != nil {
combined := []*telemetrytypes.TelemetryFieldKey{columnKey}
for _, k := range resolved {
if k == columnKey || k.FieldContext == telemetrytypes.FieldContextSpan {
continue
}
if columnMatchesDataType(column, k.FieldDataType) {
combined = append(combined, k)
}
}
resolved = combined
// logicalForResolvedColumn upgrades a directly-resolvable key (the FieldFor
// probe succeeded) to its family when the metadata map proves membership;
// otherwise the key stays a single-member logical field.
func (m *fieldMapper) logicalForResolvedColumn(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) *telemetrytypes.LogicalField {
for _, logical := range querybuilder.MatchingLogicalFields(ctx, orgID, m.fl, field, keys) {
if logical.IsFamily() &&
logical.FieldContext == field.FieldContext &&
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || logical.FieldDataType == field.FieldDataType) {
return logical
}
}
return resolved
return telemetrytypes.SingleLogicalField(field.Name, field)
}
// upgradeToFamilies swaps single-member candidates for their family when the
@@ -466,18 +415,26 @@ func (m *fieldMapper) ColumnExpressionFor(
keys map[string][]*telemetrytypes.TelemetryFieldKey,
) (string, error) {
// Resolve the candidate member key(s): the metadata union, or synthesized type-variant keys
// when the name is absent. Then upgrade members to their semantic-convention family; the
// family step never changes candidate order or non-family behavior.
raw := m.resolveReferencedField(ctx, orgID, startNs, endNs, field, keys)
if len(raw) == 0 {
// Absent from metadata: synthesize the type-variant candidate key(s).
raw = m.CandidateKeys(ctx, orgID, field, nil, candidateLookupKeys(field, keys))
// Resolve the candidate logical field(s).
var candidates []*telemetrytypes.LogicalField
switch _, err := m.FieldFor(ctx, orgID, startNs, endNs, field); {
case err == nil:
// A directly-resolvable key upgrades to its family when the metadata
// map proves membership; otherwise it stays single-member.
candidates = []*telemetrytypes.LogicalField{m.logicalForResolvedColumn(ctx, orgID, field, keys)}
case errors.Is(err, qbtypes.ErrColumnNotFound):
// The legacy candidate flow, unchanged: column (when the bare name is
// one) plus metadata matches, else synthesized type-variant keys. The
// family step below only swaps candidates for their family; it never
// changes candidate order or non-family behavior.
raw := m.CandidateKeys(ctx, orgID, field, nil, keys)
if len(raw) == 0 {
return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...)
}
candidates = m.upgradeToFamilies(ctx, orgID, field, querybuilder.WrapAsLogicalFields(field.Name, raw), keys)
default:
return "", err
}
if len(raw) == 0 {
return "", errors.Wrapf(querybuilder.NewKeyNotFoundError(field.Name), errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...)
}
candidates := m.upgradeToFamilies(ctx, orgID, field, querybuilder.WrapAsLogicalFields(field.Name, raw), keys)
// Group-by/order (String) and aggregation (String/Float64): every candidate is
// exists-guarded and coerced to requiredDataType, in a single multiIf. Raw select
@@ -633,55 +590,20 @@ func (m *fieldMapper) CandidateKeys(ctx context.Context, _ valuer.UUID, field *t
// No metadata: synthesize per context.
switch field.FieldContext {
case telemetrytypes.FieldContextUnspecified:
return append(querybuilder.SynthesizeKeys(field, value), synthScopeAttributeKey(field))
return querybuilder.SynthesizeKeys(field, value)
case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextTrace:
// honored as-is: the stripped name lives in the attribute or scope attribute maps
// honored as-is: the stripped name lives in the attribute maps
stripped := telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextUnspecified, field.FieldDataType)
return append(querybuilder.SynthesizeKeys(stripped, value), synthScopeAttributeKey(stripped))
return querybuilder.SynthesizeKeys(stripped, value)
case telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource:
// strict context honored as-is: stripped interpretation first, literal spelling second
literal := telemetrytypes.NewTelemetryFieldKey(field.FieldContext.StringValue()+"."+field.Name, field.FieldContext, field.FieldDataType)
return append(querybuilder.SynthesizeKeys(field, value), querybuilder.SynthesizeKeys(literal, value)...)
case telemetrytypes.FieldContextScope:
return []*telemetrytypes.TelemetryFieldKey{synthScopeAttributeKey(field)}
}
// contexts that don't exist on spans (log, body, …) have nothing to synthesize
// contexts that don't exist on spans (log, body, scope, …) have nothing to synthesize
return nil
}
// synthScopeAttributeKey guesses a scope attribute (scope.attributes.<name>) for a name absent
// from metadata — the scope analog of querybuilder.SynthesizeKeys.
func synthScopeAttributeKey(field *telemetrytypes.TelemetryFieldKey) *telemetrytypes.TelemetryFieldKey {
return telemetrytypes.NewTelemetryFieldKey(field.Name, telemetrytypes.FieldContextScope, telemetrytypes.FieldDataTypeString)
}
func isDeclaredScopePath(name string) bool {
f, ok := IntrinsicFields[name]
return ok && f.FieldContext == telemetrytypes.FieldContextScope
}
// scopeJSONExistsExpression renders the existence predicate for the scope JSON column, the one
// signal-specific case the generic querybuilder.ExistsExpression must not carry.
func scopeJSONExistsExpression(key *telemetrytypes.TelemetryFieldKey, fieldExpression string, exists bool) (string, bool) {
if key.FieldContext != telemetrytypes.FieldContextScope {
return "", false
}
// Declared String paths are non-Nullable (absent reads '' not NULL).
if isDeclaredScopePath(key.Name) {
if exists {
return fieldExpression + " <> ''", true
}
return fieldExpression + " = ''", true
}
// Scope attribute: the value expression casts the JSON path to String, which folds a missing
// key's NULL to '', so presence must test the raw path — drop the ::String cast.
path := strings.TrimSuffix(fieldExpression, "::String")
if exists {
return path + " IS NOT NULL", true
}
return path + " IS NULL", true
}
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
func (m *fieldMapper) ExistsFor(
ctx context.Context,
@@ -698,8 +620,5 @@ func (m *fieldMapper) ExistsFor(
if err != nil {
return "", err
}
if expr, ok := scopeJSONExistsExpression(key, fieldExpression, exists); ok {
return expr, nil
}
return querybuilder.ExistsExpression(columns, key, tsStart, tsEnd, fieldExpression, exists)
}

View File

@@ -84,33 +84,6 @@ func TestGetFieldKeyName(t *testing.T) {
expectedResult: "multiIf(resource.`deployment.environment` IS NOT NULL, resource.`deployment.environment`::String, `resource_string_deployment$$environment_exists`, `resource_string_deployment$$environment`, NULL)",
expectedError: nil,
},
{
name: "Scope field - scope.name",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.name::String",
expectedError: nil,
},
{
name: "Scope field - scope.version",
key: telemetrytypes.TelemetryFieldKey{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.version::String",
expectedError: nil,
},
{
name: "Scope field - custom attribute",
key: telemetrytypes.TelemetryFieldKey{
Name: "custom.attr",
FieldContext: telemetrytypes.FieldContextScope,
},
expectedResult: "scope.attributes.`custom.attr`::String",
expectedError: nil,
},
{
// Query like `attribute.attribute_string:string` should resolve to `attributes_string['attribute_string']`.
name: "Attribute key whose name collides with contextual map column resolves as a map lookup",
@@ -277,7 +250,7 @@ func TestColumnExpressionForTemporalColumn(t *testing.T) {
FieldDataType: telemetrytypes.FieldDataTypeString,
},
requiredDataType: telemetrytypes.FieldDataTypeString,
expectedResult: "multiIf(mapContains(attributes_string, 'user.id'), attributes_string['user.id'], mapContains(attributes_string, 'attribute.user.id'), attributes_string['attribute.user.id'], NULL)",
expectedResult: "multiIf(mapContains(attributes_string, 'user.id'), attributes_string['user.id'], NULL)",
},
}
@@ -331,78 +304,3 @@ func TestColumnExpressionForTimestampAttributeCollision(t *testing.T) {
assert.Contains(t, result, "attributes_number['timestamp']")
})
}
// TestColumnExpressionForScopeUnion covers select-side resolution of scope names that
// collide with a declared scope path. A short name under scope context (or the bare
// `scope.<x>` spelling that normalizes to it) binds to the declared path, and unions a
// same-named scope attribute when one is also in metadata. The full `scope.<x>` name under
// explicit scope context addresses the declared path alone.
func TestColumnExpressionForScopeUnion(t *testing.T) {
ctx := context.Background()
scopeKey := func(name string) *telemetrytypes.TelemetryFieldKey {
return &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
}
declaredOnly := map[string][]*telemetrytypes.TelemetryFieldKey{
"scope.name": {scopeKey("scope.name")},
"scope.version": {scopeKey("scope.version")},
}
withAttr := map[string][]*telemetrytypes.TelemetryFieldKey{
"scope.name": {scopeKey("scope.name")},
"scope.version": {scopeKey("scope.version")},
"name": {scopeKey("name")},
"version": {scopeKey("version")},
}
testCases := []struct {
name string
key telemetrytypes.TelemetryFieldKey
keys map[string][]*telemetrytypes.TelemetryFieldKey
expectedResult string
}{
{
name: "short name under scope context binds to the declared path",
key: telemetrytypes.TelemetryFieldKey{Name: "version", FieldContext: telemetrytypes.FieldContextScope},
keys: declaredOnly,
expectedResult: "multiIf(scope.version::String <> '', scope.version::String, NULL)",
},
{
name: "short name unions the declared path and a same-named scope attribute",
key: telemetrytypes.TelemetryFieldKey{Name: "version", FieldContext: telemetrytypes.FieldContextScope},
keys: withAttr,
expectedResult: "multiIf(scope.attributes.`version` IS NOT NULL, toString(scope.attributes.`version`::String), scope.version::String <> '', toString(scope.version::String), NULL)",
},
{
name: "full scope.version name under scope context addresses the declared path alone",
key: telemetrytypes.TelemetryFieldKey{Name: "scope.version", FieldContext: telemetrytypes.FieldContextScope},
keys: withAttr,
expectedResult: "multiIf(scope.version::String <> '', scope.version::String, NULL)",
},
{
name: "short scope name unions the declared scope.name and a same-named attribute",
key: telemetrytypes.TelemetryFieldKey{Name: "name", FieldContext: telemetrytypes.FieldContextScope},
keys: withAttr,
expectedResult: "multiIf(scope.attributes.`name` IS NOT NULL, toString(scope.attributes.`name`::String), scope.name::String <> '', toString(scope.name::String), NULL)",
},
{
name: "full scope.name name under scope context addresses the declared path alone",
key: telemetrytypes.TelemetryFieldKey{Name: "scope.name", FieldContext: telemetrytypes.FieldContextScope},
keys: withAttr,
expectedResult: "multiIf(scope.name::String <> '', scope.name::String, NULL)",
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
fm := NewFieldMapper(flaggertest.New(t))
result, err := fm.ColumnExpressionFor(ctx, valuer.UUID{}, 0, 0, &tc.key, telemetrytypes.FieldDataTypeUnspecified, tc.keys)
require.NoError(t, err)
assert.Equal(t, tc.expectedResult, result)
})
}
}

View File

@@ -113,20 +113,6 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
FieldDataType: telemetrytypes.FieldDataTypeBool,
},
},
"scope.name": {
{
Name: "scope.name",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"scope.version": {
{
Name: "scope.version",
FieldContext: telemetrytypes.FieldContextScope,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
// both spellings of an enabled semantic-convention family
"deployment.environment.name": {
{

View File

@@ -294,17 +294,6 @@ func TestNormalize(t *testing.T) {
FieldDataType: FieldDataTypeString,
},
},
{
name: "Normalize keeps a prefix that does not match the set context",
input: TelemetryFieldKey{
Name: "scope.name",
FieldContext: FieldContextAttribute,
},
expected: TelemetryFieldKey{
Name: "scope.name",
FieldContext: FieldContextAttribute,
},
},
{
name: "Normalize body field",
input: TelemetryFieldKey{

View File

@@ -4,9 +4,6 @@ settings:
autoInstallPeers: true
excludeLinksFromLockfile: false
overrides:
brace-expansion@>=5.0.0 <5.0.9: '>=5.0.9 <6'
importers:
.:
@@ -380,9 +377,9 @@ packages:
resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==}
engines: {node: 18 || 20 || >=22}
brace-expansion@5.0.9:
resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==}
engines: {node: 20 || >=22}
brace-expansion@5.0.5:
resolution: {integrity: sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==}
engines: {node: 18 || 20 || >=22}
cross-spawn@7.0.6:
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
@@ -848,7 +845,7 @@ snapshots:
balanced-match@4.0.4: {}
brace-expansion@5.0.9:
brace-expansion@5.0.5:
dependencies:
balanced-match: 4.0.4
@@ -1001,7 +998,7 @@ snapshots:
minimatch@10.2.5:
dependencies:
brace-expansion: 5.0.9
brace-expansion: 5.0.5
ms@2.1.3: {}

View File

@@ -1,6 +0,0 @@
# Security floors for vulnerable transitive deps. Where possible, targets are
# capped to avoid crossing breaking versions (major; and minor for 0.x).
overrides:
# via: eslint-plugin-playwright > eslint@10 > minimatch@10.2.5 (brace-expansion ^5.0.5)
# remove: blocked — minimatch@10.2.6 (latest) only widens to ^5.0.8, still vulnerable
'brace-expansion@>=5.0.0 <5.0.9': '>=5.0.9 <6'

View File

@@ -999,8 +999,6 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"trace_id": "corrupt_data",
"scope_name": "corrupt_data",
"scope.scope.name": "corrupt_data",
},
attributes={
"net.transport": "IP.TCP",
@@ -1009,10 +1007,7 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"http.request.method": "POST",
"http.response.status_code": "200",
"timestamp": "corrupt_data",
"version": "1.0.0",
"scope.scope.version": "1.0.0",
},
scope={"name": "io.signoz.http.server", "version": "2.0.0"},
),
Traces(
timestamp=now - timedelta(seconds=3.5),
@@ -1032,24 +1027,12 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"timestamp": "corrupt_data",
"scope.attributes.name": "corrupt_data",
},
attributes={
"db.name": "integration",
"db.operation": "SELECT",
"db.statement": "SELECT * FROM integration",
"trace_d": "corrupt_data",
"scope.attributes.version": "corrupt_data",
},
scope={
"name": "io.opentelemetry.contrib.http",
"version": "1.0.0",
"attributes": {
"telemetry.sdk.language": "cpp",
"name": "not-the-real-name",
"version": "not-the-real-version",
"attributes": "literally-a-key-named-attributes",
},
},
),
Traces(
@@ -1070,15 +1053,12 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"cloud.provider": "integration",
"cloud.account.id": "000",
"duration_nano": "corrupt_data",
"scope.scope.attributes.version": "corrupt_data",
},
attributes={
"http.request.method": "PATCH",
"http.status_code": "404",
"id": "1",
"scope.scope.version": "corrupt_data",
},
scope={"name": "io.signoz.http.client", "version": "2.0.0"},
),
Traces(
timestamp=now - timedelta(seconds=1),
@@ -1097,7 +1077,6 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"host.name": "linux-001",
"cloud.provider": "integration",
"cloud.account.id": "001",
"scope.scope.version": "corrupt_data",
},
attributes={
"message.type": "SENT",
@@ -1105,10 +1084,7 @@ def generate_traces_with_corrupt_metadata() -> list[Traces]:
"messaging.message.id": "001",
"duration_nano": "corrupt_data",
"id": 1,
"scope": "corrupt_data",
"scope.attributes.name": "corrupt_data",
},
scope={"name": "io.signoz.messaging", "version": "3.0.0"},
),
]

View File

@@ -302,7 +302,6 @@ class Traces(ABC):
db_operation: str
has_error: bool
is_remote: str
scope_json: dict[str, Any]
resource: list[TracesResource]
tag_attributes: list[TracesTagAttributes]
@@ -328,7 +327,6 @@ class Traces(ABC):
links: list[TracesLink] = [],
trace_state: str = "",
flags: np.uint32 = 0,
scope: dict[str, Any] = {},
resource_write_mode: Literal["legacy_only", "dual_write"] = "dual_write",
) -> None:
if timestamp is None:
@@ -410,33 +408,6 @@ class Traces(ABC):
# Calculate resource fingerprint
self.resource_fingerprint = LogsOrTracesFingerprint(self.resources_string).calculate()
# Process scope mirroring the InstrumentationScope on the OTLP span.
scope_name = scope.get("name", "")
scope_version = scope.get("version", "")
scope_string = {k: str(v) for k, v in scope.get("attributes", {}).items()}
self.scope_json = {
"name": scope_name,
"version": scope_version,
"attributes": scope_string,
}
scope_keys = {"scope.name": scope_name, "scope.version": scope_version}
scope_keys.update(scope_string)
for k, v in scope_keys.items():
if v == "":
continue
self.tag_attributes.append(
TracesTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="scope",
tag_data_type="string",
string_value=v,
number_value=None,
)
)
self.attribute_keys.append(TracesResourceOrAttributeKeys(name=k, datatype="string", tag_type="scope"))
# Process attributes by type and populate custom fields
self.attribute_string = {}
self.attributes_number = {}
@@ -688,7 +659,6 @@ class Traces(ABC):
self.has_error,
self.is_remote,
self.resource_json,
self.scope_json,
],
dtype=object,
)
@@ -719,7 +689,6 @@ class Traces(ABC):
attributes=data.get("attributes", {}),
trace_state=data.get("trace_state", ""),
flags=data.get("flags", 0),
scope=data.get("scope", {}),
)
@classmethod
@@ -859,7 +828,6 @@ def insert_traces_to_clickhouse(conn, traces: list[Traces]) -> None:
"has_error",
"is_remote",
"resource",
"scope",
],
data=[trace.np_arr() for trace in traces],
)

View File

@@ -1240,13 +1240,6 @@ def test_traces_list_span_scope(
lambda x: {"duration_nano": int(x[1].duration_nano), "span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id},
id="select_attribute_duration_order_intrinsic",
),
# Case 9: filter on the intrinsic scope.version. Only x[1] should match.
pytest.param(
BuilderQuery(signal="traces", name="A", select_fields=[TelemetryFieldKey("timestamp")], filter_expression="scope.version = '1.0.0'", limit=1),
HTTPStatus.OK,
lambda x: {"span_id": x[1].span_id, "timestamp": format_timestamp(x[1].timestamp), "trace_id": x[1].trace_id},
id="filter_scope_version",
),
],
)
def test_traces_list_with_corrupt_data(
@@ -1290,156 +1283,6 @@ def test_traces_list_with_corrupt_data(
assert get_rows(response)[0]["data"] == expected(traces)
@pytest.mark.parametrize(
"filter_expression,expected_indices",
[
# Intrinsic scope.name / scope.version resolve to the JSON sub-columns.
pytest.param("scope.name = 'io.signoz.payment'", [1], id="intrinsic_scope_name"),
pytest.param("scope.version = '2.3.1'", [0], id="intrinsic_scope_version"),
# A scope attribute resolves against the scope JSON column's attributes.
pytest.param("scope.telemetry.sdk.language = 'python'", [1], id="scope_attribute"),
# `env.tier` is a span attribute on span 0 and a scope attribute on
# span 1. Unprefixed -> no explicit context, so it is checked in every
# applicable context (attribute OR scope) and both spans match.
pytest.param("env.tier = 'gold'", [0, 1], id="bare_cross_context"),
# The explicit `scope.` prefix forces scope context only, so span 0's
# span attribute is ignored — only span 1 matches.
pytest.param("scope.env.tier = 'gold'", [1], id="scope_prefixed_cross_context"),
# `scope.name` matches BOTH the intrinsic scope.name field (span 0) and a
# scope attribute literally named `name` (span 1's scope attribute
# name='io.signoz.checkout').
pytest.param("scope.name = 'io.signoz.checkout'", [0, 1], id="scope_name_collision"),
# `scope.name` also matches a span attribute literally named `scope.name`
# (attribute context) — span 2 carries attribute scope.name='attr-scope-name'.
pytest.param("scope.name = 'attr-scope-name'", [2], id="scope_name_attribute_collision"),
# An unprefixed `name` resolves to the intrinsic span `name` column and a
# `name` scope attribute, but NOT the scope.name field. Span 2's span
# name and span 1's scope attribute `name` both equal 'io.signoz.checkout';
# span 0's scope.name field equals it too but is NOT matched.
pytest.param("name = 'io.signoz.checkout'", [1, 2], id="bare_name_excludes_scope_name_field"),
# A value that no resolvable key holds (scope.name/scope.version field,
# a `name`/`version` scope attribute, or a same-named attribute/resource)
# returns nothing.
pytest.param("scope.version = 'corrupt_data'", [], id="scope_version_no_match"),
pytest.param("scope.name = 'corrupt_data'", [], id="scope_name_no_match"),
],
)
def test_traces_list_with_scope_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[list[Traces]], None],
filter_expression: str,
expected_indices: list[int],
) -> None:
"""
Setup three spans with different scope key resolution:
- x[0]: scope.name/version 'io.signoz.checkout'/'2.3.1'; span attribute
env.tier='gold'.
- x[1]: scope.name/version 'io.signoz.payment'/'4.5.6'; scope attributes
telemetry.sdk.language='python', env.tier='gold', and a `name` scope
attribute colliding with x[0]'s scope.name value.
- x[2]: span name 'io.signoz.checkout' (colliding with x[0]'s scope.name
value) and a span attribute literally named `scope.name`.
Tests:
- Filtering on scope.name / scope.version / a scope attribute.
- An unprefixed key is resolved across contexts (scope checked alongside
attribute / intrinsic), while a `scope.`-prefixed key is scope-only.
- `scope.name` hits the intrinsic field, a `name` scope attribute, and a
span attribute `scope.name` (cross-context), while a bare `name` hits
the span name column (and a `name` scope attribute) but never the
scope.name field.
"""
now = datetime.now(tz=UTC).replace(microsecond=0)
trace_id = TraceIdGenerator.trace_id()
span_ids = [TraceIdGenerator.span_id() for _ in range(3)]
traces = [
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=2),
trace_id=trace_id,
span_id=span_ids[0],
parent_span_id="",
name="GET /checkout",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "checkout"},
attributes={"http.request.method": "GET", "env.tier": "gold"},
scope={
"name": "io.signoz.checkout",
"version": "2.3.1",
"attributes": {"telemetry.sdk.language": "go"},
},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=trace_id,
span_id=span_ids[1],
parent_span_id="",
name="POST /pay",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "payment"},
attributes={"http.request.method": "POST"},
# env.tier is a scope attribute here (cross-context with span 0);
# `name` is a scope attribute colliding with span 0's scope.name.
scope={
"name": "io.signoz.payment",
"version": "4.5.6",
"attributes": {
"telemetry.sdk.language": "python",
"env.tier": "gold",
"name": "io.signoz.checkout",
},
},
),
Traces(
timestamp=now - timedelta(seconds=1),
duration=timedelta(seconds=1),
trace_id=trace_id,
span_id=span_ids[2],
parent_span_id="",
# span name collides with span 0's scope.name value
name="io.signoz.checkout",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
resources={"service.name": "probe"},
# a span attribute named `scope.name`
attributes={"scope.name": "attr-scope-name"},
scope={"name": "span-gamma", "version": "9.9.9"},
),
]
insert_traces(traces)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
start_ms, end_ms = _query_window(now)
response = make_query_request(
signoz,
token,
start_ms=start_ms,
end_ms=end_ms,
request_type=RequestType.RAW,
queries=[
BuilderQuery(
signal="traces",
name="A",
select_fields=[TelemetryFieldKey("timestamp")],
filter_expression=filter_expression,
limit=10,
).to_dict()
],
)
assert response.status_code == HTTPStatus.OK, response.text
got_span_ids = {row["data"]["span_id"] for row in get_rows(response)}
expected_span_ids = {traces[i].span_id for i in expected_indices}
assert got_span_ids == expected_span_ids
@pytest.mark.parametrize("surface", ["filter", "select", "order"])
def test_traces_list_unknown_span_context_synthesizes(
signoz: types.SigNoz,