mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-10 23:10:47 +01:00
Compare commits
1 Commits
tvats-in-o
...
feat/semco
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1d9fe687cb |
@@ -376,19 +376,7 @@ function App(): JSX.Element {
|
||||
tracesSampleRate: 0, // Ref: https://github.com/SigNoz/platform-pod/issues/2393#issuecomment-4603658055
|
||||
replaysSessionSampleRate: 0.1, // This sets the sample rate at 10%. You may want to change it to 100% while in development and then sample at a lower rate in production.
|
||||
replaysOnErrorSampleRate: 1.0, // If you're not already sampling the entire session, change the sample rate to 100% when sampling sessions where errors occur.
|
||||
beforeSend(event, hint) {
|
||||
const error = hint?.originalException as
|
||||
| { name?: string; code?: string | number }
|
||||
| undefined;
|
||||
|
||||
// Ignore benign aborted/cancelled requests (axios + fetch).
|
||||
if (error?.code === 'ERR_CANCELED' || error?.code === 'ECONNABORTED') {
|
||||
return null;
|
||||
}
|
||||
if (error?.name === 'AbortError') {
|
||||
return null;
|
||||
}
|
||||
|
||||
beforeSend(event) {
|
||||
// Drop the event if its level is 'warning' or 'info'
|
||||
if (event.level === 'warning' || event.level === 'info') {
|
||||
return null;
|
||||
|
||||
31
frontend/src/api/v1/factor_password/resetPassword.ts
Normal file
31
frontend/src/api/v1/factor_password/resetPassword.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
import axios from 'api';
|
||||
import { ErrorResponseHandlerV2 } from 'api/ErrorResponseHandlerV2';
|
||||
import { AxiosError } from 'axios';
|
||||
import { ErrorV2Resp, SuccessResponseV2 } from 'types/api';
|
||||
import { PayloadProps, Props } from 'types/api/user/resetPassword';
|
||||
|
||||
/**
|
||||
* @deprecated Use the generated `useResetPassword` hook (or `resetPassword` fetcher) from
|
||||
* `api/generated/services/users` instead. This hand-written client targets the
|
||||
* same endpoint and will be removed once call sites migrate.
|
||||
*
|
||||
* Part of https://github.com/SigNoz/engineering-pod/issues/5289, add a comment or update when removing this method.
|
||||
*/
|
||||
const resetPassword = async (
|
||||
props: Props,
|
||||
): Promise<SuccessResponseV2<PayloadProps>> => {
|
||||
try {
|
||||
const response = await axios.post<PayloadProps>(`/resetPassword`, {
|
||||
...props,
|
||||
});
|
||||
|
||||
return {
|
||||
httpStatusCode: response.status,
|
||||
data: response.data,
|
||||
};
|
||||
} catch (error) {
|
||||
ErrorResponseHandlerV2(error as AxiosError<ErrorV2Resp>);
|
||||
}
|
||||
};
|
||||
|
||||
export default resetPassword;
|
||||
@@ -183,14 +183,15 @@ function QuerySearch({
|
||||
isProgrammaticChangeRef.current = true;
|
||||
}
|
||||
|
||||
const changes = view.state.changes({
|
||||
from: 0,
|
||||
to: currentValue.length,
|
||||
insert: value,
|
||||
});
|
||||
view.dispatch({
|
||||
changes,
|
||||
selection: { anchor: changes.newLength },
|
||||
changes: {
|
||||
from: 0,
|
||||
to: currentValue.length,
|
||||
insert: value,
|
||||
},
|
||||
selection: {
|
||||
anchor: value.length,
|
||||
},
|
||||
});
|
||||
},
|
||||
[],
|
||||
|
||||
@@ -301,66 +301,6 @@ describe('QuerySearch (Integration with Real CodeMirror)', () => {
|
||||
dispatchSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('does not crash when the expression contains CRLF line breaks (issue #5869)', async () => {
|
||||
const dispatchSpy = jest.spyOn(EditorView.prototype, 'dispatch');
|
||||
const onChange = jest.fn() as jest.MockedFunction<(v: string) => void>;
|
||||
const initialExpression = "service.name = 'frontend'";
|
||||
// Filtering on a multi-line log value (CRLF) used to throw
|
||||
// "RangeError: Selection points outside of document".
|
||||
const crlfExpression = "body CONTAINS 'line1\r\nline2\r\nline3'";
|
||||
|
||||
const baseQueryData = {
|
||||
...initialQueriesMap.logs.builder.queryData[0],
|
||||
filter: { expression: initialExpression },
|
||||
};
|
||||
|
||||
const { rerender } = render(
|
||||
<QuerySearch
|
||||
onChange={onChange}
|
||||
queryData={baseQueryData}
|
||||
dataSource={DataSource.LOGS}
|
||||
/>,
|
||||
);
|
||||
|
||||
await waitFor(
|
||||
() => {
|
||||
const editorContent = document.querySelector(
|
||||
CM_EDITOR_SELECTOR,
|
||||
) as HTMLElement;
|
||||
expect(editorContent.textContent || '').toBe(initialExpression);
|
||||
},
|
||||
{ timeout: 3000 },
|
||||
);
|
||||
|
||||
rerender(
|
||||
<QuerySearch
|
||||
onChange={onChange}
|
||||
queryData={{ ...baseQueryData, filter: { expression: crlfExpression } }}
|
||||
dataSource={DataSource.LOGS}
|
||||
/>,
|
||||
);
|
||||
|
||||
// The programmatic replace dispatched without throwing, and the selection anchor
|
||||
// stayed within the CRLF-normalized document (the bug set it past the end).
|
||||
await waitFor(() => {
|
||||
const spec = dispatchSpy.mock.calls
|
||||
.map(
|
||||
(call) =>
|
||||
call[0] as {
|
||||
selection?: { anchor?: number };
|
||||
changes?: { newLength?: number };
|
||||
},
|
||||
)
|
||||
.find((s) => s?.selection?.anchor != null && s?.changes?.newLength != null);
|
||||
expect(spec).toBeDefined();
|
||||
expect(spec?.selection?.anchor).toBeLessThanOrEqual(
|
||||
spec?.changes?.newLength as number,
|
||||
);
|
||||
});
|
||||
|
||||
dispatchSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('fetches key suggestions for metrics even without aggregateAttribute.key when showFilterSuggestionsWithoutMetric is true', async () => {
|
||||
const mockedGetKeys = getKeySuggestions as jest.MockedFunction<
|
||||
typeof getKeySuggestions
|
||||
|
||||
@@ -33,7 +33,7 @@ jest.mock('hooks/useNotifications', () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
const RESET_PASSWORD_ENDPOINT = '*/api/v2/factor_password/reset';
|
||||
const RESET_PASSWORD_ENDPOINT = '*/resetPassword';
|
||||
|
||||
const mockHistoryPush = history.push as jest.MockedFunction<
|
||||
typeof history.push
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import { useMemo, useState } from 'react';
|
||||
import { useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { useLocation } from 'react-use';
|
||||
import { Button } from '@signozhq/ui/button';
|
||||
import { Callout } from '@signozhq/ui/callout';
|
||||
import { Form, Input as AntdInput } from 'antd';
|
||||
import { Typography } from '@signozhq/ui/typography';
|
||||
import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
|
||||
import { useResetPassword } from 'api/generated/services/users';
|
||||
import resetPasswordApi from 'api/v1/factor_password/resetPassword';
|
||||
import AuthError from 'components/AuthError/AuthError';
|
||||
import AuthPageContainer from 'components/AuthPageContainer';
|
||||
import ROUTES from 'constants/routes';
|
||||
@@ -15,6 +14,7 @@ import { useNotifications } from 'hooks/useNotifications';
|
||||
import history from 'lib/history';
|
||||
import { ArrowRight, CircleAlert, KeyRound } from '@signozhq/icons';
|
||||
import { Label } from 'pages/SignUp/styles';
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
import { FormContainer } from './styles';
|
||||
|
||||
@@ -26,41 +26,40 @@ function ResetPassword({ version }: ResetPasswordProps): JSX.Element {
|
||||
const [confirmPasswordError, setConfirmPasswordError] =
|
||||
useState<boolean>(false);
|
||||
|
||||
const [errorMessage, setErrorMessage] = useState<APIError | null>();
|
||||
|
||||
const [isValidPassword, setIsValidPassword] = useState(false);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const { t } = useTranslation(['common']);
|
||||
const { search } = useLocation();
|
||||
const params = new URLSearchParams(search);
|
||||
const token = params.get('token');
|
||||
const { notifications } = useNotifications();
|
||||
|
||||
const {
|
||||
mutate: resetPassword,
|
||||
isLoading,
|
||||
error: mutationError,
|
||||
} = useResetPassword();
|
||||
|
||||
const errorMessage = useMemo(
|
||||
() => convertToApiError(mutationError),
|
||||
[mutationError],
|
||||
);
|
||||
|
||||
const [form] = Form.useForm<FormValues>();
|
||||
const handleFormSubmit = (): void => {
|
||||
const { password } = form.getFieldsValue();
|
||||
const handleFormSubmit: () => Promise<void> = async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
setErrorMessage(null);
|
||||
const { password } = form.getFieldsValue();
|
||||
|
||||
resetPassword(
|
||||
{ data: { password, token: token || '' } },
|
||||
{
|
||||
onSuccess: (): void => {
|
||||
notifications.success({
|
||||
message: t('success', {
|
||||
ns: 'common',
|
||||
}),
|
||||
});
|
||||
history.push(ROUTES.LOGIN);
|
||||
},
|
||||
},
|
||||
);
|
||||
await resetPasswordApi({
|
||||
password,
|
||||
token: token || '',
|
||||
});
|
||||
|
||||
notifications.success({
|
||||
message: t('success', {
|
||||
ns: 'common',
|
||||
}),
|
||||
});
|
||||
history.push(ROUTES.LOGIN);
|
||||
|
||||
setLoading(false);
|
||||
} catch (error) {
|
||||
setLoading(false);
|
||||
setErrorMessage(error as APIError);
|
||||
}
|
||||
};
|
||||
|
||||
const validatePassword = (): boolean => {
|
||||
@@ -223,7 +222,7 @@ function ResetPassword({ version }: ResetPasswordProps): JSX.Element {
|
||||
color="primary"
|
||||
type="submit"
|
||||
data-attr="reset-password"
|
||||
disabled={!isValidPassword || isLoading}
|
||||
disabled={!isValidPassword || loading}
|
||||
className="reset-password-submit-button"
|
||||
suffix={<ArrowRight size={16} />}
|
||||
>
|
||||
|
||||
9
frontend/src/types/api/user/resetPassword.ts
Normal file
9
frontend/src/types/api/user/resetPassword.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export interface Props {
|
||||
token: string;
|
||||
password: string;
|
||||
}
|
||||
|
||||
export interface PayloadProps {
|
||||
data: string;
|
||||
status: string;
|
||||
}
|
||||
@@ -3,16 +3,17 @@ package flagger
|
||||
import "github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
|
||||
var (
|
||||
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
|
||||
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
|
||||
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
|
||||
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
|
||||
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
|
||||
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
|
||||
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
|
||||
FeatureEnableAIObservability = featuretypes.MustNewName("enable_ai_observability")
|
||||
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
|
||||
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
|
||||
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
|
||||
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
|
||||
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
|
||||
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
|
||||
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
|
||||
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
|
||||
FeatureEnableAIObservability = featuretypes.MustNewName("enable_ai_observability")
|
||||
FeatureEnableMetricsReduction = featuretypes.MustNewName("enable_metrics_reduction")
|
||||
FeatureUsePrometheusClickhouseV2 = featuretypes.MustNewName("use_prometheus_clickhouse_v2")
|
||||
FeatureResolveSemconvFamilies = featuretypes.MustNewName("resolve_semconv_families")
|
||||
)
|
||||
|
||||
func MustNewRegistry() featuretypes.Registry {
|
||||
@@ -97,6 +98,14 @@ func MustNewRegistry() featuretypes.Registry {
|
||||
DefaultVariant: featuretypes.MustNewName("disabled"),
|
||||
Variants: featuretypes.NewBooleanVariants(),
|
||||
},
|
||||
&featuretypes.Feature{
|
||||
Name: FeatureResolveSemconvFamilies,
|
||||
Kind: featuretypes.KindBoolean,
|
||||
Stage: featuretypes.StageExperimental,
|
||||
Description: "Controls whether trace queries resolve a semantic-convention name to all the spellings of its family",
|
||||
DefaultVariant: featuretypes.MustNewName("disabled"),
|
||||
Variants: featuretypes.NewBooleanVariants(),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
@@ -40,7 +40,10 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys))
|
||||
// Rule state history fields have no family support, so every logical field
|
||||
// is single-member and flattens losslessly to its physical key.
|
||||
resolved, warning := querybuilder.ResolveLogicalFields(key, querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
|
||||
keys := querybuilder.SingleKeys(resolved)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
|
||||
@@ -64,6 +64,23 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
|
||||
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
|
||||
// Real columns always exist; labels are checked for key membership.
|
||||
func (m *fieldMapper) ExistsFor(ctx context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
|
||||
col, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if col.Name != "labels" || key.Name == "labels" {
|
||||
return "true", nil
|
||||
}
|
||||
pred := fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", strings.ReplaceAll(key.Name, "'", "\\'"))
|
||||
if exists {
|
||||
return pred, nil
|
||||
}
|
||||
return "not " + pred, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, field *telemetrytypes.TelemetryFieldKey, _ telemetrytypes.FieldDataType, _ map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
colName, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, field)
|
||||
if err != nil {
|
||||
|
||||
64
pkg/querybuilder/family_selectors.go
Normal file
64
pkg/querybuilder/family_selectors.go
Normal file
@@ -0,0 +1,64 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// semconvFamiliesEnabled evaluates the resolve_semconv_families flag for the
|
||||
// org. A nil flagger means off, so a caller without family support stays
|
||||
// literal by default.
|
||||
func semconvFamiliesEnabled(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger) bool {
|
||||
if fl == nil {
|
||||
return false
|
||||
}
|
||||
return fl.BooleanOrEmpty(ctx, flagger.FeatureResolveSemconvFamilies, featuretypes.NewFlaggerEvaluationContext(orgID))
|
||||
}
|
||||
|
||||
// ExpandKeySelectorsForFamilies adds selectors for the other members of each
|
||||
// semantic-convention family that a selector names. The metadata fetched for
|
||||
// a query then contains each spelling that MatchingLogicalFields can group.
|
||||
// This function is the prefetch of the resolution layer: statement builders
|
||||
// call it after they derive the selectors, and the metadata store stays
|
||||
// family-blind (autocomplete responses keep the literal spelling that the
|
||||
// user typed). It does nothing when the resolve_semconv_families flag is off
|
||||
// for the org. Only trace selectors expand today, because that matches the
|
||||
// family support. Fuzzy (search-style) selectors never expand.
|
||||
func ExpandKeySelectorsForFamilies(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, selectors []*telemetrytypes.FieldKeySelector) []*telemetrytypes.FieldKeySelector {
|
||||
if !semconvFamiliesEnabled(ctx, orgID, fl) {
|
||||
return selectors
|
||||
}
|
||||
|
||||
out := selectors
|
||||
seen := make(map[string]bool, len(selectors))
|
||||
for _, selector := range selectors {
|
||||
seen[selector.Name] = true
|
||||
}
|
||||
|
||||
for _, selector := range selectors {
|
||||
if selector.Signal != telemetrytypes.SignalTraces ||
|
||||
selector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeFuzzy {
|
||||
continue
|
||||
}
|
||||
members := semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: selector.Name,
|
||||
Signal: selector.Signal,
|
||||
FieldContext: selector.FieldContext,
|
||||
})
|
||||
for _, member := range members {
|
||||
if seen[member] {
|
||||
continue
|
||||
}
|
||||
seen[member] = true
|
||||
expanded := *selector
|
||||
expanded.Name = member
|
||||
out = append(out, &expanded)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -21,24 +21,25 @@ const (
|
||||
hasTokenFunctionDocURL = "https://signoz.io/docs/userguide/functions-reference/#hastoken-function"
|
||||
)
|
||||
|
||||
// ResolveKeys picks which matching field keys a filter term builds conditions for.
|
||||
// With 0 or 1 match it returns the input unchanged and no warning. When a name is
|
||||
// ambiguous it returns a warning; a resource+attribute mix defaults to the resource
|
||||
// keys (the common intent), noted in the warning.
|
||||
func ResolveKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeysForName []*telemetrytypes.TelemetryFieldKey) ([]*telemetrytypes.TelemetryFieldKey, string) {
|
||||
if len(fieldKeysForName) <= 1 {
|
||||
return fieldKeysForName, ""
|
||||
// ResolveLogicalFields picks which logical fields a filter term builds conditions
|
||||
// for. With 0 or 1 field it returns the input unchanged and no warning. When a
|
||||
// name is ambiguous (several logical fields — a family is one field and never
|
||||
// ambiguous with itself) it returns a warning; a resource+attribute mix defaults
|
||||
// to the resource fields (the common intent), noted in the warning.
|
||||
func ResolveLogicalFields(field *telemetrytypes.TelemetryFieldKey, logicalFields []*telemetrytypes.LogicalField) ([]*telemetrytypes.LogicalField, string) {
|
||||
if len(logicalFields) <= 1 {
|
||||
return logicalFields, ""
|
||||
}
|
||||
|
||||
warning := fmt.Sprintf(
|
||||
"Key `%s` is ambiguous, found %d different combinations of field context / data type: %v.",
|
||||
field.Name,
|
||||
len(fieldKeysForName),
|
||||
fieldKeysForName,
|
||||
len(logicalFields),
|
||||
logicalFields,
|
||||
)
|
||||
|
||||
hasResource, hasAttribute := false, false
|
||||
for _, item := range fieldKeysForName {
|
||||
for _, item := range logicalFields {
|
||||
switch item.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
hasResource = true
|
||||
@@ -49,18 +50,40 @@ func ResolveKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeysForName []*te
|
||||
|
||||
// when there is both resource and attribute context, default to resource only
|
||||
if hasResource && hasAttribute {
|
||||
filteredKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(fieldKeysForName))
|
||||
for _, item := range fieldKeysForName {
|
||||
filtered := make([]*telemetrytypes.LogicalField, 0, len(logicalFields))
|
||||
for _, item := range logicalFields {
|
||||
if item.FieldContext == telemetrytypes.FieldContextResource {
|
||||
filteredKeys = append(filteredKeys, item)
|
||||
filtered = append(filtered, item)
|
||||
}
|
||||
}
|
||||
fieldKeysForName = filteredKeys
|
||||
logicalFields = filtered
|
||||
warning += " " + "Using `resource` context by default. To query attributes explicitly, " +
|
||||
fmt.Sprintf("use the fully qualified name (e.g., 'attribute.%s')", field.Name)
|
||||
}
|
||||
|
||||
return fieldKeysForName, warning
|
||||
return logicalFields, warning
|
||||
}
|
||||
|
||||
// WrapAsLogicalFields wraps physical keys (candidate or synthesized) as
|
||||
// single-member logical fields addressed by the requested spelling.
|
||||
func WrapAsLogicalFields(requestedName string, keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.LogicalField {
|
||||
fields := make([]*telemetrytypes.LogicalField, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
fields = append(fields, telemetrytypes.SingleLogicalField(requestedName, key))
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// SingleKeys flattens logical fields to their single members. It is the
|
||||
// adapter for signals whose fields are single-member by construction (every
|
||||
// signal without family support); their condition builders keep compiling per
|
||||
// physical key.
|
||||
func SingleKeys(fields []*telemetrytypes.LogicalField) []*telemetrytypes.TelemetryFieldKey {
|
||||
keys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(fields))
|
||||
for _, field := range fields {
|
||||
keys = append(keys, field.Single())
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// NewKeyNotFoundError builds the error a condition builder returns when a filter term
|
||||
|
||||
94
pkg/querybuilder/logical_expr.go
Normal file
94
pkg/querybuilder/logical_expr.go
Normal file
@@ -0,0 +1,94 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// The two functions below are the only place family expressions are built.
|
||||
// They compose exclusively from the mapper's per-key primitives (FieldFor,
|
||||
// ExistsFor), so every member honors its own storage: materialized columns,
|
||||
// evolutions, and JSON plans ride the member keys, and a signal supports
|
||||
// families the moment its primitives are correct.
|
||||
|
||||
// LogicalValueExpr returns the value expression for a resolved logical field:
|
||||
// the member's own expression for a single-member field, and a current-first
|
||||
// merge across the members' expressions for a family.
|
||||
func LogicalValueExpr(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
tsStart, tsEnd uint64,
|
||||
fm qbtypes.FieldMapper,
|
||||
logical *telemetrytypes.LogicalField,
|
||||
) (string, error) {
|
||||
if !logical.IsFamily() {
|
||||
return fm.FieldFor(ctx, orgID, tsStart, tsEnd, logical.Single())
|
||||
}
|
||||
|
||||
memberExprs := make([]string, 0, len(logical.Members))
|
||||
for _, member := range logical.Members {
|
||||
expr, err := fm.FieldFor(ctx, orgID, tsStart, tsEnd, member)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
memberExprs = append(memberExprs, expr)
|
||||
}
|
||||
|
||||
if logical.FieldDataType == telemetrytypes.FieldDataTypeString {
|
||||
// The trailing '' keeps single-key semantics for rows without any
|
||||
// member: string maps read '' for an absent key, and negative
|
||||
// operators must keep including such rows (see AddDefaultExistsFilter).
|
||||
values := make([]string, 0, len(memberExprs))
|
||||
for _, expr := range memberExprs {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s, '')", expr))
|
||||
}
|
||||
return "COALESCE(" + strings.Join(values, ", ") + ", '')", nil
|
||||
}
|
||||
|
||||
// Numeric and boolean maps return zero for an absent key. If a family of
|
||||
// either type is enabled, this tail must become zero too.
|
||||
branches := make([]string, 0, len(logical.Members)*2)
|
||||
for i, member := range logical.Members {
|
||||
guard, err := fm.ExistsFor(ctx, orgID, tsStart, tsEnd, member, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
branches = append(branches, guard, memberExprs[i])
|
||||
}
|
||||
return "multiIf(" + strings.Join(branches, ", ") + ", NULL)", nil
|
||||
}
|
||||
|
||||
// LogicalExistsExpr returns the existence predicate for a resolved logical
|
||||
// field: the member's own predicate for a single-member field, presence of
|
||||
// any member for a family.
|
||||
func LogicalExistsExpr(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
tsStart, tsEnd uint64,
|
||||
fm qbtypes.FieldMapper,
|
||||
logical *telemetrytypes.LogicalField,
|
||||
exists bool,
|
||||
) (string, error) {
|
||||
if !logical.IsFamily() {
|
||||
return fm.ExistsFor(ctx, orgID, tsStart, tsEnd, logical.Single(), exists)
|
||||
}
|
||||
|
||||
guards := make([]string, 0, len(logical.Members))
|
||||
for _, member := range logical.Members {
|
||||
guard, err := fm.ExistsFor(ctx, orgID, tsStart, tsEnd, member, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
guards = append(guards, guard)
|
||||
}
|
||||
combined := "(" + strings.Join(guards, " OR ") + ")"
|
||||
if exists {
|
||||
return combined, nil
|
||||
}
|
||||
return "NOT " + combined, nil
|
||||
}
|
||||
95
pkg/querybuilder/logical_expr_test.go
Normal file
95
pkg/querybuilder/logical_expr_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// stubFieldMapper provides just the two per-key primitives the shared
|
||||
// composition builds on; the remaining FieldMapper methods are unused here.
|
||||
type stubFieldMapper struct{}
|
||||
|
||||
func (stubFieldMapper) FieldFor(_ context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
return "value(" + key.Name + ")", nil
|
||||
}
|
||||
|
||||
func (stubFieldMapper) ExistsFor(_ context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
|
||||
if exists {
|
||||
return "has(" + key.Name + ")", nil
|
||||
}
|
||||
return "NOT has(" + key.Name + ")", nil
|
||||
}
|
||||
|
||||
func (stubFieldMapper) ColumnFor(context.Context, valuer.UUID, uint64, uint64, *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (stubFieldMapper) ColumnExpressionFor(context.Context, valuer.UUID, uint64, uint64, *telemetrytypes.TelemetryFieldKey, telemetrytypes.FieldDataType, map[string][]*telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
return "", qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (stubFieldMapper) CandidateKeys(context.Context, valuer.UUID, *telemetrytypes.TelemetryFieldKey, any, map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return nil
|
||||
}
|
||||
|
||||
func stringFamily(names ...string) *telemetrytypes.LogicalField {
|
||||
members := make([]*telemetrytypes.TelemetryFieldKey, 0, len(names))
|
||||
for _, name := range names {
|
||||
members = append(members, &telemetrytypes.TelemetryFieldKey{Name: name, FieldDataType: telemetrytypes.FieldDataTypeString})
|
||||
}
|
||||
return &telemetrytypes.LogicalField{Name: names[0], FieldDataType: telemetrytypes.FieldDataTypeString, Members: members}
|
||||
}
|
||||
|
||||
func TestLogicalValueExprSingleMemberDelegatesToFieldFor(t *testing.T) {
|
||||
logical := telemetrytypes.SingleLogicalField("a", &telemetrytypes.TelemetryFieldKey{Name: "a"})
|
||||
expr, err := LogicalValueExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, logical)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "value(a)", expr)
|
||||
}
|
||||
|
||||
func TestLogicalValueExprStringFamilyMergesCurrentFirst(t *testing.T) {
|
||||
expr, err := LogicalValueExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, stringFamily("current", "old"))
|
||||
require.NoError(t, err)
|
||||
// The trailing '' preserves keyless-row semantics for negative operators.
|
||||
assert.Equal(t, "COALESCE(NULLIF(value(current), ''), NULLIF(value(old), ''), '')", expr)
|
||||
}
|
||||
|
||||
func TestLogicalValueExprNumericFamilyGuardsEveryMember(t *testing.T) {
|
||||
logical := &telemetrytypes.LogicalField{
|
||||
Name: "current",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber,
|
||||
Members: []*telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "current", FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
{Name: "old", FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
},
|
||||
}
|
||||
expr, err := LogicalValueExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, logical)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "multiIf(has(current), value(current), has(old), value(old), NULL)", expr)
|
||||
}
|
||||
|
||||
func TestLogicalExistsExprSingleMemberDelegatesToExistsFor(t *testing.T) {
|
||||
logical := telemetrytypes.SingleLogicalField("a", &telemetrytypes.TelemetryFieldKey{Name: "a"})
|
||||
expr, err := LogicalExistsExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, logical, false)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "NOT has(a)", expr)
|
||||
}
|
||||
|
||||
func TestLogicalExistsExprFamilyIsAnyMemberPresence(t *testing.T) {
|
||||
family := stringFamily("current", "old")
|
||||
|
||||
expr, err := LogicalExistsExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, family, true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "(has(current) OR has(old))", expr)
|
||||
|
||||
expr, err = LogicalExistsExpr(context.Background(), valuer.UUID{}, 0, 0, stubFieldMapper{}, family, false)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "NOT (has(current) OR has(old))", expr)
|
||||
}
|
||||
195
pkg/querybuilder/logical_fields_test.go
Normal file
195
pkg/querybuilder/logical_fields_test.go
Normal file
@@ -0,0 +1,195 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// familiesOn returns a flagger with resolve_semconv_families on.
|
||||
func familiesOn(t *testing.T) flagger.Flagger {
|
||||
return flaggertest.WithBooleanFlags(t, map[string]bool{
|
||||
flagger.FeatureResolveSemconvFamilies.String(): true,
|
||||
})
|
||||
}
|
||||
|
||||
func traceKey(name string, ctx telemetrytypes.FieldContext) *telemetrytypes.TelemetryFieldKey {
|
||||
return &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: ctx,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
}
|
||||
|
||||
func memberNames(logical *telemetrytypes.LogicalField) []string {
|
||||
names := make([]string, 0, len(logical.Members))
|
||||
for _, member := range logical.Members {
|
||||
names = append(names, member.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// The deployment.environment(.name) family (enabled in pkg/semconv) drives the
|
||||
// grouping tests below.
|
||||
|
||||
// With the resolve_semconv_families flag off, matches stay single-member and
|
||||
// selectors stay literal, even when the metadata map has both spellings.
|
||||
func TestFamiliesOffByDefault(t *testing.T) {
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": {traceKey("deployment.environment.name", telemetrytypes.FieldContextResource)},
|
||||
"deployment.environment": {traceKey("deployment.environment", telemetrytypes.FieldContextResource)},
|
||||
}
|
||||
|
||||
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, flaggertest.New(t), &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, fieldKeys)
|
||||
require.Len(t, fields, 1)
|
||||
assert.False(t, fields[0].IsFamily())
|
||||
assert.Equal(t, []string{"deployment.environment.name"}, memberNames(fields[0]))
|
||||
|
||||
selectors := []*telemetrytypes.FieldKeySelector{
|
||||
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
|
||||
}
|
||||
assert.Len(t, ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, flaggertest.New(t), selectors), 1)
|
||||
}
|
||||
|
||||
func TestMatchingLogicalFieldsGroupsFamilyMembers(t *testing.T) {
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": {traceKey("deployment.environment.name", telemetrytypes.FieldContextResource)},
|
||||
"deployment.environment": {traceKey("deployment.environment", telemetrytypes.FieldContextResource)},
|
||||
}
|
||||
|
||||
for _, requested := range []string{"deployment.environment.name", "deployment.environment"} {
|
||||
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{Name: requested}, fieldKeys)
|
||||
require.Len(t, fields, 1, "a family is one logical field, requested via %s", requested)
|
||||
logical := fields[0]
|
||||
assert.Equal(t, requested, logical.Name, "response identity is the requested spelling")
|
||||
assert.Equal(t, telemetrytypes.FieldContextResource, logical.FieldContext)
|
||||
assert.True(t, logical.IsFamily())
|
||||
assert.Equal(t, []string{"deployment.environment.name", "deployment.environment"}, memberNames(logical),
|
||||
"members are current-first regardless of the requested spelling")
|
||||
}
|
||||
}
|
||||
|
||||
// Member precedence is the family's current-first order, not lookup arrival
|
||||
// order: a current-name key found only under its context-prefixed spelling
|
||||
// arrives in the second lookup pass yet must still sort first.
|
||||
func TestMatchingLogicalFieldsOrdersMembersByFamilyRank(t *testing.T) {
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment": {traceKey("deployment.environment", telemetrytypes.FieldContextResource)},
|
||||
"resource.deployment.environment.name": {traceKey("resource.deployment.environment.name", telemetrytypes.FieldContextResource)},
|
||||
}
|
||||
|
||||
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}, fieldKeys)
|
||||
|
||||
require.Len(t, fields, 1)
|
||||
assert.Equal(t, []string{"resource.deployment.environment.name", "deployment.environment"}, memberNames(fields[0]))
|
||||
}
|
||||
|
||||
// Non-trace signals have no family support: the requested spelling stays
|
||||
// literal, and a family member name never pulls in its siblings.
|
||||
func TestMatchingLogicalFieldsKeepsLogsLiteral(t *testing.T) {
|
||||
logsKey := func(name string) *telemetrytypes.TelemetryFieldKey {
|
||||
return &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
}
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": {logsKey("deployment.environment.name")},
|
||||
"deployment.environment": {logsKey("deployment.environment")},
|
||||
}
|
||||
|
||||
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, fieldKeys)
|
||||
require.Len(t, fields, 1)
|
||||
assert.False(t, fields[0].IsFamily())
|
||||
assert.Equal(t, []string{"deployment.environment.name"}, memberNames(fields[0]))
|
||||
}
|
||||
|
||||
// A family and a genuine same-name collision stack cleanly: the family stays
|
||||
// one logical field, the collision adds another, and resource preference keeps
|
||||
// the family as a unit.
|
||||
func TestResolveLogicalFieldsKeepsFamilyThroughAmbiguity(t *testing.T) {
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": {
|
||||
traceKey("deployment.environment.name", telemetrytypes.FieldContextResource),
|
||||
traceKey("deployment.environment.name", telemetrytypes.FieldContextAttribute),
|
||||
},
|
||||
"deployment.environment": {traceKey("deployment.environment", telemetrytypes.FieldContextResource)},
|
||||
}
|
||||
|
||||
requested := &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}
|
||||
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), requested, fieldKeys)
|
||||
require.Len(t, fields, 2, "resource family + attribute collision")
|
||||
|
||||
resolved, warning := ResolveLogicalFields(requested, fields)
|
||||
assert.NotEmpty(t, warning)
|
||||
require.Len(t, resolved, 1)
|
||||
assert.Equal(t, telemetrytypes.FieldContextResource, resolved[0].FieldContext)
|
||||
assert.Equal(t, []string{"deployment.environment.name", "deployment.environment"}, memberNames(resolved[0]))
|
||||
}
|
||||
|
||||
// Members of a family with different data types never merge: the identity
|
||||
// (signal, context, data type) separates them into distinct logical fields.
|
||||
func TestMatchingLogicalFieldsNeverMergesAcrossDataTypes(t *testing.T) {
|
||||
numberKey := traceKey("deployment.environment", telemetrytypes.FieldContextResource)
|
||||
numberKey.FieldDataType = telemetrytypes.FieldDataTypeNumber
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": {traceKey("deployment.environment.name", telemetrytypes.FieldContextResource)},
|
||||
"deployment.environment": {numberKey},
|
||||
}
|
||||
|
||||
fields := MatchingLogicalFields(context.Background(), valuer.UUID{}, familiesOn(t), &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, fieldKeys)
|
||||
require.Len(t, fields, 2)
|
||||
for _, logical := range fields {
|
||||
assert.False(t, logical.IsFamily())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpandKeySelectorsForFamilies(t *testing.T) {
|
||||
selectors := []*telemetrytypes.FieldKeySelector{
|
||||
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
|
||||
{Name: "service.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
|
||||
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalLogs, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
|
||||
}
|
||||
|
||||
expanded := ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, familiesOn(t), selectors)
|
||||
|
||||
names := make([]string, 0, len(expanded))
|
||||
for _, selector := range expanded {
|
||||
names = append(names, selector.Name)
|
||||
}
|
||||
assert.Equal(t, []string{
|
||||
"deployment.environment.name",
|
||||
"service.name",
|
||||
"deployment.environment.name",
|
||||
"deployment.environment",
|
||||
}, names, "one sibling selector for the trace family member; logs and non-family names untouched")
|
||||
|
||||
sibling := expanded[len(expanded)-1]
|
||||
assert.Equal(t, telemetrytypes.SignalTraces, sibling.Signal)
|
||||
assert.Equal(t, telemetrytypes.FieldSelectorMatchTypeExact, sibling.SelectorMatchType)
|
||||
}
|
||||
|
||||
func TestExpandKeySelectorsForFamiliesDeduplicatesAndSkipsFuzzy(t *testing.T) {
|
||||
both := []*telemetrytypes.FieldKeySelector{
|
||||
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
|
||||
{Name: "deployment.environment", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact},
|
||||
}
|
||||
assert.Len(t, ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, familiesOn(t), both), 2, "both spellings already referenced")
|
||||
|
||||
fuzzy := []*telemetrytypes.FieldKeySelector{
|
||||
{Name: "deployment.environment.name", Signal: telemetrytypes.SignalTraces, SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy},
|
||||
}
|
||||
assert.Len(t, ExpandKeySelectorsForFamilies(context.Background(), valuer.UUID{}, familiesOn(t), fuzzy), 1, "fuzzy (search-style) selectors never expand")
|
||||
}
|
||||
@@ -9,7 +9,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -27,6 +29,7 @@ const stringMatchingOperatorDocURL = "https://signoz.io/docs/userguide/operators
|
||||
type filterExpressionVisitor struct {
|
||||
context context.Context
|
||||
orgID valuer.UUID
|
||||
fl flagger.Flagger
|
||||
fieldMapper qbtypes.FieldMapper
|
||||
conditionBuilder qbtypes.ConditionBuilder
|
||||
warnings []string
|
||||
@@ -48,8 +51,11 @@ type filterExpressionVisitor struct {
|
||||
}
|
||||
|
||||
type FilterExprVisitorOpts struct {
|
||||
Context context.Context
|
||||
OrgID valuer.UUID
|
||||
Context context.Context
|
||||
OrgID valuer.UUID
|
||||
// Flagger evaluates the resolve_semconv_families flag during resolution.
|
||||
// A nil Flagger keeps resolution literal.
|
||||
Flagger flagger.Flagger
|
||||
Logger *slog.Logger
|
||||
FieldMapper qbtypes.FieldMapper
|
||||
ConditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -68,6 +74,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
|
||||
return &filterExpressionVisitor{
|
||||
context: opts.Context,
|
||||
orgID: opts.OrgID,
|
||||
fl: opts.Flagger,
|
||||
fieldMapper: opts.FieldMapper,
|
||||
conditionBuilder: opts.ConditionBuilder,
|
||||
fieldKeys: opts.FieldKeys,
|
||||
@@ -360,7 +367,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
}
|
||||
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.TelemetryFieldKey{v.fullTextColumn}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText))
|
||||
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.LogicalField{telemetrytypes.SingleLogicalField(v.fullTextColumn.Name, v.fullTextColumn)}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText))
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -379,7 +386,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
// VisitComparison handles all comparison operators.
|
||||
func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
|
||||
key := v.Visit(ctx.Key()).(*telemetrytypes.TelemetryFieldKey)
|
||||
matching := MatchingFieldKeys(key, v.fieldKeys)
|
||||
matching := MatchingLogicalFields(v.context, v.orgID, v.fl, key, v.fieldKeys)
|
||||
|
||||
// Handle EXISTS specially
|
||||
if ctx.EXISTS() != nil {
|
||||
@@ -675,7 +682,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
|
||||
v.errors = append(v.errors, "full text search is not supported")
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.TelemetryFieldKey{v.fullTextColumn}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text))
|
||||
conds, ok := v.buildConditions(v.fullTextColumn, []*telemetrytypes.LogicalField{telemetrytypes.SingleLogicalField(v.fullTextColumn.Name, v.fullTextColumn)}, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text))
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -730,7 +737,7 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
|
||||
conds, ok := v.buildConditions(key, MatchingFieldKeys(key, v.fieldKeys), operator, value)
|
||||
conds, ok := v.buildConditions(key, MatchingLogicalFields(v.context, v.orgID, v.fl, key, v.fieldKeys), operator, value)
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -922,7 +929,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
|
||||
|
||||
// buildConditions invokes the condition builder for a filter term, folding its
|
||||
// warnings/errors into visitor state; returns false if an error was recorded.
|
||||
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) ([]string, bool) {
|
||||
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.LogicalField, op qbtypes.FilterOperator, value any) ([]string, bool) {
|
||||
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder)
|
||||
if err != nil {
|
||||
_, _, _, _, errURL, _ := errors.Unwrapb(err)
|
||||
@@ -979,30 +986,158 @@ func assignIfEmpty(s *string, value string) {
|
||||
}
|
||||
}
|
||||
|
||||
// MatchingFieldKeys returns the field keys from the map that match the given key,
|
||||
// honoring any context/data type the user specified.
|
||||
func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
fieldKeysForName := []*telemetrytypes.TelemetryFieldKey{}
|
||||
|
||||
// match by name; keep items whose context and data type match (unspecified matches any)
|
||||
for _, item := range fieldKeys[field.Name] {
|
||||
if (field.FieldContext == telemetrytypes.FieldContextUnspecified || field.FieldContext == item.FieldContext) &&
|
||||
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || field.FieldDataType == item.FieldDataType) {
|
||||
fieldKeysForName = append(fieldKeysForName, item)
|
||||
}
|
||||
// familyMemberNames returns the physical spellings to look up for the
|
||||
// referenced key: the semantic-convention family members (current-first) when
|
||||
// the resolve_semconv_families flag is on for the org and the key can resolve
|
||||
// to traces, else just the requested name. Only trace field mappers understand
|
||||
// families today; logs and metrics keep the requested spelling until theirs
|
||||
// land.
|
||||
func familyMemberNames(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, field *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if !semconvFamiliesEnabled(ctx, orgID, fl) {
|
||||
return []string{field.Name}
|
||||
}
|
||||
|
||||
// A context may have been split off a name that legitimately contained it (e.g.
|
||||
// `attribute.key`); also look up the context-prefixed name so both readings resolve.
|
||||
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
contextPrefixedFieldName := fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), field.Name)
|
||||
for _, item := range fieldKeys[contextPrefixedFieldName] {
|
||||
// Context already matched via the lookup key; only data type needs checking.
|
||||
if field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || item.FieldDataType == field.FieldDataType {
|
||||
fieldKeysForName = append(fieldKeysForName, item)
|
||||
}
|
||||
}
|
||||
if field.Signal != telemetrytypes.SignalUnspecified && field.Signal != telemetrytypes.SignalTraces {
|
||||
return []string{field.Name}
|
||||
}
|
||||
|
||||
return fieldKeysForName
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: field.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
// MatchingLogicalFields resolves the referenced key against the metadata map
|
||||
// into logical fields, honoring any context/data type the user specified.
|
||||
//
|
||||
// Physical keys that are members of one semantic-convention family (traces
|
||||
// only today) group into a single logical field per (signal, context, data
|
||||
// type) identity, members ordered current-first. Every other matching key
|
||||
// becomes its own single-member logical field. Ambiguity is therefore the
|
||||
// length of the returned slice, and a family is never ambiguous with itself.
|
||||
// Members alias the metadata map entries; nothing is copied or mutated.
|
||||
//
|
||||
// Family grouping only happens when the resolve_semconv_families flag is on
|
||||
// for the org. A nil flagger means off: every match then stays a
|
||||
// single-member logical field.
|
||||
func MatchingLogicalFields(ctx context.Context, orgID valuer.UUID, fl flagger.Flagger, field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.LogicalField {
|
||||
members := familyMemberNames(ctx, orgID, fl, field)
|
||||
matches := collectMemberMatches(field, members, fieldKeys)
|
||||
return groupIntoLogicalFields(field.Name, len(members) > 1, matches)
|
||||
}
|
||||
|
||||
// memberMatch pairs a metadata entry with the family rank of the member name
|
||||
// it matched under. The stored name of a context-prefixed match differs from
|
||||
// the member name, so the rank must travel with the match.
|
||||
type memberMatch struct {
|
||||
key *telemetrytypes.TelemetryFieldKey
|
||||
rank int
|
||||
}
|
||||
|
||||
// matchesRequestedIdentity reports whether the entry fits the context and data
|
||||
// type that the request specified; unspecified matches any. A context-prefixed
|
||||
// lookup already matched the context through the lookup key itself.
|
||||
func matchesRequestedIdentity(field, item *telemetrytypes.TelemetryFieldKey, contextMatched bool) bool {
|
||||
if !contextMatched && field.FieldContext != telemetrytypes.FieldContextUnspecified && field.FieldContext != item.FieldContext {
|
||||
return false
|
||||
}
|
||||
if field.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && field.FieldDataType != item.FieldDataType {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// inFamilyScope reports whether a match found under a sibling member name is
|
||||
// legitimate: the entry must be trace metadata, and the member must be in the
|
||||
// family of the requested name for the entry's context. A member lookup can
|
||||
// otherwise find a same-named field in a scope where the family does not
|
||||
// apply.
|
||||
func inFamilyScope(field, item *telemetrytypes.TelemetryFieldKey, memberName string) bool {
|
||||
if item.Signal != telemetrytypes.SignalTraces {
|
||||
return false
|
||||
}
|
||||
return slices.Contains(semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: item.FieldContext,
|
||||
}), memberName)
|
||||
}
|
||||
|
||||
// collectMemberMatches finds the metadata entries for every member spelling:
|
||||
// first under the member names, then under their context-prefixed spellings
|
||||
// (a context can be a legitimate part of a stored name, e.g. `attribute.key`).
|
||||
func collectMemberMatches(field *telemetrytypes.TelemetryFieldKey, members []string, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []memberMatch {
|
||||
matches := make([]memberMatch, 0)
|
||||
collect := func(lookupName string, rank int, memberName string, contextMatched bool) {
|
||||
for _, item := range fieldKeys[lookupName] {
|
||||
if !matchesRequestedIdentity(field, item, contextMatched) {
|
||||
continue
|
||||
}
|
||||
if memberName != field.Name && !inFamilyScope(field, item, memberName) {
|
||||
continue
|
||||
}
|
||||
matches = append(matches, memberMatch{key: item, rank: rank})
|
||||
}
|
||||
}
|
||||
|
||||
for rank, member := range members {
|
||||
collect(member, rank, member, false)
|
||||
}
|
||||
if field.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
for rank, member := range members {
|
||||
collect(fmt.Sprintf("%s.%s", field.FieldContext.StringValue(), member), rank, member, true)
|
||||
}
|
||||
}
|
||||
return matches
|
||||
}
|
||||
|
||||
// groupIntoLogicalFields turns matches into logical fields. Trace entries in
|
||||
// family mode group by their (signal, context, data type) identity; every
|
||||
// other entry becomes its own single-member field. Members sort by family
|
||||
// rank at the end: precedence is a property of the family, not of the order
|
||||
// in which the lookups found the members.
|
||||
func groupIntoLogicalFields(requestedName string, familyMode bool, matches []memberMatch) []*telemetrytypes.LogicalField {
|
||||
fields := make([]*telemetrytypes.LogicalField, 0, len(matches))
|
||||
groups := make(map[string]*telemetrytypes.LogicalField)
|
||||
ranks := make(map[*telemetrytypes.TelemetryFieldKey]int)
|
||||
|
||||
for _, match := range matches {
|
||||
if !familyMode || match.key.Signal != telemetrytypes.SignalTraces {
|
||||
fields = append(fields, telemetrytypes.SingleLogicalField(requestedName, match.key))
|
||||
continue
|
||||
}
|
||||
|
||||
identity := match.key.Signal.StringValue() + ";" + match.key.FieldContext.StringValue() + ";" + match.key.FieldDataType.StringValue()
|
||||
group, ok := groups[identity]
|
||||
if !ok {
|
||||
group = &telemetrytypes.LogicalField{
|
||||
Name: requestedName,
|
||||
Signal: match.key.Signal,
|
||||
FieldContext: match.key.FieldContext,
|
||||
FieldDataType: match.key.FieldDataType,
|
||||
}
|
||||
groups[identity] = group
|
||||
fields = append(fields, group)
|
||||
}
|
||||
if groupHasMemberNamed(group, match.key.Name) {
|
||||
continue
|
||||
}
|
||||
ranks[match.key] = match.rank
|
||||
group.Members = append(group.Members, match.key)
|
||||
}
|
||||
|
||||
for _, logical := range fields {
|
||||
slices.SortStableFunc(logical.Members, func(a, b *telemetrytypes.TelemetryFieldKey) int {
|
||||
return ranks[a] - ranks[b]
|
||||
})
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func groupHasMemberNamed(group *telemetrytypes.LogicalField, name string) bool {
|
||||
for _, member := range group.Members {
|
||||
if member.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -588,9 +588,11 @@ func TestVisitKey(t *testing.T) {
|
||||
|
||||
// VisitKey only parses; the condition builder matches, resolves ambiguity
|
||||
// and decides not-found handling. Replay that here against the generic
|
||||
// builder behavior (error unless the key is ignored).
|
||||
matching := MatchingFieldKeys(key, tt.fieldKeys)
|
||||
keys, warning := ResolveKeys(key, matching)
|
||||
// builder behavior (error unless the key is ignored). The test maps carry
|
||||
// no signal, so every logical field is single-member and flattens losslessly.
|
||||
matching := MatchingLogicalFields(context.Background(), valuer.UUID{}, nil, key, tt.fieldKeys)
|
||||
resolved, warning := ResolveLogicalFields(key, matching)
|
||||
keys := SingleKeys(resolved)
|
||||
|
||||
var gotErrors []string
|
||||
var gotMainErrURL, gotMainWrnURL string
|
||||
@@ -766,7 +768,8 @@ func (b *resourceConditionBuilder) ConditionFor(
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
keys, warning := ResolveKeys(key, MatchingFieldKeys(key, fieldKeys))
|
||||
resolved, warning := ResolveLogicalFields(key, MatchingLogicalFields(context.Background(), valuer.UUID{}, nil, key, fieldKeys))
|
||||
keys := SingleKeys(resolved)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
@@ -808,7 +811,8 @@ func (b *conditionBuilder) ConditionFor(
|
||||
return []string{fmt.Sprintf("%s_cond", key.Name)}, nil, nil
|
||||
}
|
||||
|
||||
keys, warning := ResolveKeys(key, MatchingFieldKeys(key, fieldKeys))
|
||||
resolved, warning := ResolveLogicalFields(key, MatchingLogicalFields(context.Background(), valuer.UUID{}, nil, key, fieldKeys))
|
||||
keys := SingleKeys(resolved)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
|
||||
@@ -469,8 +469,6 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
// The `[*]` path is extracted per value, not as an Array(String) compared to a
|
||||
// scalar — ClickHouse rejects that outright (code 130).
|
||||
name: "IN operator with json search",
|
||||
requestType: qbtypes.RequestTypeRaw,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -481,7 +479,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
Limit: 10,
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSON_VALUE(body, '$.\"user_names\"[*]') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((JSONExtract(JSON_QUERY(body, '$.\"user_names\"[*]'), 'Array(String)') = ?) AND JSON_EXISTS(body, '$.\"user_names\"[*]')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"john_doe", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
|
||||
Warnings: []string{querybuilder.NewKeyNotFoundWarning("user_names[*]")},
|
||||
},
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -14,12 +15,15 @@ import (
|
||||
|
||||
type defaultConditionBuilder struct {
|
||||
fm qbtypes.FieldMapper
|
||||
// fl evaluates the resolve_semconv_families flag during resolution.
|
||||
// A nil flagger keeps resolution literal.
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.ConditionBuilder = (*defaultConditionBuilder)(nil)
|
||||
|
||||
func NewConditionBuilder(fm qbtypes.FieldMapper) *defaultConditionBuilder {
|
||||
return &defaultConditionBuilder{fm: fm}
|
||||
func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *defaultConditionBuilder {
|
||||
return &defaultConditionBuilder{fm: fm, fl: fl}
|
||||
}
|
||||
|
||||
func valueForIndexFilter(op qbtypes.FilterOperator, key *telemetrytypes.TelemetryFieldKey, value any) any {
|
||||
@@ -44,10 +48,74 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
|
||||
return fmt.Sprintf(`%%%s%%`, key.Name)
|
||||
}
|
||||
|
||||
// The three helpers below take the members of one logical field. With a single
|
||||
// member they render exactly the pre-family shapes; a family widens key/value
|
||||
// index hints to any-member and presence to any-member (all-absent when negated).
|
||||
|
||||
func keyIndexCondition(sb *sqlbuilder.SelectBuilder, column string, members []*telemetrytypes.TelemetryFieldKey) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
conditions = append(conditions, sb.Like(column, keyIndexFilter(member)))
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
|
||||
func valueIndexCondition(
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
column string,
|
||||
members []*telemetrytypes.TelemetryFieldKey,
|
||||
op qbtypes.FilterOperator,
|
||||
value any,
|
||||
caseInsensitive bool,
|
||||
) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
patterns := valueForIndexFilter(op, member, value)
|
||||
switch values := patterns.(type) {
|
||||
case []string:
|
||||
for _, pattern := range values {
|
||||
conditions = append(conditions, sb.Like(column, pattern))
|
||||
}
|
||||
default:
|
||||
if caseInsensitive {
|
||||
conditions = append(conditions, sb.ILike(column, values))
|
||||
} else {
|
||||
conditions = append(conditions, sb.Like(column, values))
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
|
||||
func memberPresenceCondition(sb *sqlbuilder.SelectBuilder, column string, members []*telemetrytypes.TelemetryFieldKey, exists bool) string {
|
||||
conditions := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
field := fmt.Sprintf("simpleJSONHas(%s, '%s')", column, member.Name)
|
||||
if exists {
|
||||
conditions = append(conditions, sb.E(field, true))
|
||||
} else {
|
||||
conditions = append(conditions, sb.NE(field, true))
|
||||
}
|
||||
}
|
||||
if exists {
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0]
|
||||
}
|
||||
return sb.Or(conditions...)
|
||||
}
|
||||
return sb.And(conditions...)
|
||||
}
|
||||
|
||||
// SkipResourceFilter is not applicable here: the fingerprint table only stores resource attributes.
|
||||
func (b *defaultConditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
_ valuer.UUID,
|
||||
orgID valuer.UUID,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
@@ -57,7 +125,7 @@ func (b *defaultConditionBuilder) ConditionFor(
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) ([]string, []string, error) {
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
matches := querybuilder.MatchingLogicalFields(ctx, orgID, b.fl, key, fieldKeys)
|
||||
|
||||
// has/hasAny/hasAll/hasToken are logs-body-only functions; they never apply to the
|
||||
// resource fingerprint table, so skip them (the main query still evaluates them).
|
||||
@@ -65,21 +133,21 @@ func (b *defaultConditionBuilder) ConditionFor(
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, matches)
|
||||
logicalFields, warning := querybuilder.ResolveLogicalFields(key, matches)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
}
|
||||
|
||||
conds := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
// the resource fingerprint table only stores resource attributes; keys from
|
||||
conds := make([]string, 0, len(logicalFields))
|
||||
for _, logical := range logicalFields {
|
||||
// the resource fingerprint table only stores resource attributes; fields from
|
||||
// any other context contribute no condition and are omitted. An empty result
|
||||
// (including an unknown key) lets the caller skip this filter entirely.
|
||||
if k.FieldContext != telemetrytypes.FieldContextResource {
|
||||
if logical.FieldContext != telemetrytypes.FieldContextResource {
|
||||
continue
|
||||
}
|
||||
cond, err := b.conditionForKey(ctx, startNs, endNs, k, op, value, sb)
|
||||
cond, err := b.conditionForLogicalField(ctx, orgID, startNs, endNs, logical, op, value, sb)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -88,11 +156,12 @@ func (b *defaultConditionBuilder) ConditionFor(
|
||||
return conds, warnings, nil
|
||||
}
|
||||
|
||||
func (b *defaultConditionBuilder) conditionForKey(
|
||||
func (b *defaultConditionBuilder) conditionForLogicalField(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
logical *telemetrytypes.LogicalField,
|
||||
op qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
@@ -102,7 +171,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
// as we store resource values as string
|
||||
formattedValue := querybuilder.FormatValueForContains(value)
|
||||
|
||||
columns, err := b.fm.ColumnFor(ctx, valuer.UUID{}, startNs, endNs, key)
|
||||
columns, err := b.fm.ColumnFor(ctx, orgID, startNs, endNs, logical.Single())
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -115,10 +184,12 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
// as we have not changed the resource column in the resource fingerprint table.
|
||||
column := columns[0]
|
||||
|
||||
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
|
||||
valueForIndexFilter := valueForIndexFilter(op, key, value)
|
||||
members := logical.Members
|
||||
isFamily := logical.IsFamily()
|
||||
keyIdxFilter := keyIndexCondition(sb, column.Name, members)
|
||||
singleValueIndexFilter := valueForIndexFilter(op, members[0], value)
|
||||
|
||||
fieldName, err := b.fm.FieldFor(ctx, valuer.UUID{}, startNs, endNs, key)
|
||||
fieldName, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, b.fm, logical)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -128,12 +199,17 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.E(fieldName, formattedValue),
|
||||
keyIdxFilter,
|
||||
sb.Like(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, members, op, value, false),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
if isFamily {
|
||||
// A negated value-index hint would drop rows where another member
|
||||
// holds the value; the fingerprint scan is small enough without it.
|
||||
return sb.NE(fieldName, formattedValue), nil
|
||||
}
|
||||
return sb.And(
|
||||
sb.NE(fieldName, formattedValue),
|
||||
sb.NotLike(column.Name, valueForIndexFilter),
|
||||
sb.NotLike(column.Name, singleValueIndexFilter),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.And(sb.GT(fieldName, formattedValue), keyIdxFilter), nil
|
||||
@@ -148,7 +224,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.ILike(fieldName, formattedValue),
|
||||
keyIdxFilter,
|
||||
sb.ILike(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, members, op, value, true),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotLike, qbtypes.FilterOperatorNotILike:
|
||||
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
|
||||
@@ -180,16 +256,18 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
// each value carries its own index filter, since `=` derives one from the value
|
||||
inConditions := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
cond, err := b.conditionForKey(ctx, startNs, endNs, key, qbtypes.FilterOperatorEqual, v, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
inConditions = append(inConditions, cond)
|
||||
inConditions = append(inConditions, sb.E(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
return sb.Or(inConditions...), nil
|
||||
mainCondition := sb.Or(inConditions...)
|
||||
mainCondition = sb.And(
|
||||
mainCondition,
|
||||
keyIdxFilter,
|
||||
valueIndexCondition(sb, column.Name, members, op, value, false),
|
||||
)
|
||||
|
||||
return mainCondition, nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -197,23 +275,30 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
}
|
||||
notInConditions := make([]string, 0, len(values))
|
||||
for _, v := range values {
|
||||
cond, err := b.conditionForKey(ctx, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, v, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
notInConditions = append(notInConditions, cond)
|
||||
notInConditions = append(notInConditions, sb.NE(fieldName, querybuilder.FormatValueForContains(v)))
|
||||
}
|
||||
return sb.And(notInConditions...), nil
|
||||
mainCondition := sb.And(notInConditions...)
|
||||
if isFamily {
|
||||
// A negated value-index hint would drop rows where another member
|
||||
// holds the value; the fingerprint scan is small enough without it.
|
||||
return mainCondition, nil
|
||||
}
|
||||
valConditions := make([]string, 0, len(values))
|
||||
if valuesForIndexFilter, ok := singleValueIndexFilter.([]string); ok {
|
||||
for _, v := range valuesForIndexFilter {
|
||||
valConditions = append(valConditions, sb.NotLike(column.Name, v))
|
||||
}
|
||||
}
|
||||
mainCondition = sb.And(mainCondition, sb.And(valConditions...))
|
||||
return mainCondition, nil
|
||||
|
||||
case qbtypes.FilterOperatorExists:
|
||||
return sb.And(
|
||||
sb.E(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
|
||||
memberPresenceCondition(sb, column.Name, members, true),
|
||||
keyIdxFilter,
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotExists:
|
||||
return sb.And(
|
||||
sb.NE(fmt.Sprintf("simpleJSONHas(%s, '%s')", column.Name, key.Name), true),
|
||||
), nil
|
||||
return memberPresenceCondition(sb, column.Name, members, false), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
return sb.And(
|
||||
@@ -229,7 +314,7 @@ func (b *defaultConditionBuilder) conditionForKey(
|
||||
return sb.And(
|
||||
sb.ILike(fieldName, fmt.Sprintf(`%%%s%%`, formattedValue)),
|
||||
keyIdxFilter,
|
||||
sb.ILike(column.Name, valueForIndexFilter),
|
||||
valueIndexCondition(sb, column.Name, members, op, value, true),
|
||||
), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
// no index filter: as cannot apply `not contains x%y` as y can be somewhere else
|
||||
|
||||
@@ -2,6 +2,7 @@ package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"testing"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -109,8 +110,8 @@ func TestConditionBuilder(t *testing.T) {
|
||||
},
|
||||
op: qbtypes.FilterOperatorIn,
|
||||
value: []any{"watch", "redis"},
|
||||
expected: "((simpleJSONExtractString(labels, 'k8s.namespace.name') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'k8s.namespace.name') = ? AND labels LIKE ? AND labels LIKE ?))",
|
||||
expectedArgs: []any{"watch", "%k8s.namespace.name%", "%k8s.namespace.name\":\"watch%", "redis", "%k8s.namespace.name%", "%k8s.namespace.name\":\"redis%"},
|
||||
expected: "(simpleJSONExtractString(labels, 'k8s.namespace.name') = ? OR simpleJSONExtractString(labels, 'k8s.namespace.name') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)",
|
||||
expectedArgs: []any{"watch", "redis", "%k8s.namespace.name%", "%k8s.namespace.name\":\"watch%", "%k8s.namespace.name\":\"redis%"},
|
||||
},
|
||||
{
|
||||
name: "string_not_in",
|
||||
@@ -120,8 +121,8 @@ func TestConditionBuilder(t *testing.T) {
|
||||
},
|
||||
op: qbtypes.FilterOperatorNotIn,
|
||||
value: []any{"watch", "redis"},
|
||||
expected: "((simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND labels NOT LIKE ?) AND (simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND labels NOT LIKE ?))",
|
||||
expectedArgs: []any{"watch", "%k8s.namespace.name\":\"watch%", "redis", "%k8s.namespace.name\":\"redis%"},
|
||||
expected: "(simpleJSONExtractString(labels, 'k8s.namespace.name') <> ? AND simpleJSONExtractString(labels, 'k8s.namespace.name') <> ?) AND (labels NOT LIKE ? AND labels NOT LIKE ?)",
|
||||
expectedArgs: []any{"watch", "redis", "%k8s.namespace.name\":\"watch%", "%k8s.namespace.name\":\"redis%"},
|
||||
},
|
||||
{
|
||||
name: "string_exists",
|
||||
@@ -173,8 +174,8 @@ func TestConditionBuilder(t *testing.T) {
|
||||
},
|
||||
op: qbtypes.FilterOperatorIn,
|
||||
value: []any{1, 2},
|
||||
expected: "((simpleJSONExtractString(labels, 'test_num') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'test_num') = ? AND labels LIKE ? AND labels LIKE ?))",
|
||||
expectedArgs: []any{"1", "%test_num%", "%test_num\":\"1%", "2", "%test_num%", "%test_num\":\"2%"},
|
||||
expected: "(simpleJSONExtractString(labels, 'test_num') = ? OR simpleJSONExtractString(labels, 'test_num') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)",
|
||||
expectedArgs: []any{"1", "2", "%test_num%", "%test_num\":\"1%", "%test_num\":\"2%"},
|
||||
},
|
||||
{
|
||||
name: "number_between",
|
||||
@@ -201,7 +202,7 @@ func TestConditionBuilder(t *testing.T) {
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
|
||||
92
pkg/statementbuilder/resourcefilter/family_test.go
Normal file
92
pkg/statementbuilder/resourcefilter/family_test.go
Normal file
@@ -0,0 +1,92 @@
|
||||
package resourcefilter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func familyFieldKeys() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
newKey := func(name string) *telemetrytypes.TelemetryFieldKey {
|
||||
return &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
}
|
||||
return map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": {newKey("deployment.environment.name")},
|
||||
"deployment.environment": {newKey("deployment.environment")},
|
||||
}
|
||||
}
|
||||
|
||||
func familyConditionSQL(t *testing.T, op qbtypes.FilterOperator, value any) (string, []any) {
|
||||
t.Helper()
|
||||
cb := NewConditionBuilder(NewFieldMapper(), flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureResolveSemconvFamilies.String(): true}))
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
|
||||
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
|
||||
familyFieldKeys(), qbtypes.ConditionBuilderOptions{}, op, value, sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
sb.Where(conds...)
|
||||
return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
}
|
||||
|
||||
const familyValueExpr = "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '')"
|
||||
|
||||
func TestFamilyEqualWidensIndexHintsToAnyMember(t *testing.T) {
|
||||
sql, args := familyConditionSQL(t, qbtypes.FilterOperatorEqual, "production")
|
||||
assert.Contains(t, sql, familyValueExpr+" = ?")
|
||||
// key hint: either member name may appear in the labels JSON
|
||||
assert.Contains(t, sql, "(labels LIKE ? OR labels LIKE ?)")
|
||||
assert.Contains(t, args, "%deployment.environment.name%")
|
||||
assert.Contains(t, args, "%deployment.environment%")
|
||||
assert.Contains(t, args, `%deployment.environment.name":"production%`)
|
||||
assert.Contains(t, args, `%deployment.environment":"production%`)
|
||||
}
|
||||
|
||||
func TestFamilyNotEqualDropsNegatedValueHint(t *testing.T) {
|
||||
sql, args := familyConditionSQL(t, qbtypes.FilterOperatorNotEqual, "production")
|
||||
assert.Contains(t, sql, familyValueExpr+" <> ?")
|
||||
// A negated per-member value hint would drop rows where the other member
|
||||
// holds the value, so the family form carries no index hints at all.
|
||||
assert.NotContains(t, sql, "NOT LIKE")
|
||||
assert.Equal(t, []any{"production"}, args)
|
||||
}
|
||||
|
||||
func TestFamilyExistsIsAnyMemberPresence(t *testing.T) {
|
||||
sql, _ := familyConditionSQL(t, qbtypes.FilterOperatorExists, nil)
|
||||
assert.Contains(t, sql, "(simpleJSONHas(labels, 'deployment.environment.name') = ? OR simpleJSONHas(labels, 'deployment.environment') = ?)")
|
||||
|
||||
sql, _ = familyConditionSQL(t, qbtypes.FilterOperatorNotExists, nil)
|
||||
assert.Contains(t, sql, "(simpleJSONHas(labels, 'deployment.environment.name') <> ? AND simpleJSONHas(labels, 'deployment.environment') <> ?)")
|
||||
}
|
||||
|
||||
// With only one member in metadata the SQL keeps the exact pre-family shape,
|
||||
// including the negated value hint on !=.
|
||||
func TestSingleMemberShapesUnchanged(t *testing.T) {
|
||||
cb := NewConditionBuilder(NewFieldMapper(), flaggertest.WithBooleanFlags(t, map[string]bool{flagger.FeatureResolveSemconvFamilies.String(): true}))
|
||||
soloKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"deployment.environment.name": familyFieldKeys()["deployment.environment.name"],
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, 0, 0,
|
||||
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"},
|
||||
soloKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotEqual, "production", sb)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conds...)
|
||||
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, "simpleJSONExtractString(labels, 'deployment.environment.name') <> ?")
|
||||
assert.Contains(t, sql, "labels NOT LIKE ?")
|
||||
assert.NotContains(t, sql, "COALESCE")
|
||||
}
|
||||
@@ -71,6 +71,33 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
// ExistsFor reports key presence in the fingerprint labels JSON. Only resource
|
||||
// context keys have a presence notion here; anything else is a real column and
|
||||
// always present.
|
||||
func (m *defaultFieldMapper) ExistsFor(
|
||||
ctx context.Context,
|
||||
_ valuer.UUID,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
exists bool,
|
||||
) (string, error) {
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
if exists {
|
||||
return "true", nil
|
||||
}
|
||||
return "false", nil
|
||||
}
|
||||
pred := fmt.Sprintf("simpleJSONHas(%s, '%s')", columns[0].Name, key.Name)
|
||||
if exists {
|
||||
return pred, nil
|
||||
}
|
||||
return "NOT " + pred, nil
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
|
||||
@@ -47,7 +47,7 @@ func New[T any](
|
||||
) *resourceFilterStatementBuilder[T] {
|
||||
set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/statementbuilder/resourcefilter")
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
return &resourceFilterStatementBuilder[T]{
|
||||
logger: set.Logger(),
|
||||
dbName: dbName,
|
||||
@@ -99,7 +99,7 @@ func (b *resourceFilterStatementBuilder[T]) Build(
|
||||
q.Select("fingerprint")
|
||||
q.From(fmt.Sprintf("%s.%s", b.dbName, b.tableName))
|
||||
|
||||
keySelectors := b.getKeySelectors(query)
|
||||
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.flagger, b.getKeySelectors(query))
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -164,6 +164,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
filterWhereClause, err := querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
OrgID: orgID,
|
||||
Flagger: b.flagger,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fieldMapper,
|
||||
ConditionBuilder: b.conditionBuilder,
|
||||
|
||||
@@ -229,8 +229,8 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
|
||||
start: testStartNs,
|
||||
end: testEndNs,
|
||||
expected: &qbtypes.Statement{
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "%service.name%", "%service.name\":\"redis%", "postgres", "%service.name%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? OR simpleJSONExtractString(labels, 'service.name') = ?) AND labels LIKE ? AND (labels LIKE ? OR labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "postgres", "%service.name%", "%service.name\":\"redis%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -244,8 +244,8 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
|
||||
start: testStartNs,
|
||||
end: testEndNs,
|
||||
expected: &qbtypes.Statement{
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') <> ? AND labels NOT LIKE ?) AND (simpleJSONExtractString(labels, 'service.name') <> ? AND labels NOT LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "%service.name\":\"redis%", "postgres", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
Query: "SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE ((simpleJSONExtractString(labels, 'service.name') <> ? AND simpleJSONExtractString(labels, 'service.name') <> ?) AND (labels NOT LIKE ? AND labels NOT LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint",
|
||||
Args: []any{"redis", "postgres", "%service.name\":\"redis%", "%service.name\":\"postgres%", expectedBucketStart, expectedBucketEnd},
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -38,6 +38,7 @@ type scopedTraceStatementBuilder struct {
|
||||
scope TraceScope
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.TraceAggregation] = (*scopedTraceStatementBuilder)(nil)
|
||||
@@ -59,8 +60,8 @@ func NewFactory(
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
return NewScopedTraceStatementBuilder(settings, metadataStore, fm, cb, scope, traceStmtBuilder, fl), nil
|
||||
},
|
||||
)
|
||||
@@ -98,6 +99,7 @@ func NewScopedTraceStatementBuilder(
|
||||
scope: scope,
|
||||
traceStmtBuilder: traceStmtBuilder,
|
||||
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
|
||||
fl: fl,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -266,7 +268,7 @@ func (b *scopedTraceStatementBuilder) fetchKeys(ctx context.Context, orgID value
|
||||
SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
|
||||
})
|
||||
}
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.fl, selectors))
|
||||
return keys, err
|
||||
}
|
||||
|
||||
@@ -442,13 +444,14 @@ func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context,
|
||||
for i := range selectors {
|
||||
selectors[i].Signal = telemetrytypes.SignalTraces
|
||||
}
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, selectors)
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.fl, selectors))
|
||||
if err != nil {
|
||||
return "", nil, "", err
|
||||
}
|
||||
prepared, err := querybuilder.PrepareWhereClause(expr, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
OrgID: orgID,
|
||||
Flagger: b.fl,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
|
||||
@@ -31,6 +31,7 @@ type traceQueryStatementBuilder struct {
|
||||
cb qbtypes.ConditionBuilder
|
||||
resourceFilterResolver *resourcefilter.ResourceFingerprintResolver[qbtypes.TraceAggregation]
|
||||
aggExprRewriter qbtypes.AggExprRewriter
|
||||
fl flagger.Flagger
|
||||
skipResourceFingerprintEnabled bool
|
||||
}
|
||||
|
||||
@@ -47,8 +48,8 @@ func NewFactory(
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("traces"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.StatementBuilder[qbtypes.TraceAggregation], error) {
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
|
||||
return NewTraceQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
|
||||
@@ -91,6 +92,7 @@ func NewTraceQueryStatementBuilder(
|
||||
cb: conditionBuilder,
|
||||
resourceFilterResolver: resourceFilterResolver,
|
||||
aggExprRewriter: aggExprRewriter,
|
||||
fl: flagger,
|
||||
skipResourceFingerprintEnabled: skipResourceFingerprintEnable,
|
||||
}
|
||||
}
|
||||
@@ -119,7 +121,7 @@ func (b *traceQueryStatementBuilder) Build(
|
||||
|
||||
// We modify SelectFields above (injecting default fields), and those default
|
||||
// fields can carry keys that need evolutions, so fetch keys after that.
|
||||
keySelectors := getKeySelectors(query)
|
||||
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, b.fl, getKeySelectors(query))
|
||||
|
||||
keys, _, err := b.metadataStore.GetKeysMulti(ctx, orgID, keySelectors)
|
||||
if err != nil {
|
||||
@@ -795,6 +797,7 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
OrgID: orgID,
|
||||
Flagger: b.fl,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
cmock "github.com/SigNoz/clickhouse-go-mock"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -376,8 +377,8 @@ func TestStatementBuilder(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
@@ -677,8 +678,8 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
@@ -804,8 +805,8 @@ func TestStatementBuilderListQueryWithCorruptData(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = c.keysMap
|
||||
if mockMetadataStore.KeysMap == nil {
|
||||
@@ -879,8 +880,8 @@ func TestStatementBuilderGroupByResourceEvolution(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
@@ -1046,8 +1047,8 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
@@ -1684,8 +1685,8 @@ func newSkipResourceFingerprintBuilder(
|
||||
t.Helper()
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
@@ -1714,8 +1715,8 @@ func TestStatementBuilderGroupByUnseenKey(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
@@ -1756,8 +1757,8 @@ func TestStatementBuilderAggregationUnseenKey(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
@@ -1787,3 +1788,72 @@ func TestStatementBuilderAggregationUnseenKey(t *testing.T) {
|
||||
assert.Contains(t, q.Query, "attributes_number['error.type']")
|
||||
assert.Contains(t, q.Query, "attributes_bool['error.type']")
|
||||
}
|
||||
|
||||
// TestStatementBuilderSemconvFamilies builds the same family-member filter
|
||||
// with the resolve_semconv_families flag on and off. On: the resource filter
|
||||
// merges both spellings and widens the index hints to any member. Off: the
|
||||
// query uses only the requested spelling, so users see no change.
|
||||
func TestStatementBuilderSemconvFamilies(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
query := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.TraceAggregation{{Expression: "count()"}},
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: "deployment.environment.name = 'production'",
|
||||
},
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
flag bool
|
||||
expected qbtypes.Statement
|
||||
}{
|
||||
{
|
||||
name: "flag on merges both spellings",
|
||||
flag: true,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') = ? AND (labels LIKE ? OR labels LIKE ?) AND (labels LIKE ? OR labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC",
|
||||
Args: []any{"production", "%deployment.environment.name%", "%deployment.environment%", "%deployment.environment.name\":\"production%", "%deployment.environment\":\"production%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "flag off keeps the literal spelling",
|
||||
flag: false,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (simpleJSONExtractString(labels, 'deployment.environment.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ? GROUP BY fingerprint) SELECT count() AS __result_0 FROM signoz_traces.distributed_signoz_index_v3 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND timestamp >= ? AND timestamp < ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC",
|
||||
Args: []any{"production", "%deployment.environment.name%", "%deployment.environment.name\":\"production%", uint64(1747945619), uint64(1747983448), "1747947419000000000", "1747983448000000000", uint64(1747945619), uint64(1747983448)},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
fl := flaggertest.WithBooleanFlags(t, map[string]bool{
|
||||
flagger.FeatureResolveSemconvFamilies.String(): c.flag,
|
||||
})
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
instrumentationtest.New().ToProviderSettings(),
|
||||
mockMetadataStore,
|
||||
fm,
|
||||
cb,
|
||||
aggExprRewriter,
|
||||
nil,
|
||||
fl,
|
||||
false,
|
||||
100000,
|
||||
)
|
||||
|
||||
q, err := statementBuilder.Build(context.Background(), valuer.UUID{}, 1747947419000, 1747983448000, qbtypes.RequestTypeScalar, query, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.expected.Query, q.Query)
|
||||
require.Equal(t, c.expected.Args, q.Args)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,7 +212,7 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
|
||||
return cteName, nil
|
||||
}
|
||||
|
||||
keySelectors := getKeySelectors(*query)
|
||||
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, b.orgID, b.stmtBuilder.fl, getKeySelectors(*query))
|
||||
b.stmtBuilder.logger.DebugContext(ctx, "Key selectors for query", slog.String("query_name", queryName), slog.Any("key_selectors", keySelectors))
|
||||
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, b.orgID, keySelectors)
|
||||
if err != nil {
|
||||
@@ -265,6 +265,7 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
|
||||
querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
OrgID: b.orgID,
|
||||
Flagger: b.stmtBuilder.fl,
|
||||
Logger: b.stmtBuilder.logger,
|
||||
FieldMapper: b.stmtBuilder.fm,
|
||||
ConditionBuilder: b.stmtBuilder.cb,
|
||||
@@ -442,7 +443,7 @@ func (b *traceOperatorCTEBuilder) buildFinalQuery(ctx context.Context, selectFro
|
||||
}
|
||||
}
|
||||
|
||||
keySelectors := b.getKeySelectors()
|
||||
keySelectors := querybuilder.ExpandKeySelectorsForFamilies(ctx, b.orgID, b.stmtBuilder.fl, b.getKeySelectors())
|
||||
keys, _, err := b.stmtBuilder.metadataStore.GetKeysMulti(ctx, b.orgID, keySelectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -20,8 +20,8 @@ func newTestTraceOperatorStatementBuilder(t *testing.T) *traceOperatorStatementB
|
||||
t.Helper()
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
@@ -25,6 +25,7 @@ type traceOperatorStatementBuilder struct {
|
||||
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
|
||||
aggExprRewriter qbtypes.AggExprRewriter
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.TraceOperatorStatementBuilder = (*traceOperatorStatementBuilder)(nil)
|
||||
@@ -41,8 +42,8 @@ func NewOperatorFactory(
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName("traceoperator"),
|
||||
func(_ context.Context, settings factory.ProviderSettings, cfg statementbuilder.Config) (qbtypes.TraceOperatorStatementBuilder, error) {
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, fm, cb, fl)
|
||||
traceStmtBuilder := NewTraceQueryStatementBuilder(
|
||||
settings, metadataStore, fm, cb, aggExprRewriter, telemetryStore, fl,
|
||||
@@ -85,6 +86,7 @@ func NewTraceOperatorStatementBuilder(
|
||||
traceStmtBuilder: traceStmtBuilder,
|
||||
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
|
||||
aggExprRewriter: aggExprRewriter,
|
||||
fl: flagger,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,8 +21,9 @@ import (
|
||||
func TestTraceTimeRangeOptimization(t *testing.T) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
|
||||
fm := tracestelemetryschema.NewFieldMapper()
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm)
|
||||
fl := flaggertest.New(t)
|
||||
fm := tracestelemetryschema.NewFieldMapper(fl)
|
||||
cb := tracestelemetryschema.NewConditionBuilder(fm, fl)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
|
||||
mockMetadataStore.KeysMap = tracestelemetryschema.BuildCompleteFieldKeyMap(releaseTime)
|
||||
@@ -39,7 +40,6 @@ func TestTraceTimeRangeOptimization(t *testing.T) {
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
}}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, fl)
|
||||
|
||||
statementBuilder := NewTraceQueryStatementBuilder(
|
||||
|
||||
@@ -38,8 +38,11 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// an unknown key simply yields no condition rather than an error.
|
||||
keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys))
|
||||
// an unknown key simply yields no condition rather than an error. Metadata
|
||||
// fields have no family support, so every logical field is single-member
|
||||
// and flattens losslessly to its physical key.
|
||||
resolved, warning := querybuilder.ResolveLogicalFields(key, querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
|
||||
keys := querybuilder.SingleKeys(resolved)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
|
||||
@@ -58,6 +58,19 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsE
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
|
||||
func (m *fieldMapper) ExistsFor(ctx context.Context, _ valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
pred := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
|
||||
if exists {
|
||||
return pred, nil
|
||||
}
|
||||
return "NOT " + pred, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
|
||||
@@ -94,11 +94,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -108,11 +104,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
@@ -147,7 +139,10 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, querybuilder.MatchingFieldKeys(key, fieldKeys))
|
||||
// Audit fields have no family support, so every logical field is
|
||||
// single-member and flattens losslessly to its physical key.
|
||||
resolved, warning := querybuilder.ResolveLogicalFields(key, querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
|
||||
keys := querybuilder.SingleKeys(resolved)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
|
||||
@@ -97,6 +97,19 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
|
||||
return m.getColumn(ctx, key)
|
||||
}
|
||||
|
||||
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
|
||||
func (m *fieldMapper) ExistsFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
|
||||
fieldExpression, err := m.FieldFor(ctx, orgID, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
columns, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return querybuilder.ExistsExpression(columns, key, tsStart, tsEnd, fieldExpression, exists)
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
|
||||
@@ -410,11 +410,7 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
cond, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -425,11 +421,7 @@ func (c *conditionBuilder) conditionForResolvedKey(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
cond, err := c.conditionForResolvedKey(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
@@ -460,7 +452,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) ([]string, []string, error) {
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
matches := querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys)
|
||||
skipResourceFilter := options.SkipResourceFilter
|
||||
|
||||
// search() resolves its own (optional) scope; handle it before key resolution.
|
||||
@@ -468,7 +460,10 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return c.conditionForSearch(ctx, orgID, key, value, sb)
|
||||
}
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, matches)
|
||||
// Logs fields have no family support yet, so every logical field is
|
||||
// single-member and flattens losslessly to its physical key.
|
||||
resolved, warning := querybuilder.ResolveLogicalFields(key, matches)
|
||||
keys := querybuilder.SingleKeys(resolved)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
|
||||
@@ -905,52 +905,3 @@ func TestConditionForJSONBodySearch(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// IN on the body column routes each value back through the `=` path; the SQL it produces
|
||||
// must stay what the shared IN handling produced before, including for a mixed-type list.
|
||||
func TestConditionForBodyIn(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
values []any
|
||||
expectedSQL string
|
||||
expectedArgs []any
|
||||
}{
|
||||
{
|
||||
name: "strings",
|
||||
values: []any{"alpha", "beta"},
|
||||
expectedSQL: "(body = ? OR body = ?)",
|
||||
expectedArgs: []any{"alpha", "beta"},
|
||||
},
|
||||
{
|
||||
name: "mixed types are stringified before they reach the column",
|
||||
values: []any{"alpha", float64(1), true},
|
||||
expectedSQL: "(body = ? OR body = ? OR body = ?)",
|
||||
expectedArgs: []any{"alpha", "1", "true"},
|
||||
},
|
||||
}
|
||||
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
conditionBuilder := NewConditionBuilder(fm, fl)
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
key := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "body",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("1").From("t")
|
||||
cond, _, err := conditionBuilder.ConditionFor(context.Background(), valuer.UUID{}, 0, 0, &key,
|
||||
map[string][]*telemetrytypes.TelemetryFieldKey{key.Name: {&key}}, qbtypes.ConditionBuilderOptions{},
|
||||
qbtypes.FilterOperatorIn, tc.values, sb)
|
||||
require.NoError(t, err)
|
||||
sb.Where(cond...)
|
||||
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, tc.expectedSQL)
|
||||
assert.Equal(t, tc.expectedArgs, args)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,7 +279,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
var stmts []string
|
||||
for _, key := range candidates {
|
||||
guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, key, true)
|
||||
guard, err := m.ExistsFor(ctx, orgID, tsStart, tsEnd, key, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -308,7 +308,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
if !m.membershipGuarded(ctx, orgID, tsStart, tsEnd, candidates[0]) {
|
||||
return m.FieldFor(ctx, orgID, tsStart, tsEnd, candidates[0])
|
||||
}
|
||||
guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, candidates[0], true)
|
||||
guard, err := m.ExistsFor(ctx, orgID, tsStart, tsEnd, candidates[0], true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -326,7 +326,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
|
||||
var stmts []string
|
||||
for _, key := range candidates {
|
||||
guard, err := m.existsExpressionFor(ctx, orgID, tsStart, tsEnd, key, true)
|
||||
guard, err := m.ExistsFor(ctx, orgID, tsStart, tsEnd, key, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -569,7 +569,8 @@ func (m *fieldMapper) membershipGuarded(ctx context.Context, orgID valuer.UUID,
|
||||
return columnType == schema.ColumnTypeEnumMap || columnType == schema.ColumnTypeEnumJSON
|
||||
}
|
||||
|
||||
func (m *fieldMapper) existsExpressionFor(
|
||||
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
|
||||
func (m *fieldMapper) ExistsFor(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
tsStart, tsEnd uint64,
|
||||
|
||||
@@ -162,7 +162,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
keys := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
// Metric labels have no family support, so every logical field is
|
||||
// single-member and flattens losslessly to its physical key.
|
||||
keys := querybuilder.SingleKeys(querybuilder.MatchingLogicalFields(ctx, orgID, nil, key, fieldKeys))
|
||||
var warnings []string
|
||||
if len(keys) == 0 {
|
||||
if _, isColumn := timeSeriesV4Columns[key.Name]; isColumn {
|
||||
|
||||
@@ -97,6 +97,18 @@ func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsE
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
}
|
||||
|
||||
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
|
||||
// Intrinsic fields always exist; labels are checked for key membership.
|
||||
func (m *fieldMapper) ExistsFor(_ context.Context, _ valuer.UUID, _, _ uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error) {
|
||||
if slices.Contains(IntrinsicFields, key.Name) {
|
||||
return "true", nil
|
||||
}
|
||||
if exists {
|
||||
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
|
||||
}
|
||||
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -19,12 +20,15 @@ import (
|
||||
|
||||
type conditionBuilder struct {
|
||||
fm qbtypes.FieldMapper
|
||||
// fl evaluates the resolve_semconv_families flag during resolution.
|
||||
// A nil flagger keeps resolution literal.
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.ConditionBuilder = (*conditionBuilder)(nil)
|
||||
|
||||
func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
return &conditionBuilder{fm: fm}
|
||||
func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionBuilder {
|
||||
return &conditionBuilder{fm: fm, fl: fl}
|
||||
}
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
@@ -32,7 +36,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
orgID valuer.UUID,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
logical *telemetrytypes.LogicalField,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
@@ -42,13 +46,13 @@ func (c *conditionBuilder) conditionFor(
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
fieldExpression, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, c.fm, logical)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// TODO(srikanthccv): maybe extend this to every possible attribute
|
||||
if key.Name == "duration_nano" || key.Name == "durationNano" { // QoL improvement
|
||||
if logical.Name == "duration_nano" || logical.Name == "durationNano" { // QoL improvement
|
||||
switch v := value.(type) {
|
||||
case string:
|
||||
if duration, err := time.ParseDuration(v); err == nil {
|
||||
@@ -65,7 +69,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
}
|
||||
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(logical.Single(), value, fieldExpression, operator)
|
||||
|
||||
// regular operators
|
||||
switch operator {
|
||||
@@ -135,11 +139,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -150,11 +150,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
cond, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorNotEqual, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
conditions = append(conditions, cond)
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
@@ -162,11 +158,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// in the query builder, `exists` and `not exists` are used for
|
||||
// key membership checks, so depending on the column type, the condition changes
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
columns, err := c.fm.ColumnFor(ctx, orgID, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
pred, err := querybuilder.ExistsExpression(columns, key, startNs, endNs, fieldExpression, operator == qbtypes.FilterOperatorExists)
|
||||
pred, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, c.fm, logical, operator == qbtypes.FilterOperatorExists)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -218,10 +210,10 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
matches := querybuilder.MatchingLogicalFields(ctx, orgID, c.fl, key, fieldKeys)
|
||||
skipResourceFilter := options.SkipResourceFilter
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, matches)
|
||||
logicalFields, warning := querybuilder.ResolveLogicalFields(key, matches)
|
||||
var warnings []string
|
||||
if warning != "" {
|
||||
warnings = append(warnings, warning)
|
||||
@@ -229,10 +221,10 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// A bare key that names a real column filters on the column too — first. When metadata
|
||||
// only knows the name under other contexts, prepend the column and keep metadata matches
|
||||
// only where their type is consistent with it (a corrupt entry can't degrade the column).
|
||||
if key.FieldContext == telemetrytypes.FieldContextUnspecified && len(keys) > 0 {
|
||||
if key.FieldContext == telemetrytypes.FieldContextUnspecified && len(logicalFields) > 0 {
|
||||
hasColumn := false
|
||||
for _, k := range keys {
|
||||
if k.FieldContext == telemetrytypes.FieldContextSpan {
|
||||
for _, logical := range logicalFields {
|
||||
if logical.FieldContext == telemetrytypes.FieldContextSpan {
|
||||
hasColumn = true
|
||||
break
|
||||
}
|
||||
@@ -240,49 +232,49 @@ func (c *conditionBuilder) ConditionFor(
|
||||
if !hasColumn {
|
||||
probe := telemetrytypes.NewTelemetryFieldKey(key.Name, telemetrytypes.FieldContextSpan, key.FieldDataType)
|
||||
if cols, colErr := c.fm.ColumnFor(ctx, orgID, startNs, endNs, probe); colErr == nil && len(cols) > 0 {
|
||||
combined := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys)+1)
|
||||
combined = append(combined, probe)
|
||||
for _, k := range keys {
|
||||
if columnMatchesDataType(cols[0], k.FieldDataType) {
|
||||
combined = append(combined, k)
|
||||
combined := make([]*telemetrytypes.LogicalField, 0, len(logicalFields)+1)
|
||||
combined = append(combined, telemetrytypes.SingleLogicalField(key.Name, probe))
|
||||
for _, logical := range logicalFields {
|
||||
if columnMatchesDataType(cols[0], logical.FieldDataType) {
|
||||
combined = append(combined, logical)
|
||||
}
|
||||
}
|
||||
keys = combined
|
||||
logicalFields = combined
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
synthesized := false
|
||||
if len(keys) == 0 {
|
||||
if len(logicalFields) == 0 {
|
||||
// Not in metadata. CandidateKeys resolves it: fold contexts (span/trace) get the
|
||||
// metadata map so it can honor a real column, correct to a stripped-name metadata
|
||||
// match, or synthesize; strict contexts pass nil and keep their synthesize path.
|
||||
keys = c.fm.CandidateKeys(ctx, orgID, key, value, candidateLookupKeys(key, fieldKeys))
|
||||
if len(keys) == 0 {
|
||||
logicalFields = querybuilder.WrapAsLogicalFields(key.Name, c.fm.CandidateKeys(ctx, orgID, key, value, candidateLookupKeys(key, fieldKeys)))
|
||||
if len(logicalFields) == 0 {
|
||||
return nil, warnings, querybuilder.NewKeyNotFoundError(key.Name)
|
||||
}
|
||||
synthesized = true
|
||||
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
|
||||
}
|
||||
|
||||
// When a resource sub-query already covers the term, drop resource keys from the main
|
||||
// When a resource sub-query already covers the term, drop resource fields from the main
|
||||
// query. Synthesized keys are exempt: the sub-query skips keys absent from metadata.
|
||||
if skipResourceFilter && !synthesized {
|
||||
filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
if k.FieldContext != telemetrytypes.FieldContextResource {
|
||||
filtered = append(filtered, k)
|
||||
filtered := make([]*telemetrytypes.LogicalField, 0, len(logicalFields))
|
||||
for _, logical := range logicalFields {
|
||||
if logical.FieldContext != telemetrytypes.FieldContextResource {
|
||||
filtered = append(filtered, logical)
|
||||
}
|
||||
}
|
||||
if len(filtered) == 0 {
|
||||
return nil, warnings, nil
|
||||
}
|
||||
keys = filtered
|
||||
logicalFields = filtered
|
||||
}
|
||||
|
||||
conds := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
cond, err := c.conditionForKey(ctx, orgID, startNs, endNs, k, operator, value, sb)
|
||||
conds := make([]string, 0, len(logicalFields))
|
||||
for _, logical := range logicalFields {
|
||||
cond, err := c.conditionForLogicalField(ctx, orgID, startNs, endNs, logical, operator, value, sb)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -291,28 +283,28 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return conds, warnings, nil
|
||||
}
|
||||
|
||||
func (c *conditionBuilder) conditionForKey(
|
||||
func (c *conditionBuilder) conditionForLogicalField(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
logical *telemetrytypes.LogicalField,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
if c.isSpanScopeField(key.Name) {
|
||||
return c.buildSpanScopeCondition(key, operator, value, startNs)
|
||||
if c.isSpanScopeField(logical.Name) {
|
||||
return c.buildSpanScopeCondition(logical.Single(), operator, value, startNs)
|
||||
}
|
||||
|
||||
condition, err := c.conditionFor(ctx, orgID, startNs, endNs, key, operator, value, sb)
|
||||
condition, err := c.conditionFor(ctx, orgID, startNs, endNs, logical, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if operator.AddDefaultExistsFilter() {
|
||||
// skip adding exists filter for intrinsic fields
|
||||
field, _ := c.fm.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
field, _ := c.fm.FieldFor(ctx, orgID, startNs, endNs, logical.Single())
|
||||
if slices.Contains(maps.Keys(IntrinsicFields), field) ||
|
||||
slices.Contains(maps.Keys(IntrinsicFieldsDeprecated), field) ||
|
||||
slices.Contains(maps.Keys(CalculatedFields), field) ||
|
||||
@@ -320,7 +312,7 @@ func (c *conditionBuilder) conditionForKey(
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
existsCondition, err := c.conditionFor(ctx, orgID, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
existsCondition, err := c.conditionFor(ctx, orgID, startNs, endNs, logical, qbtypes.FilterOperatorExists, nil, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package tracestelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -288,8 +289,8 @@ func TestConditionFor(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
@@ -375,8 +376,8 @@ func TestConditionForResourceWithEvolution(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
conditionBuilder := NewConditionBuilder(fm, flaggertest.New(t))
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
@@ -395,8 +396,8 @@ func TestConditionForResourceWithEvolution(t *testing.T) {
|
||||
// user input and queries anyway, emitting a warning instead of failing.
|
||||
func TestConditionForSynthesizedKeys(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
cb := NewConditionBuilder(fm, flaggertest.New(t))
|
||||
|
||||
// no metadata matches -> the builder must synthesize from user input
|
||||
var noMatches map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
|
||||
183
pkg/telemetryschema/tracestelemetryschema/family_test.go
Normal file
183
pkg/telemetryschema/tracestelemetryschema/family_test.go
Normal file
@@ -0,0 +1,183 @@
|
||||
package tracestelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// familyFixture returns the deployment.environment(.name) family keys as trace
|
||||
// resource attributes (with the canonical evolution timeline), a metadata map
|
||||
// holding them, and a time range inside the JSON-column window.
|
||||
// familyFlagOn returns a flagger with resolve_semconv_families on.
|
||||
func familyFlagOn(t *testing.T) flagger.Flagger {
|
||||
return flaggertest.WithBooleanFlags(t, map[string]bool{
|
||||
flagger.FeatureResolveSemconvFamilies.String(): true,
|
||||
})
|
||||
}
|
||||
|
||||
func familyFixture() (current, old *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, startNs, endNs uint64) {
|
||||
releaseTime := time.Date(2025, 5, 22, 22, 0, 0, 0, time.UTC)
|
||||
newKey := func(name string) *telemetrytypes.TelemetryFieldKey {
|
||||
return &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: MockEvolutionData(releaseTime),
|
||||
}
|
||||
}
|
||||
current = newKey("deployment.environment.name")
|
||||
old = newKey("deployment.environment")
|
||||
fieldKeys = map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {current},
|
||||
old.Name: {old},
|
||||
}
|
||||
return current, old, fieldKeys, uint64(1747947419000000000), uint64(1747983448000000000)
|
||||
}
|
||||
|
||||
// memberValueExprs returns each member's own FieldFor output; family
|
||||
// expressions must be exactly the composition of these.
|
||||
func memberValueExprs(t *testing.T, fm qbtypes.FieldMapper, startNs, endNs uint64, members ...*telemetrytypes.TelemetryFieldKey) []string {
|
||||
t.Helper()
|
||||
exprs := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
expr, err := fm.FieldFor(context.Background(), valuer.UUID{}, startNs, endNs, member)
|
||||
require.NoError(t, err)
|
||||
exprs = append(exprs, expr)
|
||||
}
|
||||
return exprs
|
||||
}
|
||||
|
||||
func TestConditionForFamilyMergesMembersCurrentFirst(t *testing.T) {
|
||||
current, old, fieldKeys, startNs, endNs := familyFixture()
|
||||
fl := familyFlagOn(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
// The requested spelling is the old name; precedence must still be
|
||||
// current-first.
|
||||
requested := &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment"}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, warnings, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs, requested, fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, warnings, "a family is one logical field, never ambiguous with itself")
|
||||
require.Len(t, conds, 1)
|
||||
|
||||
exprs := memberValueExprs(t, fm, startNs, endNs, current, old)
|
||||
family := fmt.Sprintf("COALESCE(NULLIF(%s, ''), NULLIF(%s, ''), '')", exprs[0], exprs[1])
|
||||
|
||||
sb.Where(conds...)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, family+" = ?")
|
||||
// Equal adds the default exists filter: presence of any member.
|
||||
assert.Contains(t, sql, fmt.Sprintf("(%s IS NOT NULL OR %s IS NOT NULL)", exprs[0], exprs[1]))
|
||||
assert.Equal(t, []any{"production"}, args)
|
||||
}
|
||||
|
||||
func TestConditionForFamilyNegativeKeepsKeylessRows(t *testing.T) {
|
||||
current, old, fieldKeys, startNs, endNs := familyFixture()
|
||||
fl := familyFlagOn(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
requested := &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs, requested, fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotEqual, "production", sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
|
||||
exprs := memberValueExprs(t, fm, startNs, endNs, current, old)
|
||||
sb.Where(conds...)
|
||||
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
// The trailing '' makes rows without any member read '' (single-key map
|
||||
// semantics), so `!=` keeps including them; no exists filter is added.
|
||||
assert.Contains(t, sql, fmt.Sprintf("COALESCE(NULLIF(%s, ''), NULLIF(%s, ''), '') <> ?", exprs[0], exprs[1]))
|
||||
assert.NotContains(t, sql, "IS NOT NULL OR")
|
||||
}
|
||||
|
||||
func TestConditionForFamilyExists(t *testing.T) {
|
||||
current, old, fieldKeys, startNs, endNs := familyFixture()
|
||||
fl := familyFlagOn(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
requested := &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs, requested, fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorNotExists, nil, sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
|
||||
exprs := memberValueExprs(t, fm, startNs, endNs, current, old)
|
||||
sb.Where(conds...)
|
||||
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, fmt.Sprintf("NOT (%s IS NOT NULL OR %s IS NOT NULL)", exprs[0], exprs[1]))
|
||||
}
|
||||
|
||||
// With the flag off, both spellings can be in the metadata map and the
|
||||
// condition still uses only the requested key. Users see no change until the
|
||||
// flag is on.
|
||||
func TestConditionForFamilyOffByDefault(t *testing.T) {
|
||||
current, _, fieldKeys, startNs, endNs := familyFixture()
|
||||
fl := flaggertest.New(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
requested := &telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs, requested, fieldKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
|
||||
exprs := memberValueExprs(t, fm, startNs, endNs, current)
|
||||
sb.Where(conds...)
|
||||
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, exprs[0]+" = ?")
|
||||
assert.NotContains(t, sql, "COALESCE")
|
||||
}
|
||||
|
||||
// A single-member key's condition is byte-identical to the pre-family shape:
|
||||
// composition only appears when metadata proves a second member.
|
||||
func TestConditionForSingleMemberIsUnchanged(t *testing.T) {
|
||||
current, _, _, startNs, endNs := familyFixture()
|
||||
fl := familyFlagOn(t)
|
||||
fm := NewFieldMapper(fl)
|
||||
cb := NewConditionBuilder(fm, fl)
|
||||
|
||||
soloKeys := map[string][]*telemetrytypes.TelemetryFieldKey{current.Name: {current}}
|
||||
requested := &telemetrytypes.TelemetryFieldKey{Name: current.Name}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conds, _, err := cb.ConditionFor(context.Background(), valuer.UUID{}, startNs, endNs, requested, soloKeys, qbtypes.ConditionBuilderOptions{}, qbtypes.FilterOperatorEqual, "production", sb)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, conds, 1)
|
||||
|
||||
exprs := memberValueExprs(t, fm, startNs, endNs, current)
|
||||
sb.Where(conds...)
|
||||
sql, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, exprs[0]+" = ?")
|
||||
assert.NotContains(t, sql, "COALESCE")
|
||||
}
|
||||
|
||||
func TestColumnExpressionForFamilyGroupBy(t *testing.T) {
|
||||
current, old, fieldKeys, startNs, endNs := familyFixture()
|
||||
fm := NewFieldMapper(familyFlagOn(t))
|
||||
|
||||
expr, err := fm.ColumnExpressionFor(context.Background(), valuer.UUID{}, startNs, endNs,
|
||||
&telemetrytypes.TelemetryFieldKey{Name: "deployment.environment.name"}, telemetrytypes.FieldDataTypeString, fieldKeys)
|
||||
require.NoError(t, err)
|
||||
|
||||
exprs := memberValueExprs(t, fm, startNs, endNs, current, old)
|
||||
family := fmt.Sprintf("COALESCE(NULLIF(%s, ''), NULLIF(%s, ''), '')", exprs[0], exprs[1])
|
||||
guard := fmt.Sprintf("(%s IS NOT NULL OR %s IS NOT NULL)", exprs[0], exprs[1])
|
||||
assert.Equal(t, fmt.Sprintf("multiIf(%s, %s, NULL)", guard, family), expr)
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -159,12 +160,16 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
type fieldMapper struct{}
|
||||
type fieldMapper struct {
|
||||
// fl evaluates the resolve_semconv_families flag during resolution.
|
||||
// A nil flagger keeps resolution literal.
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
var _ qbtypes.FieldMapper = (*fieldMapper)(nil)
|
||||
|
||||
func NewFieldMapper() *fieldMapper {
|
||||
return &fieldMapper{}
|
||||
func NewFieldMapper(fl flagger.Flagger) *fieldMapper {
|
||||
return &fieldMapper{fl: fl}
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(
|
||||
@@ -336,6 +341,68 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
return exprs, existExprs, columns, nil
|
||||
}
|
||||
|
||||
// logicalForResolvedColumn upgrades a directly-resolvable key (the FieldFor
|
||||
// probe succeeded) to its family when the metadata map proves membership;
|
||||
// otherwise the key stays a single-member logical field.
|
||||
func (m *fieldMapper) logicalForResolvedColumn(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) *telemetrytypes.LogicalField {
|
||||
for _, logical := range querybuilder.MatchingLogicalFields(ctx, orgID, m.fl, field, keys) {
|
||||
if logical.IsFamily() &&
|
||||
logical.FieldContext == field.FieldContext &&
|
||||
(field.FieldDataType == telemetrytypes.FieldDataTypeUnspecified || logical.FieldDataType == field.FieldDataType) {
|
||||
return logical
|
||||
}
|
||||
}
|
||||
return telemetrytypes.SingleLogicalField(field.Name, field)
|
||||
}
|
||||
|
||||
// upgradeToFamilies swaps single-member candidates for their family when the
|
||||
// metadata map proves membership. Candidate order and every non-family
|
||||
// candidate stay exactly as the legacy flow produced them; sibling candidates
|
||||
// of an already-emitted family are dropped rather than duplicated.
|
||||
func (m *fieldMapper) upgradeToFamilies(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, candidates []*telemetrytypes.LogicalField, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.LogicalField {
|
||||
var families []*telemetrytypes.LogicalField
|
||||
for _, logical := range querybuilder.MatchingLogicalFields(ctx, orgID, m.fl, field, keys) {
|
||||
if logical.IsFamily() {
|
||||
families = append(families, logical)
|
||||
}
|
||||
}
|
||||
if len(families) == 0 {
|
||||
return candidates
|
||||
}
|
||||
|
||||
out := make([]*telemetrytypes.LogicalField, 0, len(candidates))
|
||||
emitted := make(map[*telemetrytypes.LogicalField]bool)
|
||||
for _, candidate := range candidates {
|
||||
var family *telemetrytypes.LogicalField
|
||||
for _, fam := range families {
|
||||
if fam.FieldContext != candidate.FieldContext || fam.FieldDataType != candidate.FieldDataType {
|
||||
continue
|
||||
}
|
||||
memberOfFamily := candidate.Single().Name == field.Name
|
||||
for _, member := range fam.Members {
|
||||
if member.Name == candidate.Single().Name {
|
||||
memberOfFamily = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if memberOfFamily {
|
||||
family = fam
|
||||
break
|
||||
}
|
||||
}
|
||||
if family == nil {
|
||||
out = append(out, candidate)
|
||||
continue
|
||||
}
|
||||
if emitted[family] {
|
||||
continue
|
||||
}
|
||||
emitted[family] = true
|
||||
out = append(out, family)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ColumnExpressionFor returns the bare (unaliased) SQL expression for the field, resolving
|
||||
// unknown keys via CandidateKeys and wrapping guardable columns with exists-guard multiIfs
|
||||
// so an absent key yields NULL.
|
||||
@@ -348,18 +415,23 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
// Resolve the candidate column(s).
|
||||
var candidates []*telemetrytypes.TelemetryFieldKey
|
||||
// Resolve the candidate logical field(s).
|
||||
var candidates []*telemetrytypes.LogicalField
|
||||
switch _, err := m.FieldFor(ctx, orgID, startNs, endNs, field); {
|
||||
case err == nil:
|
||||
candidates = []*telemetrytypes.TelemetryFieldKey{field}
|
||||
// A directly-resolvable key upgrades to its family when the metadata
|
||||
// map proves membership; otherwise it stays single-member.
|
||||
candidates = []*telemetrytypes.LogicalField{m.logicalForResolvedColumn(ctx, orgID, field, keys)}
|
||||
case errors.Is(err, qbtypes.ErrColumnNotFound):
|
||||
// column (when the bare name is one) plus metadata matches, else synthesized
|
||||
// type-variant keys.
|
||||
candidates = m.CandidateKeys(ctx, orgID, field, nil, keys)
|
||||
if len(candidates) == 0 {
|
||||
// The legacy candidate flow, unchanged: column (when the bare name is
|
||||
// one) plus metadata matches, else synthesized type-variant keys. The
|
||||
// family step below only swaps candidates for their family; it never
|
||||
// changes candidate order or non-family behavior.
|
||||
raw := m.CandidateKeys(ctx, orgID, field, nil, keys)
|
||||
if len(raw) == 0 {
|
||||
return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name).WithSuggestions(errors.NewSuggestionsOnLevenshteinDistance(field.Name, errors.NounKeys, maps.Keys(keys))...)
|
||||
}
|
||||
candidates = m.upgradeToFamilies(ctx, orgID, field, querybuilder.WrapAsLogicalFields(field.Name, raw), keys)
|
||||
default:
|
||||
return "", err
|
||||
}
|
||||
@@ -373,21 +445,21 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
dummyValue = 0.0
|
||||
}
|
||||
stmts := make([]string, 0, len(candidates)*2)
|
||||
for _, key := range candidates {
|
||||
value, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
for _, logical := range candidates {
|
||||
value, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, m, logical)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, key, true)
|
||||
guard, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, m, logical, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
coerced := value
|
||||
// a time column keeps its native type; coercing it would yield seconds
|
||||
if temporal, err := m.columnIsTemporal(ctx, startNs, endNs, key); err != nil {
|
||||
if temporal, err := m.logicalIsTemporal(ctx, startNs, endNs, logical); err != nil {
|
||||
return "", err
|
||||
} else if !temporal {
|
||||
coerced, _ = querybuilder.DataTypeCollisionHandledFieldName(key, dummyValue, value, qbtypes.FilterOperatorUnknown)
|
||||
coerced, _ = querybuilder.DataTypeCollisionHandledFieldName(logical.Single(), dummyValue, value, qbtypes.FilterOperatorUnknown)
|
||||
}
|
||||
stmts = append(stmts, guard, coerced)
|
||||
}
|
||||
@@ -395,13 +467,14 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
|
||||
if len(candidates) == 1 {
|
||||
value, err := m.FieldFor(ctx, orgID, startNs, endNs, candidates[0])
|
||||
logical := candidates[0]
|
||||
value, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, m, logical)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
exprs, existExprs, _, _ := m.resolveColumnExprs(ctx, startNs, endNs, candidates[0])
|
||||
if len(exprs) == 1 && len(existExprs) == 1 {
|
||||
guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, candidates[0], true)
|
||||
exprs, existExprs, _, _ := m.resolveColumnExprs(ctx, startNs, endNs, logical.Single())
|
||||
if !logical.IsFamily() && len(exprs) == 1 && len(existExprs) == 1 {
|
||||
guard, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, m, logical, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -413,12 +486,12 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
// Multiple candidates (collision / synth): multiIf picks the first that exists,
|
||||
// stringified so branches share a type.
|
||||
args := make([]string, 0, len(candidates))
|
||||
for _, key := range candidates {
|
||||
value, err := m.FieldFor(ctx, orgID, startNs, endNs, key)
|
||||
for _, logical := range candidates {
|
||||
value, err := querybuilder.LogicalValueExpr(ctx, orgID, startNs, endNs, m, logical)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
guard, err := m.existsExpressionFor(ctx, orgID, startNs, endNs, key, true)
|
||||
guard, err := querybuilder.LogicalExistsExpr(ctx, orgID, startNs, endNs, m, logical, true)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -427,6 +500,15 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
return fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", ")), nil
|
||||
}
|
||||
|
||||
// logicalIsTemporal reports whether the logical field resolves to a single time
|
||||
// column. A family is attribute-backed and never temporal.
|
||||
func (m *fieldMapper) logicalIsTemporal(ctx context.Context, startNs, endNs uint64, logical *telemetrytypes.LogicalField) (bool, error) {
|
||||
if logical.IsFamily() {
|
||||
return false, nil
|
||||
}
|
||||
return m.columnIsTemporal(ctx, startNs, endNs, logical.Single())
|
||||
}
|
||||
|
||||
// columnIsTemporal reports whether key resolves to a single time column, after evolution
|
||||
// selection. Multiple columns mean an attribute-map union, which is never temporal.
|
||||
func (m *fieldMapper) columnIsTemporal(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (bool, error) {
|
||||
@@ -522,7 +604,8 @@ func (m *fieldMapper) CandidateKeys(ctx context.Context, _ valuer.UUID, field *t
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) existsExpressionFor(
|
||||
// ExistsFor implements the per-key existence primitive of qbtypes.FieldMapper.
|
||||
func (m *fieldMapper) ExistsFor(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
tsStart, tsEnd uint64,
|
||||
|
||||
@@ -2,6 +2,7 @@ package tracestelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -107,7 +108,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
result, err := fm.FieldFor(ctx, valuer.UUID{}, uint64(time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC).UnixNano()), uint64(time.Date(2024, 6, 5, 0, 0, 0, 0, time.UTC).UnixNano()), &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
@@ -195,7 +196,7 @@ func TestFieldForResourceWithEvolution(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
result, err := fm.FieldFor(ctx, valuer.UUID{}, tc.tsStart, tc.tsEnd, &tc.key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedResult, result)
|
||||
@@ -255,7 +256,7 @@ func TestColumnExpressionForTemporalColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
result, err := fm.ColumnExpressionFor(ctx, valuer.UUID{}, tsStart, tsEnd, &tc.key, tc.requiredDataType, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedResult, result)
|
||||
@@ -283,7 +284,7 @@ func TestColumnExpressionForTimestampAttributeCollision(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
|
||||
t.Run("bare timestamp resolves to the intrinsic column alone", func(t *testing.T) {
|
||||
bare := telemetrytypes.TelemetryFieldKey{Name: "timestamp"}
|
||||
|
||||
@@ -2,6 +2,7 @@ package tracestelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/SigNoz/signoz/pkg/flagger/flaggertest"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
@@ -14,8 +15,8 @@ import (
|
||||
|
||||
func TestSpanScopeFilterExpression(t *testing.T) {
|
||||
// Test that span scope fields work in filter expressions
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
cb := NewConditionBuilder(fm, flaggertest.New(t))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -128,8 +129,8 @@ func TestSpanScopeWithResourceFilter(t *testing.T) {
|
||||
// For now, just verify the expression parses correctly
|
||||
// In a real implementation, we'd need to check that the resource filter
|
||||
// is properly skipped when span scope fields are present
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
fm := NewFieldMapper(flaggertest.New(t))
|
||||
cb := NewConditionBuilder(fm, flaggertest.New(t))
|
||||
|
||||
// Prepare field keys for the test
|
||||
fieldKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
|
||||
|
||||
@@ -113,6 +113,21 @@ func BuildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytype
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
},
|
||||
// both spellings of an enabled semantic-convention family
|
||||
"deployment.environment.name": {
|
||||
{
|
||||
Name: "deployment.environment.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
},
|
||||
"deployment.environment": {
|
||||
{
|
||||
Name: "deployment.environment",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
|
||||
@@ -18,7 +18,14 @@ var (
|
||||
ErrUnsupportedOperator = errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator")
|
||||
)
|
||||
|
||||
// FieldMapper maps the telemetry field key to the table field name.
|
||||
// FieldMapper is a signal's storage dialect: given a resolved physical key, it
|
||||
// renders how this signal's tables express that key — its value (FieldFor),
|
||||
// its presence (ExistsFor), and its backing columns (ColumnFor) — honoring
|
||||
// materialization and column-generation choices carried on the key.
|
||||
//
|
||||
// ColumnExpressionFor and CandidateKeys additionally carry per-signal
|
||||
// resolution behavior (unknown-name fallback, candidate ordering); they
|
||||
// predate the resolution layer and are slated to move behind it.
|
||||
type FieldMapper interface {
|
||||
// FieldFor returns the field name for the given key.
|
||||
FieldFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error)
|
||||
@@ -34,6 +41,11 @@ type FieldMapper interface {
|
||||
// the name (or `{context}.{name}`) first, else synthesized type-variant keys for sources
|
||||
// that support it, else nil (caller errors). value is the filter operand, nil otherwise.
|
||||
CandidateKeys(ctx context.Context, orgID valuer.UUID, field *telemetrytypes.TelemetryFieldKey, value any, keys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey
|
||||
// ExistsFor returns the existence predicate for a single physical key (negated when
|
||||
// exists is false), self-contained and arg-free so it can guard column expressions.
|
||||
// It is the per-member primitive querybuilder.LogicalExistsExpr and the numeric branch
|
||||
// of querybuilder.LogicalValueExpr compose family expressions from.
|
||||
ExistsFor(ctx context.Context, orgID valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, exists bool) (string, error)
|
||||
}
|
||||
|
||||
// ConditionBuilder builds the conditions for a filter term. The builder owns key resolution:
|
||||
|
||||
@@ -3,13 +3,11 @@ from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.querier import build_order_by, build_raw_query, get_rows, make_query_request
|
||||
|
||||
|
||||
def test_logs_json_body_simple_searches(
|
||||
@@ -913,61 +911,3 @@ def test_logs_json_body_listing(
|
||||
assert len(results) == 1
|
||||
count = results[0]["data"][0][0]
|
||||
assert count == 4 # 4 logs have status="success"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression,expected_services",
|
||||
[
|
||||
pytest.param("body.service IN ['auth', 'payment']", {"auth", "payment"}, id="in_scalar_path"),
|
||||
pytest.param("body.status IN [200, 500]", {"auth", "payment"}, id="in_number_path"),
|
||||
pytest.param("body.service NOT IN ['auth']", {"payment", "search"}, id="not_in_scalar_path"),
|
||||
# An `[]` path is extracted as an array. Comparing that array to each scalar in the
|
||||
# list is something ClickHouse rejects outright (code 130), so this shape used to
|
||||
# fail the whole query; per-value extraction reads the first element instead.
|
||||
pytest.param("body.user_names[*] IN ['alpha', 'gamma']", {"auth", "payment"}, id="in_array_path"),
|
||||
],
|
||||
)
|
||||
def test_logs_json_body_in_operator(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
expression: str,
|
||||
expected_services: set[str],
|
||||
) -> None:
|
||||
"""IN over a body JSON path fans out to one comparison per value."""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
specs = [("auth", 200, ["alpha", "beta"]), ("payment", 500, ["gamma"]), ("search", 404, ["beta", "alpha"])]
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": "api"},
|
||||
body=json.dumps({"service": service, "status": status, "user_names": user_names}),
|
||||
)
|
||||
for i, (service, status, user_names) in enumerate(specs)
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=5)).timestamp() * 1000),
|
||||
end_ms=int(now.timestamp() * 1000),
|
||||
request_type="raw",
|
||||
queries=[
|
||||
build_raw_query(
|
||||
"A",
|
||||
"logs",
|
||||
filter_expression=expression,
|
||||
order=[build_order_by("timestamp", "desc")],
|
||||
limit=100,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
assert response.json()["status"] == "success"
|
||||
# flag off: the body comes back as the raw JSON string
|
||||
assert {json.loads(row["data"]["body"])["service"] for row in get_rows(response)} == expected_services
|
||||
|
||||
Reference in New Issue
Block a user