Compare commits

..

1 Commits

Author SHA1 Message Date
srikanthccv
e165bff99b feat: resolve semconv families in the deprecated services read paths
The v2 services module needs no change: it renders a QBv5 filter
expression and runs through the querier, so the merged LogicalField
resolution covers it when the resolve_semconv_families flag is on.

This layer covers the read paths that do not go through QBv5, behind
the same flag (default: disabled):

- The v1 services endpoints (services list, top operations) build the
  legacy resource sub-query. It now merges the family members with
  current-wins precedence. The value expression keeps the trailing ''
  so keyless rows stay in negative filters. Index hints widen to any
  member for positive operators, and negated hints are dropped, because
  a negated hint can remove a row where another member holds the value.
- The dependency graph accepts every family spelling as a filter key;
  each spelling targets the historical deployment_environment column.
- The reader evaluates the flag per request from the org in the
  request claims; no claims means off. The legacy logs and traces v4
  explorer paths pass the flag as off and stay literal.

Every field-name interpolation in the touched functions is hardened:
plain literals go through querybuilder.ClickHouseStringLiteral, and
keys inside LIKE index hints go through QuoteEscapedStringForContains,
the same escaping the hint values already use. For ordinary names the
output is byte identical.

With the flag off, every generated query is the same as main, and
tests pin this. The quick-filter default change and the stored-row
migration from the earlier version of this layer are deferred to the
rollout phase: persisted rows cannot be gated by a flag.

Assisted-by: Claude Fable 5
2026-08-18 05:43:42 +05:30
22 changed files with 302 additions and 266 deletions

View File

@@ -1,30 +0,0 @@
#!/bin/sh
# Installs dispatcher stubs into the repository's hooks directory. Safe to run
# repeatedly; invoked automatically from the Makefile and from the frontend
# postinstall so no manual setup is required.
set -e
git rev-parse --git-dir >/dev/null 2>&1 || exit 0
# core.hooksPath shadows the hooks directory git would otherwise use, and an
# absolute value leaks one worktree's path into every other worktree.
if [ -n "$(git config --get core.hooksPath 2>/dev/null || true)" ]; then
git config --unset-all core.hooksPath 2>/dev/null || true
fi
# Resolves to the common directory for linked worktrees, which is where git
# looks for hooks regardless of which worktree the commit happens in.
hooks_dir="$(git rev-parse --git-path hooks)"
mkdir -p "$hooks_dir"
for hook in pre-commit commit-msg; do
cat >"$hooks_dir/$hook" <<EOF
#!/bin/sh
# Generated by .githooks/install.
root="\$(git rev-parse --show-toplevel)"
[ -x "\$root/.githooks/$hook" ] || exit 0
exec "\$root/.githooks/$hook" "\$@"
EOF
chmod +x "$hooks_dir/$hook"
done

View File

@@ -1,22 +0,0 @@
#!/bin/sh
set -e
staged_go_files=$(git diff --cached --name-only --diff-filter=ACM -- '*.go')
if [ -n "$staged_go_files" ]; then
if command -v gofmt >/dev/null 2>&1; then
echo "$staged_go_files" | xargs gofmt -l -w
echo "$staged_go_files" | xargs git add
else
echo "pre-commit: gofmt not found, skipping go formatting" >&2
fi
fi
staged_web_files=$(git diff --cached --name-only --diff-filter=ACM -- 'frontend/*')
if [ -n "$staged_web_files" ]; then
if [ -d frontend/node_modules ] && command -v pnpm >/dev/null 2>&1; then
cd frontend && pnpm lint-staged
else
echo "pre-commit: frontend dependencies not installed, skipping lint-staged" >&2
fi
fi

View File

@@ -35,13 +35,6 @@ DOCKERFILE_ENTERPRISE = $(SRC)/cmd/enterprise/Dockerfile
DOCKER_REGISTRY_ENTERPRISE ?= docker.io/signoz/signoz
JS_BUILD_CONTEXT = $(SRC)/frontend
##############################################################
# git hooks
##############################################################
# Hooks live outside the working tree, so they cannot be checked in. Expanded
# while this file is parsed, which wires them up on any make invocation.
GITHOOKS_INSTALL := $(shell sh $(SRC)/.githooks/install >/dev/null 2>&1 || true)
##############################################################
# directories
##############################################################

View File

@@ -1,12 +1,7 @@
#!/bin/sh
. "$(dirname "$0")/_/husky.sh"
set -e
if [ -d frontend/node_modules ] && command -v pnpm >/dev/null 2>&1; then
(cd frontend && pnpm run commitlint --edit "$1")
else
echo "commit-msg: frontend dependencies not installed, skipping commitlint" >&2
fi
cd frontend && pnpm run commitlint --edit $1
branch="$(git rev-parse --abbrev-ref HEAD)"

4
frontend/.husky/pre-commit Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
. "$(dirname "$0")/_/husky.sh"
cd frontend && pnpm lint-staged

View File

@@ -20,7 +20,8 @@
"jest": "jest",
"jest:coverage": "jest --coverage",
"jest:watch": "jest --watch",
"postinstall": "pnpm i18n:generate-hash && (is-ci || sh ../.githooks/install) && node scripts/update-registry.cjs",
"postinstall": "pnpm i18n:generate-hash && (is-ci || pnpm husky:configure) && node scripts/update-registry.cjs",
"husky:configure": "cd .. && husky install frontend/.husky && cd frontend && chmod ug+x .husky/*",
"commitlint": "commitlint --edit $1",
"test": "jest",
"test:changedsince": "jest --changedSince=main --coverage --silent",
@@ -185,6 +186,7 @@
"@typescript/native-preview": "7.0.0-dev.20260430.1",
"eslint-plugin-sonarjs": "4.0.2",
"glob": "^13.0.6",
"husky": "^7.0.4",
"is-ci": "^3.0.1",
"jest-environment-jsdom": "29.7.0",
"jest-styled-components": "^7.2.0",

View File

@@ -440,6 +440,9 @@ importers:
glob:
specifier: ^13.0.6
version: 13.0.6
husky:
specifier: ^7.0.4
version: 7.0.4
is-ci:
specifier: ^3.0.1
version: 3.0.1
@@ -5526,6 +5529,11 @@ packages:
resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==}
engines: {node: '>=18.18.0'}
husky@7.0.4:
resolution: {integrity: sha512-vbaCKN2QLtP/vD4yvs6iz6hBEo6wkSzs8HpRah1Z6aGmF2KW5PdYuAd7uX5a+OyBZHBhd+TFLqgjUgytQr4RvQ==}
engines: {node: '>=12'}
hasBin: true
hyphenate-style-name@1.0.4:
resolution: {integrity: sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==}
@@ -14628,6 +14636,8 @@ snapshots:
human-signals@8.0.1: {}
husky@7.0.4: {}
hyphenate-style-name@1.0.4: {}
i18next-browser-languagedetector@6.1.8:

View File

@@ -2,17 +2,14 @@ import { useTranslation } from 'react-i18next';
import { Input } from '@signozhq/ui/input';
import { Switch } from '@signozhq/ui/switch';
import { Form, Select, Space } from 'antd';
import { FeatureKeys } from 'constants/features';
import { ModalFooterTitle } from 'container/PipelinePage/styles';
import { useAppContext } from 'providers/App/App';
import { ProcessorData } from 'types/api/pipeline/def';
import { formValidationRules } from '../config';
import { ProcessorFormField } from './config';
import { processorFields, ProcessorFormField } from './config';
import CSVInput from './FormFields/CSVInput';
import JsonFlattening from './FormFields/JsonFlattening';
import { FormWrapper, PipelineIndexIcon, StyledSelect } from './styles';
import { resolveProcessorFields } from './utils';
import './styles.scss';
@@ -136,23 +133,16 @@ function ProcessorForm({
selectedProcessorData,
isAdd,
}: ProcessorFormProps): JSX.Element {
const { featureFlags } = useAppContext();
const isBodyJsonEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.USE_JSON_BODY)
?.active || false;
return (
<div className="processor-form-container">
{resolveProcessorFields(processorType, isBodyJsonEnabled).map(
(fieldData: ProcessorFormField) => (
<ProcessorFieldInput
key={fieldData.name + String(fieldData.initialValue)}
fieldData={fieldData}
selectedProcessorData={selectedProcessorData}
isAdd={isAdd}
/>
),
)}
{processorFields[processorType]?.map((fieldData: ProcessorFormField) => (
<ProcessorFieldInput
key={fieldData.name + String(fieldData.initialValue)}
fieldData={fieldData}
selectedProcessorData={selectedProcessorData}
isAdd={isAdd}
/>
))}
</div>
);
}

View File

@@ -1,24 +0,0 @@
import { processorFields, ProcessorFormField } from './config';
const BODY_PARSE_FROM = 'body';
const JSON_BODY_PARSE_FROM = 'body.message';
// With use_json_body the collector normalizes every body into a map before user
// operators run, so a parser pointed at `body` gets a map it cannot read and
// silently extracts nothing. The log text lives at body.message.
export function resolveProcessorFields(
processorType: string,
isBodyJsonEnabled: boolean,
): Array<ProcessorFormField> {
const fields = processorFields[processorType] ?? [];
if (!isBodyJsonEnabled) {
return fields;
}
return fields.map((field) =>
field.name === 'parse_from' && field.initialValue === BODY_PARSE_FROM
? { ...field, initialValue: JSON_BODY_PARSE_FROM }
: field,
);
}

View File

@@ -1,45 +0,0 @@
import { processorFields } from '../PipelineListsView/AddNewProcessor/config';
import { resolveProcessorFields } from '../PipelineListsView/AddNewProcessor/utils';
const parseFromDefault = (
fields: ReturnType<typeof resolveProcessorFields>,
): unknown => fields.find((field) => field.name === 'parse_from')?.initialValue;
describe('resolveProcessorFields', () => {
it.each(['grok_parser', 'regex_parser', 'json_parser'])(
'defaults %s parse_from to body.message when use_json_body is on',
(processorType) => {
expect(parseFromDefault(resolveProcessorFields(processorType, true))).toBe(
'body.message',
);
},
);
it.each(['grok_parser', 'regex_parser', 'json_parser'])(
'keeps %s parse_from as body when use_json_body is off',
(processorType) => {
expect(parseFromDefault(resolveProcessorFields(processorType, false))).toBe(
'body',
);
},
);
it('leaves parse_from defaults that do not point at the body alone', () => {
expect(parseFromDefault(resolveProcessorFields('time_parser', true))).toBe(
'attributes.timestamp',
);
expect(
parseFromDefault(resolveProcessorFields('severity_parser', true)),
).toBe('attributes.logLevel');
});
it('does not mutate the shared config', () => {
resolveProcessorFields('grok_parser', true);
expect(parseFromDefault(processorFields.grok_parser)).toBe('body');
});
it('returns an empty list for an unknown processor type', () => {
expect(resolveProcessorFields('does_not_exist', true)).toStrictEqual([]);
});
});

View File

@@ -29,7 +29,6 @@ type builderQuery[T any] struct {
telemetryStore telemetrystore.TelemetryStore
orgID valuer.UUID
stmtBuilder qbtypes.StatementBuilder[T]
queryType qbtypes.QueryType
spec qbtypes.QueryBuilderQuery[T]
variables map[string]qbtypes.VariableItem
@@ -52,7 +51,6 @@ func newBuilderQuery[T any](
telemetryStore telemetrystore.TelemetryStore,
orgID valuer.UUID,
stmtBuilder qbtypes.StatementBuilder[T],
queryType qbtypes.QueryType,
spec qbtypes.QueryBuilderQuery[T],
tr qbtypes.TimeRange,
kind qbtypes.RequestType,
@@ -64,7 +62,6 @@ func newBuilderQuery[T any](
telemetryStore: telemetryStore,
orgID: orgID,
stmtBuilder: stmtBuilder,
queryType: queryType,
spec: spec,
variables: variables,
fromMS: tr.From,
@@ -84,7 +81,7 @@ func (q *builderQuery[T]) Fingerprint() string {
// Create a deterministic fingerprint for builder queries
// This needs to include all fields that affect the query results
parts := []string{q.queryType.StringValue()}
parts := []string{"builder"}
// Add signal type
parts = append(parts, fmt.Sprintf("signal=%s", q.spec.Signal.StringValue()))

View File

@@ -3,7 +3,6 @@ package querier
import (
"strings"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -21,8 +20,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "fingerprint includes shiftby when ShiftBy field is set",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 3600,
@@ -42,8 +40,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "fingerprint includes shiftby but not other functions",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 3600,
@@ -66,8 +63,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "no shiftby in fingerprint when ShiftBy is zero",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 0,
@@ -98,29 +94,6 @@ func TestBuilderQueryFingerprint(t *testing.T) {
}
}
func TestBuilderQueryFingerprintQueryType(t *testing.T) {
spec := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "count()"}},
Filter: &qbtypes.Filter{Expression: "gen_ai.request.model EXISTS"},
}
regular := &builderQuery[qbtypes.TraceAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
spec: spec,
}
ai := &builderQuery[qbtypes.TraceAggregation]{
queryType: qbtypes.QueryTypeBuilderAI,
kind: qbtypes.RequestTypeTimeSeries,
spec: spec,
}
assert.True(t, strings.HasPrefix(regular.Fingerprint(), qbtypes.QueryTypeBuilder.StringValue()+"&"))
assert.True(t, strings.HasPrefix(ai.Fingerprint(), qbtypes.QueryTypeBuilderAI.StringValue()+"&"))
assert.NotEqual(t, regular.Fingerprint(), ai.Fingerprint())
}
func TestMakeBucketsOrder(t *testing.T) {
// Test that makeBuckets returns buckets in reverse chronological order by default
// Using milliseconds as input - need > 1 hour range to get multiple buckets

View File

@@ -305,7 +305,7 @@ func (q *querier) buildQueries(
}
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryTypeBuilder:
@@ -313,7 +313,7 @@ func (q *querier) buildQueries(
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
@@ -323,7 +323,7 @@ func (q *querier) buildQueries(
if spec.Source == telemetrytypes.SourceAudit {
stmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
@@ -340,9 +340,9 @@ func (q *querier) buildQueries(
if spec.Source == telemetrytypes.SourceMeter {
event.Source = telemetrytypes.SourceMeter.StringValue()
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
} else {
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
}
queries[spec.Name] = bq
@@ -618,7 +618,7 @@ func (q *querier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qb
if spec.Source == telemetrytypes.SourceAudit {
liveTailStmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, liveTailStmtBuilder, query.Type, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, liveTailStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
"id": {
Value: updatedLogID,
},
@@ -941,9 +941,8 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
// reuse the original query's statement builder and type so an AI query
// keeps its AI builder and cache key
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, qt.builderConfig)
// reuse the original query's statement builder so an AI query keeps its AI builder
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
case *builderQuery[qbtypes.LogAggregation]:
specCopy := qt.spec.Copy()
@@ -953,16 +952,16 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
if qt.spec.Source == telemetrytypes.SourceAudit {
shiftStmtBuilder = q.auditStmtBuilder
}
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, q.builderConfig)
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, q.builderConfig)
case *builderQuery[qbtypes.MetricAggregation]:
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
if qt.spec.Source == telemetrytypes.SourceMeter {
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.meterStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.meterStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
}
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.metricStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.metricStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
case *traceOperatorQuery:
specCopy := qt.spec.Copy()
return &traceOperatorQuery{

View File

@@ -336,7 +336,7 @@ func (r *ClickHouseReader) GetTopLevelOperations(ctx context.Context, start, end
return &operations, nil
}
func (r *ClickHouseReader) buildResourceSubQuery(tags []model.TagQueryParam, svc string, start, end time.Time) (string, error) {
func (r *ClickHouseReader) buildResourceSubQuery(ctx context.Context, orgID valuer.UUID, tags []model.TagQueryParam, svc string, start, end time.Time) (string, error) {
// assuming all will be resource attributes.
// and resource attributes are string for traces
filterSet := v3.FilterSet{}
@@ -387,7 +387,8 @@ func (r *ClickHouseReader) buildResourceSubQuery(tags []model.TagQueryParam, svc
&filterSet,
[]v3.AttributeKey{},
v3.AttributeKey{},
false)
false,
r.fl.BooleanOrEmpty(ctx, flagger.FeatureResolveSemconvFamilies, featuretypes.NewFlaggerEvaluationContext(orgID)))
if err != nil {
r.logger.Error("Error in processing sql query", errorsV2.Attr(err))
return "", err
@@ -395,7 +396,7 @@ func (r *ClickHouseReader) buildResourceSubQuery(tags []model.TagQueryParam, svc
return resourceSubQuery, nil
}
func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.GetServicesParams) (*[]model.ServiceItem, *model.ApiError) {
func (r *ClickHouseReader) GetServices(ctx context.Context, orgID valuer.UUID, queryParams *model.GetServicesParams) (*[]model.ServiceItem, *model.ApiError) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalTraces.StringValue(),
@@ -467,7 +468,7 @@ func (r *ClickHouseReader) GetServices(ctx context.Context, queryParams *model.G
clickhouse.Named("names", ops),
)
resourceSubQuery, err := r.buildResourceSubQuery(queryParams.Tags, svc, *queryParams.Start, *queryParams.End)
resourceSubQuery, err := r.buildResourceSubQuery(ctx, orgID, queryParams.Tags, svc, *queryParams.Start, *queryParams.End)
if err != nil {
r.logger.Error("Error in processing sql query", errorsV2.Attr(err))
return
@@ -703,9 +704,9 @@ func addExistsOperator(item model.TagQuery, tagMapType string, not bool) (string
return fmt.Sprintf(" AND %s (%s)", notStr, strings.Join(tagOperatorPair, " OR ")), args
}
func (r *ClickHouseReader) GetEntryPointOperations(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, error) {
func (r *ClickHouseReader) GetEntryPointOperations(ctx context.Context, orgID valuer.UUID, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, error) {
// Step 1: Get top operations for the given service
topOps, err := r.GetTopOperations(ctx, queryParams)
topOps, err := r.GetTopOperations(ctx, orgID, queryParams)
if err != nil {
return nil, errorsV2.Wrapf(err, errorsV2.TypeInternal, errorsV2.CodeInternal, "Error in getting Top Operations")
}
@@ -757,7 +758,7 @@ func (r *ClickHouseReader) GetEntryPointOperations(ctx context.Context, queryPar
return &filtered, nil
}
func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) {
func (r *ClickHouseReader) GetTopOperations(ctx context.Context, orgID valuer.UUID, queryParams *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalTraces.StringValue(),
@@ -787,7 +788,7 @@ func (r *ClickHouseReader) GetTopOperations(ctx context.Context, queryParams *mo
r.TraceDB, r.traceTableName,
)
resourceSubQuery, err := r.buildResourceSubQuery(queryParams.Tags, queryParams.ServiceName, *queryParams.Start, *queryParams.End)
resourceSubQuery, err := r.buildResourceSubQuery(ctx, orgID, queryParams.Tags, queryParams.ServiceName, *queryParams.Start, *queryParams.End)
if err != nil {
r.logger.Error("Error in processing sql query", errorsV2.Attr(err))
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing sql query")}
@@ -858,7 +859,7 @@ func (r *ClickHouseReader) GetUsage(ctx context.Context, queryParams *model.GetU
return &usageItems, nil
}
func (r *ClickHouseReader) GetDependencyGraph(ctx context.Context, queryParams *model.GetServicesParams) (*[]model.ServiceMapDependencyResponseItem, error) {
func (r *ClickHouseReader) GetDependencyGraph(ctx context.Context, orgID valuer.UUID, queryParams *model.GetServicesParams) (*[]model.ServiceMapDependencyResponseItem, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalTraces.StringValue(),
@@ -895,7 +896,7 @@ func (r *ClickHouseReader) GetDependencyGraph(ctx context.Context, queryParams *
)
tags := createTagQueryFromTagQueryParams(queryParams.Tags)
filterQuery, filterArgs := services.BuildServiceMapQuery(tags)
filterQuery, filterArgs := services.BuildServiceMapQuery(tags, r.fl.BooleanOrEmpty(ctx, flagger.FeatureResolveSemconvFamilies, featuretypes.NewFlaggerEvaluationContext(orgID)))
query += filterQuery + " GROUP BY src, dest;"
args = append(args, filterArgs...)

View File

@@ -1128,13 +1128,19 @@ func (aH *APIHandler) registerEvent(w http.ResponseWriter, r *http.Request) {
}
func (aH *APIHandler) getTopOperations(w http.ResponseWriter, r *http.Request) {
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, err)
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
query, err := parseGetTopOperationsRequest(r)
if aH.HandleError(w, err, http.StatusBadRequest) {
return
}
result, apiErr := aH.reader.GetTopOperations(r.Context(), query)
result, apiErr := aH.reader.GetTopOperations(r.Context(), orgID, query)
if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) {
return
@@ -1145,13 +1151,20 @@ func (aH *APIHandler) getTopOperations(w http.ResponseWriter, r *http.Request) {
}
func (aH *APIHandler) getEntryPointOps(w http.ResponseWriter, r *http.Request) {
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, err)
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
query, err := parseGetTopOperationsRequest(r)
if err != nil {
render.Error(w, err)
return
}
result, apiErr := aH.reader.GetEntryPointOperations(r.Context(), query)
result, apiErr := aH.reader.GetEntryPointOperations(r.Context(), orgID, query)
if apiErr != nil {
render.Error(w, apiErr)
return
@@ -1226,12 +1239,19 @@ func (aH *APIHandler) getServicesTopLevelOps(w http.ResponseWriter, r *http.Requ
}
func (aH *APIHandler) getServices(w http.ResponseWriter, r *http.Request) {
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, err)
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
query, err := parseGetServicesRequest(r)
if aH.HandleError(w, err, http.StatusBadRequest) {
return
}
result, apiErr := aH.reader.GetServices(r.Context(), query)
result, apiErr := aH.reader.GetServices(r.Context(), orgID, query)
if apiErr != nil && aH.HandleError(w, apiErr.Err, http.StatusInternalServerError) {
return
}
@@ -1240,13 +1260,19 @@ func (aH *APIHandler) getServices(w http.ResponseWriter, r *http.Request) {
}
func (aH *APIHandler) dependencyGraph(w http.ResponseWriter, r *http.Request) {
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
render.Error(w, err)
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
query, err := parseGetServicesRequest(r)
if aH.HandleError(w, err, http.StatusBadRequest) {
return
}
result, err := aH.reader.GetDependencyGraph(r.Context(), query)
result, err := aH.reader.GetDependencyGraph(r.Context(), orgID, query)
if aH.HandleError(w, err, http.StatusBadRequest) {
return
}

View File

@@ -383,7 +383,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build
}
// build the where clause for resource table
resourceSubQuery, err := resource.BuildResourceSubQuery(DB_NAME, DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false)
resourceSubQuery, err := resource.BuildResourceSubQuery(DB_NAME, DISTRIBUTED_LOGS_V2_RESOURCE, bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false, false)
if err != nil {
return "", err
}
@@ -475,7 +475,7 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) {
}
// no values for bucket start and end
resourceSubQuery, err := resource.BuildResourceSubQuery(DB_NAME, DISTRIBUTED_LOGS_V2_RESOURCE, 0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true)
resourceSubQuery, err := resource.BuildResourceSubQuery(DB_NAME, DISTRIBUTED_LOGS_V2_RESOURCE, 0, 0, mq.Filters, mq.GroupBy, mq.AggregateAttribute, true, false)
if err != nil {
return "", err
}

View File

@@ -6,6 +6,9 @@ import (
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/SigNoz/signoz/pkg/query-service/utils"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
var resourceLogOperators = map[v3.FilterOperator]string{
@@ -30,22 +33,49 @@ var resourceLogOperators = map[v3.FilterOperator]string{
}
// buildResourceFilter builds a clickhouse filter string for resource labels
func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value interface{}) string {
func buildResourceFilter(logsOp string, key string, op v3.FilterOperator, value interface{}, members []string) string {
// for all operators except contains and like
searchKey := fmt.Sprintf("simpleJSONExtractString(labels, '%s')", key)
searchKey := fmt.Sprintf("simpleJSONExtractString(labels, %s)", querybuilder.ClickHouseStringLiteral(key))
if len(members) > 1 {
values := make([]string, 0, len(members))
for _, member := range members {
values = append(values, fmt.Sprintf("NULLIF(simpleJSONExtractString(labels, %s), '')", querybuilder.ClickHouseStringLiteral(member)))
}
searchKey = "COALESCE(" + strings.Join(values, ", ") + ", '')"
}
// for contains and like it will be case insensitive
lowerSearchKey := fmt.Sprintf("simpleJSONExtractString(lower(labels), '%s')", key)
lowerSearchKey := fmt.Sprintf("simpleJSONExtractString(lower(labels), %s)", querybuilder.ClickHouseStringLiteral(key))
if len(members) > 1 {
lowerSearchKey = "lower(" + searchKey + ")"
}
chFmtVal := utils.ClickHouseFormattedValue(value)
lowerValue := strings.ToLower(fmt.Sprintf("%s", value))
switch op {
case v3.FilterOperatorExists:
return fmt.Sprintf("simpleJSONHas(labels, '%s')", key)
case v3.FilterOperatorNotExists:
return fmt.Sprintf("not simpleJSONHas(labels, '%s')", key)
case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
exists := op == v3.FilterOperatorExists
if len(members) == 1 {
if exists {
return fmt.Sprintf("simpleJSONHas(labels, %s)", querybuilder.ClickHouseStringLiteral(key))
}
return fmt.Sprintf("not simpleJSONHas(labels, %s)", querybuilder.ClickHouseStringLiteral(key))
}
presence := make([]string, 0, len(members))
for _, member := range members {
if exists {
presence = append(presence, fmt.Sprintf("simpleJSONHas(labels, %s)", querybuilder.ClickHouseStringLiteral(member)))
} else {
presence = append(presence, fmt.Sprintf("not simpleJSONHas(labels, %s)", querybuilder.ClickHouseStringLiteral(member)))
}
}
separator := " OR "
if !exists {
separator = " AND "
}
return "(" + strings.Join(presence, separator) + ")"
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex:
return fmt.Sprintf(logsOp, searchKey, chFmtVal)
case v3.FilterOperatorContains, v3.FilterOperatorNotContains:
@@ -93,9 +123,10 @@ func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value inter
// if there are no values to filter on, return an empty string
if len(values) > 0 {
escapedKey := utils.QuoteEscapedStringForContains(key, true)
for _, v := range values {
value := utils.QuoteEscapedStringForContains(v, true)
conditions = append(conditions, fmt.Sprintf("labels %s '%%\"%s\":\"%s\"%%'", sqlOp, key, value))
conditions = append(conditions, fmt.Sprintf("labels %s '%%\"%s\":\"%s\"%%'", sqlOp, escapedKey, value))
}
return "(" + strings.Join(conditions, separator) + ")"
}
@@ -109,8 +140,34 @@ func buildIndexFilterForInOperator(key string, op v3.FilterOperator, value inter
// for like/contains we will use lower index
// we can use lower index for =, in etc but it's difficult to do it for !=, NIN etc
// if as x != "ABC" we cannot predict something like "not lower(labels) like '%%x%%abc%%'". It has it be "not lower(labels) like '%%x%%ABC%%'"
func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{}) string {
func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{}, members []string) string {
if len(members) > 1 {
// A negated hint would drop rows where another member holds the value.
switch op {
case v3.FilterOperatorNotEqual,
v3.FilterOperatorNotLike,
v3.FilterOperatorNotILike,
v3.FilterOperatorNotContains,
v3.FilterOperatorNotExists,
v3.FilterOperatorNotRegex,
v3.FilterOperatorNotIn:
return ""
}
conditions := make([]string, 0, len(members))
for _, member := range members {
if condition := buildResourceIndexFilter(member, op, value, []string{member}); condition != "" {
conditions = append(conditions, condition)
}
}
if len(conditions) == 0 {
return ""
}
return "(" + strings.Join(conditions, " OR ") + ")"
}
// not using clickhouseFormattedValue as we don't wan't the quotes
escapedKey := utils.QuoteEscapedStringForContains(key, true)
strVal := fmt.Sprintf("%s", value)
fmtValEscapedForContains := utils.QuoteEscapedStringForContains(strVal, true)
fmtValEscapedForContainsLower := strings.ToLower(fmtValEscapedForContains)
@@ -119,36 +176,36 @@ func buildResourceIndexFilter(key string, op v3.FilterOperator, value interface{
// add index filters
switch op {
case v3.FilterOperatorEqual:
return fmt.Sprintf("labels like '%%%s\":\"%s%%'", key, fmtValEscapedForContains)
return fmt.Sprintf("labels like '%%%s\":\"%s%%'", escapedKey, fmtValEscapedForContains)
case v3.FilterOperatorNotEqual:
return fmt.Sprintf("labels not like '%%%s\":\"%s%%'", key, fmtValEscapedForContains)
return fmt.Sprintf("labels not like '%%%s\":\"%s%%'", escapedKey, fmtValEscapedForContains)
case v3.FilterOperatorLike, v3.FilterOperatorILike:
return fmt.Sprintf("lower(labels) like '%%%s%%%s%%'", key, fmtValEscapedLower)
return fmt.Sprintf("lower(labels) like '%%%s%%%s%%'", escapedKey, fmtValEscapedLower)
case v3.FilterOperatorNotLike, v3.FilterOperatorNotILike:
// cannot apply not contains x%y as y can be somewhere else
return ""
case v3.FilterOperatorContains:
return fmt.Sprintf("lower(labels) like '%%%s%%%s%%'", key, fmtValEscapedForContainsLower)
return fmt.Sprintf("lower(labels) like '%%%s%%%s%%'", escapedKey, fmtValEscapedForContainsLower)
case v3.FilterOperatorNotContains:
// cannot apply not contains x%y as y can be somewhere else
return ""
case v3.FilterOperatorExists:
return fmt.Sprintf("lower(labels) like '%%%s%%'", key)
return fmt.Sprintf("lower(labels) like '%%%s%%'", escapedKey)
case v3.FilterOperatorNotExists:
return fmt.Sprintf("lower(labels) not like '%%%s%%'", key)
return fmt.Sprintf("lower(labels) not like '%%%s%%'", escapedKey)
case v3.FilterOperatorRegex, v3.FilterOperatorNotRegex:
// don't try to do anything for regex.
return ""
case v3.FilterOperatorIn, v3.FilterOperatorNotIn:
return buildIndexFilterForInOperator(key, op, value)
default:
return fmt.Sprintf("labels like '%%%s%%'", key)
return fmt.Sprintf("labels like '%%%s%%'", escapedKey)
}
}
// buildResourceFiltersFromFilterItems builds a list of clickhouse filter strings for resource labels from a FilterSet.
// It skips any filter items that are not resource attributes and checks that the operator is supported and the data type is correct.
func buildResourceFiltersFromFilterItems(fs *v3.FilterSet) ([]string, error) {
func buildResourceFiltersFromFilterItems(fs *v3.FilterSet, resolveSemconvFamilies bool) ([]string, error) {
var conditions []string
if fs == nil || len(fs.Items) == 0 {
return nil, nil
@@ -182,12 +239,20 @@ func buildResourceFiltersFromFilterItems(fs *v3.FilterSet) ([]string, error) {
}
if logsOp, ok := resourceLogOperators[op]; ok {
members := []string{keyName}
if resolveSemconvFamilies {
members = semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: keyName,
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
})
}
// the filter
if resourceFilter := buildResourceFilter(logsOp, keyName, op, value); resourceFilter != "" {
if resourceFilter := buildResourceFilter(logsOp, keyName, op, value, members); resourceFilter != "" {
conditions = append(conditions, resourceFilter)
}
// the additional filter for better usage of the index
if resourceIndexFilter := buildResourceIndexFilter(keyName, op, value); resourceIndexFilter != "" {
if resourceIndexFilter := buildResourceIndexFilter(keyName, op, value, members); resourceIndexFilter != "" {
conditions = append(conditions, resourceIndexFilter)
}
} else {
@@ -219,12 +284,12 @@ func buildResourceFiltersFromAggregateAttribute(aggregateAttribute v3.AttributeK
return ""
}
func BuildResourceSubQuery(dbName, tableName string, bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool) (string, error) {
func BuildResourceSubQuery(dbName, tableName string, bucketStart, bucketEnd int64, fs *v3.FilterSet, groupBy []v3.AttributeKey, aggregateAttribute v3.AttributeKey, isLiveTail bool, resolveSemconvFamilies bool) (string, error) {
// BUILD THE WHERE CLAUSE
var conditions []string
// only add the resource attributes to the filters here
rs, err := buildResourceFiltersFromFilterItems(fs)
rs, err := buildResourceFiltersFromFilterItems(fs, resolveSemconvFamilies)
if err != nil {
return "", err
}

View File

@@ -5,6 +5,7 @@ import (
"testing"
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
"github.com/stretchr/testify/require"
)
func Test_buildResourceFilter(t *testing.T) {
@@ -88,7 +89,7 @@ func Test_buildResourceFilter(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := buildResourceFilter(tt.args.logsOp, tt.args.key, tt.args.op, tt.args.value); got != tt.want {
if got := buildResourceFilter(tt.args.logsOp, tt.args.key, tt.args.op, tt.args.value, []string{tt.args.key}); got != tt.want {
t.Errorf("buildResourceFilter() = %v, want %v", got, tt.want)
}
})
@@ -282,7 +283,7 @@ func Test_buildResourceIndexFilter(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := buildResourceIndexFilter(tt.args.key, tt.args.op, tt.args.value); got != tt.want {
if got := buildResourceIndexFilter(tt.args.key, tt.args.op, tt.args.value, []string{tt.args.key}); got != tt.want {
t.Errorf("buildResourceIndexFilter() = %v, want %v", got, tt.want)
}
})
@@ -379,7 +380,7 @@ func Test_buildResourceFiltersFromFilterItems(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := buildResourceFiltersFromFilterItems(tt.args.fs)
got, err := buildResourceFiltersFromFilterItems(tt.args.fs, false)
if (err != nil) != tt.wantErr {
t.Errorf("buildResourceFiltersFromFilterItems() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -541,7 +542,7 @@ func Test_buildResourceSubQuery(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := BuildResourceSubQuery("signoz_logs", "distributed_logs_v2_resource", tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false)
got, err := BuildResourceSubQuery("signoz_logs", "distributed_logs_v2_resource", tt.args.bucketStart, tt.args.bucketEnd, tt.args.fs, tt.args.groupBy, tt.args.aggregateAttribute, false, false)
if (err != nil) != tt.wantErr {
t.Errorf("buildResourceSubQuery() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -552,3 +553,58 @@ func Test_buildResourceSubQuery(t *testing.T) {
})
}
}
func Test_buildResourceFilterFamily(t *testing.T) {
members := []string{"deployment.environment.name", "deployment.environment"}
require.Equal(t,
"COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') = 'production'",
buildResourceFilter("=", "deployment.environment.name", v3.FilterOperatorEqual, "production", members))
require.Equal(t,
"COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') != 'production'",
buildResourceFilter("!=", "deployment.environment.name", v3.FilterOperatorNotEqual, "production", members))
require.Equal(t,
"(simpleJSONHas(labels, 'deployment.environment.name') OR simpleJSONHas(labels, 'deployment.environment'))",
buildResourceFilter("", "deployment.environment.name", v3.FilterOperatorExists, nil, members))
require.Equal(t,
"(not simpleJSONHas(labels, 'deployment.environment.name') AND not simpleJSONHas(labels, 'deployment.environment'))",
buildResourceFilter("", "deployment.environment.name", v3.FilterOperatorNotExists, nil, members))
}
func Test_buildResourceIndexFilterFamily(t *testing.T) {
members := []string{"deployment.environment.name", "deployment.environment"}
require.Equal(t,
`(labels like '%deployment.environment.name":"production%' OR labels like '%deployment.environment":"production%')`,
buildResourceIndexFilter("deployment.environment.name", v3.FilterOperatorEqual, "production", members))
require.Equal(t, "",
buildResourceIndexFilter("deployment.environment.name", v3.FilterOperatorNotEqual, "production", members))
require.Equal(t, "",
buildResourceIndexFilter("deployment.environment.name", v3.FilterOperatorNotIn, []interface{}{"production"}, members))
}
func TestBuildResourceSubQueryFamily(t *testing.T) {
fs := &v3.FilterSet{Items: []v3.FilterItem{{
Key: v3.AttributeKey{
Key: "deployment.environment.name",
DataType: v3.AttributeKeyDataTypeString,
Type: v3.AttributeKeyTypeResource,
},
Operator: v3.FilterOperatorEqual,
Value: "production",
}}}
familyOn, err := BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", 1, 2, fs, nil, v3.AttributeKey{}, false, true)
require.NoError(t, err)
require.Contains(t, familyOn, "COALESCE(NULLIF(simpleJSONExtractString(labels, 'deployment.environment.name'), ''), NULLIF(simpleJSONExtractString(labels, 'deployment.environment'), ''), '') = 'production'")
require.Contains(t, familyOn, `(labels like '%deployment.environment.name":"production%' OR labels like '%deployment.environment":"production%')`)
familyOff, err := BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", 1, 2, fs, nil, v3.AttributeKey{}, false, false)
require.NoError(t, err)
require.Contains(t, familyOff, "simpleJSONExtractString(labels, 'deployment.environment.name') = 'production'")
require.NotContains(t, familyOff, "COALESCE")
}

View File

@@ -6,17 +6,25 @@ import (
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/SigNoz/signoz/pkg/query-service/model"
"github.com/SigNoz/signoz/pkg/semconv"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
var (
columns = map[string]struct{}{
"deployment_environment": {},
"k8s_cluster_name": {},
"k8s_namespace_name": {},
func BuildServiceMapQuery(tags []model.TagQuery, resolveSemconvFamilies bool) (string, []interface{}) {
columns := map[string]string{
"deployment_environment": "deployment_environment",
"k8s_cluster_name": "k8s_cluster_name",
"k8s_namespace_name": "k8s_namespace_name",
}
if resolveSemconvFamilies {
for _, member := range semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
Name: "deployment.environment.name",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextResource,
}) {
columns[strings.ReplaceAll(member, ".", "_")] = "deployment_environment"
}
}
)
func BuildServiceMapQuery(tags []model.TagQuery) (string, []interface{}) {
var filterQuery string
var namedArgs []interface{}
for _, tag := range tags {
@@ -24,39 +32,40 @@ func BuildServiceMapQuery(tags []model.TagQuery) (string, []interface{}) {
operator := tag.GetOperator()
value := tag.GetValues()
if _, ok := columns[key]; !ok {
column, ok := columns[key]
if !ok {
continue
}
switch operator {
case model.InOperator:
filterQuery += fmt.Sprintf(" AND %s IN @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s IN @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.NotInOperator:
filterQuery += fmt.Sprintf(" AND %s NOT IN @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s NOT IN @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.EqualOperator:
filterQuery += fmt.Sprintf(" AND %s = @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s = @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.NotEqualOperator:
filterQuery += fmt.Sprintf(" AND %s != @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s != @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, value))
case model.ContainsOperator:
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%%%s%%", value)))
case model.NotContainsOperator:
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%%%s%%", value)))
case model.StartsWithOperator:
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%s%%", value)))
case model.NotStartsWithOperator:
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", key, key)
filterQuery += fmt.Sprintf(" AND %s NOT LIKE @%s", column, key)
namedArgs = append(namedArgs, clickhouse.Named(key, fmt.Sprintf("%s%%", value)))
case model.ExistsOperator:
filterQuery += fmt.Sprintf(" AND %s IS NOT NULL", key)
filterQuery += fmt.Sprintf(" AND %s IS NOT NULL", column)
case model.NotExistsOperator:
filterQuery += fmt.Sprintf(" AND %s IS NULL", key)
filterQuery += fmt.Sprintf(" AND %s IS NULL", column)
}
}
return filterQuery, namedArgs

View File

@@ -0,0 +1,37 @@
package services
import (
"testing"
"github.com/SigNoz/signoz/pkg/query-service/model"
"github.com/stretchr/testify/require"
)
func TestBuildServiceMapQueryFamily(t *testing.T) {
newSpelling := []model.TagQuery{model.NewTagQueryString(model.TagQueryParam{
Key: "deployment.environment.name",
StringValues: []string{"production"},
Operator: model.EqualOperator,
})}
oldSpelling := []model.TagQuery{model.NewTagQueryString(model.TagQueryParam{
Key: "deployment.environment",
StringValues: []string{"production"},
Operator: model.EqualOperator,
})}
query, args := BuildServiceMapQuery(newSpelling, true)
require.Equal(t, " AND deployment_environment = @deployment_environment_name", query)
require.Len(t, args, 1)
query, args = BuildServiceMapQuery(oldSpelling, true)
require.Equal(t, " AND deployment_environment = @deployment_environment", query)
require.Len(t, args, 1)
query, args = BuildServiceMapQuery(newSpelling, false)
require.Equal(t, "", query)
require.Empty(t, args)
query, args = BuildServiceMapQuery(oldSpelling, false)
require.Equal(t, " AND deployment_environment = @deployment_environment", query)
require.Len(t, args, 1)
}

View File

@@ -282,7 +282,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.
filterSubQuery = filterSubQuery + " AND " + emptyValuesInGroupByFilter
}
resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false)
resourceSubQuery, err := resource.BuildResourceSubQuery("signoz_traces", "distributed_traces_v3_resource", bucketStart, bucketEnd, mq.Filters, mq.GroupBy, mq.AggregateAttribute, false, false)
if err != nil {
return "", err
}

View File

@@ -17,12 +17,12 @@ type Reader interface {
GetInstantQueryMetricsResult(ctx context.Context, query *model.InstantQueryMetricsParams) (*promql.Result, *stats.QueryStats, *model.ApiError)
GetQueryRangeResult(ctx context.Context, query *model.QueryRangeParams) (*promql.Result, *stats.QueryStats, *model.ApiError)
GetTopLevelOperations(ctx context.Context, start, end time.Time, services []string) (*map[string][]string, *model.ApiError)
GetEntryPointOperations(ctx context.Context, query *model.GetTopOperationsParams) (*[]model.TopOperationsItem, error)
GetServices(ctx context.Context, query *model.GetServicesParams) (*[]model.ServiceItem, *model.ApiError)
GetTopOperations(ctx context.Context, query *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError)
GetEntryPointOperations(ctx context.Context, orgID valuer.UUID, query *model.GetTopOperationsParams) (*[]model.TopOperationsItem, error)
GetServices(ctx context.Context, orgID valuer.UUID, query *model.GetServicesParams) (*[]model.ServiceItem, *model.ApiError)
GetTopOperations(ctx context.Context, orgID valuer.UUID, query *model.GetTopOperationsParams) (*[]model.TopOperationsItem, *model.ApiError)
GetUsage(ctx context.Context, query *model.GetUsageParams) (*[]model.UsageItem, error)
GetServicesList(ctx context.Context) (*[]string, error)
GetDependencyGraph(ctx context.Context, query *model.GetServicesParams) (*[]model.ServiceMapDependencyResponseItem, error)
GetDependencyGraph(ctx context.Context, orgID valuer.UUID, query *model.GetServicesParams) (*[]model.ServiceMapDependencyResponseItem, error)
GetTTL(ctx context.Context, orgID string, ttlParams *retentiontypes.GetTTLParams) (*retentiontypes.GetTTLResponseItem, *model.ApiError)
GetCustomRetentionTTL(ctx context.Context, orgID string) (*retentiontypes.GetCustomRetentionTTLResponse, error)