Compare commits

..

1 Commits

Author SHA1 Message Date
srikanthccv
7497a7b971 refactor(qb): quote field names with the ClickHouse quoting helpers
Every plain field-name literal and identifier in the statement-builder
layer now goes through querybuilder.ClickHouseStringLiteral and
querybuilder.ClickHouseIdentifier: map reads, mapContains, JSON
subcolumn paths, JSONExtractString and JSONExtractKeys on labels,
simpleJSONExtractString and simpleJSONHas on the fingerprint labels,
and the column aliases.

The helpers escape backslashes, quotes, and backticks, so a field name
that contains a quote or a backtick can no longer break out of its
literal or identifier. For every other name the output is byte
identical, and the full test suite passes unchanged.

Out of scope, on purpose: LIKE index-hint fragments and body JSON path
builders (they have pattern and path semantics, not plain literals),
and the materialized-column name builders in telemetrytypes (importing
querybuilder there is an import cycle).

Assisted-by: Claude Fable 5
2026-08-18 04:34:59 +05:30
25 changed files with 76 additions and 231 deletions

View File

@@ -1,30 +0,0 @@
#!/bin/sh
# Installs dispatcher stubs into the repository's hooks directory. Safe to run
# repeatedly; invoked automatically from the Makefile and from the frontend
# postinstall so no manual setup is required.
set -e
git rev-parse --git-dir >/dev/null 2>&1 || exit 0
# core.hooksPath shadows the hooks directory git would otherwise use, and an
# absolute value leaks one worktree's path into every other worktree.
if [ -n "$(git config --get core.hooksPath 2>/dev/null || true)" ]; then
git config --unset-all core.hooksPath 2>/dev/null || true
fi
# Resolves to the common directory for linked worktrees, which is where git
# looks for hooks regardless of which worktree the commit happens in.
hooks_dir="$(git rev-parse --git-path hooks)"
mkdir -p "$hooks_dir"
for hook in pre-commit commit-msg; do
cat >"$hooks_dir/$hook" <<EOF
#!/bin/sh
# Generated by .githooks/install.
root="\$(git rev-parse --show-toplevel)"
[ -x "\$root/.githooks/$hook" ] || exit 0
exec "\$root/.githooks/$hook" "\$@"
EOF
chmod +x "$hooks_dir/$hook"
done

View File

@@ -1,22 +0,0 @@
#!/bin/sh
set -e
staged_go_files=$(git diff --cached --name-only --diff-filter=ACM -- '*.go')
if [ -n "$staged_go_files" ]; then
if command -v gofmt >/dev/null 2>&1; then
echo "$staged_go_files" | xargs gofmt -l -w
echo "$staged_go_files" | xargs git add
else
echo "pre-commit: gofmt not found, skipping go formatting" >&2
fi
fi
staged_web_files=$(git diff --cached --name-only --diff-filter=ACM -- 'frontend/*')
if [ -n "$staged_web_files" ]; then
if [ -d frontend/node_modules ] && command -v pnpm >/dev/null 2>&1; then
cd frontend && pnpm lint-staged
else
echo "pre-commit: frontend dependencies not installed, skipping lint-staged" >&2
fi
fi

View File

@@ -35,13 +35,6 @@ DOCKERFILE_ENTERPRISE = $(SRC)/cmd/enterprise/Dockerfile
DOCKER_REGISTRY_ENTERPRISE ?= docker.io/signoz/signoz
JS_BUILD_CONTEXT = $(SRC)/frontend
##############################################################
# git hooks
##############################################################
# Hooks live outside the working tree, so they cannot be checked in. Expanded
# while this file is parsed, which wires them up on any make invocation.
GITHOOKS_INSTALL := $(shell sh $(SRC)/.githooks/install >/dev/null 2>&1 || true)
##############################################################
# directories
##############################################################

View File

@@ -1,12 +1,7 @@
#!/bin/sh
. "$(dirname "$0")/_/husky.sh"
set -e
if [ -d frontend/node_modules ] && command -v pnpm >/dev/null 2>&1; then
(cd frontend && pnpm run commitlint --edit "$1")
else
echo "commit-msg: frontend dependencies not installed, skipping commitlint" >&2
fi
cd frontend && pnpm run commitlint --edit $1
branch="$(git rev-parse --abbrev-ref HEAD)"

4
frontend/.husky/pre-commit Executable file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
. "$(dirname "$0")/_/husky.sh"
cd frontend && pnpm lint-staged

View File

@@ -20,7 +20,8 @@
"jest": "jest",
"jest:coverage": "jest --coverage",
"jest:watch": "jest --watch",
"postinstall": "pnpm i18n:generate-hash && (is-ci || sh ../.githooks/install) && node scripts/update-registry.cjs",
"postinstall": "pnpm i18n:generate-hash && (is-ci || pnpm husky:configure) && node scripts/update-registry.cjs",
"husky:configure": "cd .. && husky install frontend/.husky && cd frontend && chmod ug+x .husky/*",
"commitlint": "commitlint --edit $1",
"test": "jest",
"test:changedsince": "jest --changedSince=main --coverage --silent",
@@ -185,6 +186,7 @@
"@typescript/native-preview": "7.0.0-dev.20260430.1",
"eslint-plugin-sonarjs": "4.0.2",
"glob": "^13.0.6",
"husky": "^7.0.4",
"is-ci": "^3.0.1",
"jest-environment-jsdom": "29.7.0",
"jest-styled-components": "^7.2.0",

View File

@@ -440,6 +440,9 @@ importers:
glob:
specifier: ^13.0.6
version: 13.0.6
husky:
specifier: ^7.0.4
version: 7.0.4
is-ci:
specifier: ^3.0.1
version: 3.0.1
@@ -5526,6 +5529,11 @@ packages:
resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==}
engines: {node: '>=18.18.0'}
husky@7.0.4:
resolution: {integrity: sha512-vbaCKN2QLtP/vD4yvs6iz6hBEo6wkSzs8HpRah1Z6aGmF2KW5PdYuAd7uX5a+OyBZHBhd+TFLqgjUgytQr4RvQ==}
engines: {node: '>=12'}
hasBin: true
hyphenate-style-name@1.0.4:
resolution: {integrity: sha512-ygGZLjmXfPHj+ZWh6LwbC37l43MhfztxetbFCoYTM2VjkIUpeHgSNn7QIyVFj7YQ1Wl9Cbw5sholVJPzWvC2MQ==}
@@ -14628,6 +14636,8 @@ snapshots:
human-signals@8.0.1: {}
husky@7.0.4: {}
hyphenate-style-name@1.0.4: {}
i18next-browser-languagedetector@6.1.8:

View File

@@ -2,17 +2,14 @@ import { useTranslation } from 'react-i18next';
import { Input } from '@signozhq/ui/input';
import { Switch } from '@signozhq/ui/switch';
import { Form, Select, Space } from 'antd';
import { FeatureKeys } from 'constants/features';
import { ModalFooterTitle } from 'container/PipelinePage/styles';
import { useAppContext } from 'providers/App/App';
import { ProcessorData } from 'types/api/pipeline/def';
import { formValidationRules } from '../config';
import { ProcessorFormField } from './config';
import { processorFields, ProcessorFormField } from './config';
import CSVInput from './FormFields/CSVInput';
import JsonFlattening from './FormFields/JsonFlattening';
import { FormWrapper, PipelineIndexIcon, StyledSelect } from './styles';
import { resolveProcessorFields } from './utils';
import './styles.scss';
@@ -136,23 +133,16 @@ function ProcessorForm({
selectedProcessorData,
isAdd,
}: ProcessorFormProps): JSX.Element {
const { featureFlags } = useAppContext();
const isBodyJsonEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.USE_JSON_BODY)
?.active || false;
return (
<div className="processor-form-container">
{resolveProcessorFields(processorType, isBodyJsonEnabled).map(
(fieldData: ProcessorFormField) => (
<ProcessorFieldInput
key={fieldData.name + String(fieldData.initialValue)}
fieldData={fieldData}
selectedProcessorData={selectedProcessorData}
isAdd={isAdd}
/>
),
)}
{processorFields[processorType]?.map((fieldData: ProcessorFormField) => (
<ProcessorFieldInput
key={fieldData.name + String(fieldData.initialValue)}
fieldData={fieldData}
selectedProcessorData={selectedProcessorData}
isAdd={isAdd}
/>
))}
</div>
);
}

View File

@@ -1,24 +0,0 @@
import { processorFields, ProcessorFormField } from './config';
const BODY_PARSE_FROM = 'body';
const JSON_BODY_PARSE_FROM = 'body.message';
// With use_json_body the collector normalizes every body into a map before user
// operators run, so a parser pointed at `body` gets a map it cannot read and
// silently extracts nothing. The log text lives at body.message.
export function resolveProcessorFields(
processorType: string,
isBodyJsonEnabled: boolean,
): Array<ProcessorFormField> {
const fields = processorFields[processorType] ?? [];
if (!isBodyJsonEnabled) {
return fields;
}
return fields.map((field) =>
field.name === 'parse_from' && field.initialValue === BODY_PARSE_FROM
? { ...field, initialValue: JSON_BODY_PARSE_FROM }
: field,
);
}

View File

@@ -1,45 +0,0 @@
import { processorFields } from '../PipelineListsView/AddNewProcessor/config';
import { resolveProcessorFields } from '../PipelineListsView/AddNewProcessor/utils';
const parseFromDefault = (
fields: ReturnType<typeof resolveProcessorFields>,
): unknown => fields.find((field) => field.name === 'parse_from')?.initialValue;
describe('resolveProcessorFields', () => {
it.each(['grok_parser', 'regex_parser', 'json_parser'])(
'defaults %s parse_from to body.message when use_json_body is on',
(processorType) => {
expect(parseFromDefault(resolveProcessorFields(processorType, true))).toBe(
'body.message',
);
},
);
it.each(['grok_parser', 'regex_parser', 'json_parser'])(
'keeps %s parse_from as body when use_json_body is off',
(processorType) => {
expect(parseFromDefault(resolveProcessorFields(processorType, false))).toBe(
'body',
);
},
);
it('leaves parse_from defaults that do not point at the body alone', () => {
expect(parseFromDefault(resolveProcessorFields('time_parser', true))).toBe(
'attributes.timestamp',
);
expect(
parseFromDefault(resolveProcessorFields('severity_parser', true)),
).toBe('attributes.logLevel');
});
it('does not mutate the shared config', () => {
resolveProcessorFields('grok_parser', true);
expect(parseFromDefault(processorFields.grok_parser)).toBe('body');
});
it('returns an empty list for an unknown processor type', () => {
expect(resolveProcessorFields('does_not_exist', true)).toStrictEqual([]);
});
});

View File

@@ -88,5 +88,5 @@ func (m *fieldMapper) ColumnExpressionFor(ctx context.Context, orgID valuer.UUID
if err != nil {
return "", err
}
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
return fmt.Sprintf("%s AS %s", sqlbuilder.Escape(colName), querybuilder.ClickHouseIdentifier(field.Name)), nil
}

View File

@@ -29,7 +29,6 @@ type builderQuery[T any] struct {
telemetryStore telemetrystore.TelemetryStore
orgID valuer.UUID
stmtBuilder qbtypes.StatementBuilder[T]
queryType qbtypes.QueryType
spec qbtypes.QueryBuilderQuery[T]
variables map[string]qbtypes.VariableItem
@@ -52,7 +51,6 @@ func newBuilderQuery[T any](
telemetryStore telemetrystore.TelemetryStore,
orgID valuer.UUID,
stmtBuilder qbtypes.StatementBuilder[T],
queryType qbtypes.QueryType,
spec qbtypes.QueryBuilderQuery[T],
tr qbtypes.TimeRange,
kind qbtypes.RequestType,
@@ -64,7 +62,6 @@ func newBuilderQuery[T any](
telemetryStore: telemetryStore,
orgID: orgID,
stmtBuilder: stmtBuilder,
queryType: queryType,
spec: spec,
variables: variables,
fromMS: tr.From,
@@ -84,7 +81,7 @@ func (q *builderQuery[T]) Fingerprint() string {
// Create a deterministic fingerprint for builder queries
// This needs to include all fields that affect the query results
parts := []string{q.queryType.StringValue()}
parts := []string{"builder"}
// Add signal type
parts = append(parts, fmt.Sprintf("signal=%s", q.spec.Signal.StringValue()))

View File

@@ -3,7 +3,6 @@ package querier
import (
"strings"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -21,8 +20,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "fingerprint includes shiftby when ShiftBy field is set",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 3600,
@@ -42,8 +40,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "fingerprint includes shiftby but not other functions",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 3600,
@@ -66,8 +63,7 @@ func TestBuilderQueryFingerprint(t *testing.T) {
{
name: "no shiftby in fingerprint when ShiftBy is zero",
query: &builderQuery[qbtypes.MetricAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
kind: qbtypes.RequestTypeTimeSeries,
spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Signal: telemetrytypes.SignalMetrics,
ShiftBy: 0,
@@ -98,29 +94,6 @@ func TestBuilderQueryFingerprint(t *testing.T) {
}
}
func TestBuilderQueryFingerprintQueryType(t *testing.T) {
spec := qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
Signal: telemetrytypes.SignalTraces,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.TraceAggregation{{Expression: "count()"}},
Filter: &qbtypes.Filter{Expression: "gen_ai.request.model EXISTS"},
}
regular := &builderQuery[qbtypes.TraceAggregation]{
queryType: qbtypes.QueryTypeBuilder,
kind: qbtypes.RequestTypeTimeSeries,
spec: spec,
}
ai := &builderQuery[qbtypes.TraceAggregation]{
queryType: qbtypes.QueryTypeBuilderAI,
kind: qbtypes.RequestTypeTimeSeries,
spec: spec,
}
assert.True(t, strings.HasPrefix(regular.Fingerprint(), qbtypes.QueryTypeBuilder.StringValue()+"&"))
assert.True(t, strings.HasPrefix(ai.Fingerprint(), qbtypes.QueryTypeBuilderAI.StringValue()+"&"))
assert.NotEqual(t, regular.Fingerprint(), ai.Fingerprint())
}
func TestMakeBucketsOrder(t *testing.T) {
// Test that makeBuckets returns buckets in reverse chronological order by default
// Using milliseconds as input - need > 1 hour range to get multiple buckets

View File

@@ -305,7 +305,7 @@ func (q *querier) buildQueries(
}
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.aiTraceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryTypeBuilder:
@@ -313,7 +313,7 @@ func (q *querier) buildQueries(
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, q.traceStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
@@ -323,7 +323,7 @@ func (q *querier) buildQueries(
if spec.Source == telemetrytypes.SourceAudit {
stmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, stmtBuilder, spec, timeRange, req.RequestType, tmplVars, q.builderConfig)
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
@@ -340,9 +340,9 @@ func (q *querier) buildQueries(
if spec.Source == telemetrytypes.SourceMeter {
event.Source = telemetrytypes.SourceMeter.StringValue()
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.meterStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
} else {
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, query.Type, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
bq = newBuilderQuery(q.logger, q.telemetryStore, orgID, q.metricStmtBuilder, spec, timeRange, req.RequestType, tmplVars, builderConfig{})
}
queries[spec.Name] = bq
@@ -618,7 +618,7 @@ func (q *querier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qb
if spec.Source == telemetrytypes.SourceAudit {
liveTailStmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, liveTailStmtBuilder, query.Type, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
bq := newBuilderQuery(q.logger, q.telemetryStore, orgID, liveTailStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
"id": {
Value: updatedLogID,
},
@@ -941,9 +941,8 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
// reuse the original query's statement builder and type so an AI query
// keeps its AI builder and cache key
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, qt.builderConfig)
// reuse the original query's statement builder so an AI query keeps its AI builder
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, qt.stmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
case *builderQuery[qbtypes.LogAggregation]:
specCopy := qt.spec.Copy()
@@ -953,16 +952,16 @@ func (q *querier) createRangedQuery(_ valuer.UUID, originalQuery qbtypes.Query,
if qt.spec.Source == telemetrytypes.SourceAudit {
shiftStmtBuilder = q.auditStmtBuilder
}
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, q.builderConfig)
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, shiftStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, q.builderConfig)
case *builderQuery[qbtypes.MetricAggregation]:
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
if qt.spec.Source == telemetrytypes.SourceMeter {
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.meterStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.meterStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
}
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.metricStmtBuilder, qt.queryType, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
return newBuilderQuery(q.logger, q.telemetryStore, qt.orgID, q.metricStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables, builderConfig{})
case *traceOperatorQuery:
specCopy := qt.spec.Copy()
return &traceOperatorQuery{

View File

@@ -43,7 +43,7 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil {
columnName = evolutionsEntries[0].ColumnName
}
rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name)
rawPath := fmt.Sprintf("%s.%s", columnName, ClickHouseIdentifier(key.Name))
if exists {
return rawPath + " IS NOT NULL", nil
}
@@ -88,7 +88,7 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
leftOperand := fmt.Sprintf("mapContains(%s, %s)", column.Name, ClickHouseStringLiteral(key.Name))
if key.Materialized {
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
}

View File

@@ -96,7 +96,7 @@ func valueIndexCondition(
func memberPresenceCondition(sb *sqlbuilder.SelectBuilder, column string, members []*telemetrytypes.TelemetryFieldKey, exists bool) string {
conditions := make([]string, 0, len(members))
for _, member := range members {
field := fmt.Sprintf("simpleJSONHas(%s, '%s')", column, member.Name)
field := fmt.Sprintf("simpleJSONHas(%s, %s)", column, querybuilder.ClickHouseStringLiteral(member.Name))
if exists {
conditions = append(conditions, sb.E(field, true))
} else {

View File

@@ -5,6 +5,7 @@ import (
"fmt"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -66,7 +67,7 @@ func (m *defaultFieldMapper) FieldFor(
return "", err
}
if key.FieldContext == telemetrytypes.FieldContextResource {
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
return fmt.Sprintf("simpleJSONExtractString(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return columns[0].Name, nil
}
@@ -91,7 +92,7 @@ func (m *defaultFieldMapper) ExistsFor(
}
return "false", nil
}
pred := fmt.Sprintf("simpleJSONHas(%s, '%s')", columns[0].Name, key.Name)
pred := fmt.Sprintf("simpleJSONHas(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name))
if exists {
return pred, nil
}
@@ -110,5 +111,5 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
if err != nil {
return "", err
}
return fmt.Sprintf("%s AS `%s`", fieldExpression, key.Name), nil
return fmt.Sprintf("%s AS %s", fieldExpression, querybuilder.ClickHouseIdentifier(key.Name)), nil
}

View File

@@ -168,7 +168,7 @@ func (c *conditionBuilder) conditionForKey(
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
ValueType: schema.ColumnTypeString,
}:
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
leftOperand := fmt.Sprintf("mapContains(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name))
if operator == qbtypes.FilterOperatorExists {
cond = sb.E(leftOperand, true)
} else {

View File

@@ -7,6 +7,7 @@ import (
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -64,7 +65,7 @@ func (m *fieldMapper) ExistsFor(ctx context.Context, _ valuer.UUID, tsStart, tsE
if err != nil {
return "", err
}
pred := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
pred := fmt.Sprintf("mapContains(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name))
if exists {
return pred, nil
}
@@ -82,7 +83,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endN
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
ValueType: schema.ColumnTypeString,
}:
return fmt.Sprintf("%s['%s']", columns[0].Name, key.Name), nil
return fmt.Sprintf("%s[%s]", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return columns[0].Name, nil
}
@@ -130,5 +131,5 @@ func (m *fieldMapper) ColumnExpressionFor(
}
}
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
return fmt.Sprintf("%s AS %s", sqlbuilder.Escape(fieldExpression), querybuilder.ClickHouseIdentifier(field.Name)), nil
}

View File

@@ -68,7 +68,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
if key.FieldContext != telemetrytypes.FieldContextResource {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "only resource context fields are supported for json columns in audit, got %s", key.FieldContext.String)
}
return fmt.Sprintf("%s.`%s`::String", column.Name, key.Name), nil
return fmt.Sprintf("%s.%s::String", column.Name, querybuilder.ClickHouseIdentifier(key.Name)), nil
case schema.ColumnTypeEnumLowCardinality:
return column.Name, nil
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
@@ -84,7 +84,7 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, _, _ uint64,
if key.Materialized {
return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
}
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
return fmt.Sprintf("%s[%s]", column.Name, querybuilder.ClickHouseStringLiteral(key.Name)), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported map value type %s", valueType)
}
@@ -156,7 +156,7 @@ func (m *fieldMapper) ColumnExpressionFor(
return fmt.Sprintf("multiIf(%s, %s, NULL)", guard, coerced), nil
}
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
return fmt.Sprintf("%s AS %s", sqlbuilder.Escape(fieldExpression), querybuilder.ClickHouseIdentifier(field.Name)), nil
}
// CandidateKeys returns nil: audit has no synthesize-on-unknown-key fallback, so an

View File

@@ -141,8 +141,8 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
case schema.ColumnTypeEnumJSON:
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
exprs = append(exprs, fmt.Sprintf("%s.%s::String", columnName, querybuilder.ClickHouseIdentifier(key.Name)))
existExpr = append(existExpr, fmt.Sprintf("%s.%s IS NOT NULL", columnName, querybuilder.ClickHouseIdentifier(key.Name)))
case telemetrytypes.FieldContextBody:
if key.Name == messageSubField {
exprs = append(exprs, messageSubColumn)
@@ -186,8 +186,8 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
existExpr = append(existExpr, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
} else {
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
exprs = append(exprs, fmt.Sprintf("%s[%s]", columnName, querybuilder.ClickHouseStringLiteral(key.Name)))
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, %s)", columnName, querybuilder.ClickHouseStringLiteral(key.Name)))
}
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
@@ -415,7 +415,7 @@ func (m *fieldMapper) buildFieldForJSON(key *telemetrytypes.TelemetryFieldKey) (
elemType = telemetrytypes.String
}
fieldPath := fmt.Sprintf("%s.`%s`", LogsV2BodyV2Column, key.Name)
fieldPath := fmt.Sprintf("%s.%s", LogsV2BodyV2Column, querybuilder.ClickHouseIdentifier(key.Name))
return fmt.Sprintf("dynamicElement(%s, '%s')", fieldPath, elemType.StringValue()), nil
}

View File

@@ -41,7 +41,7 @@ func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperato
// path index
if operator.AddDefaultExistsFilter() {
pathIndex := fmt.Sprintf(`has(%s, '%s')`, schemamigrator.JSONPathsIndexExpr(LogsV2BodyV2Column), c.key.ArrayParentPaths()[0])
pathIndex := fmt.Sprintf(`has(%s, %s)`, schemamigrator.JSONPathsIndexExpr(LogsV2BodyV2Column), querybuilder.ClickHouseStringLiteral(c.key.ArrayParentPaths()[0]))
return sb.And(baseCond, pathIndex), nil
}

View File

@@ -136,9 +136,9 @@ func (c *conditionBuilder) conditionFor(
}
if operator == qbtypes.FilterOperatorExists {
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
return fmt.Sprintf("has(JSONExtractKeys(labels), %s)", querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
return fmt.Sprintf("not has(JSONExtractKeys(labels), %s)", querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
}

View File

@@ -6,6 +6,7 @@ import (
"slices"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -79,14 +80,14 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endN
switch key.FieldContext {
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
return fmt.Sprintf("JSONExtractString(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name)), nil
case telemetrytypes.FieldContextMetric:
return columns[0].Name, nil
case telemetrytypes.FieldContextUnspecified:
if slices.Contains(IntrinsicFields, key.Name) {
return columns[0].Name, nil
}
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
return fmt.Sprintf("JSONExtractString(%s, %s)", columns[0].Name, querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return columns[0].Name, nil
@@ -103,9 +104,9 @@ func (m *fieldMapper) ExistsFor(_ context.Context, _ valuer.UUID, _, _ uint64, k
return "true", nil
}
if exists {
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
return fmt.Sprintf("has(JSONExtractKeys(labels), %s)", querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
return fmt.Sprintf("not has(JSONExtractKeys(labels), %s)", querybuilder.ClickHouseStringLiteral(key.Name)), nil
}
func (m *fieldMapper) ColumnExpressionFor(

View File

@@ -298,8 +298,8 @@ func (m *fieldMapper) resolveColumnExprs(
}
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
// once clickHouse dependency is updated, we need to check if we can remove it.
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
exprs = append(exprs, fmt.Sprintf("%s.%s::String", columnName, querybuilder.ClickHouseIdentifier(key.Name)))
existExprs = append(existExprs, fmt.Sprintf("%s.%s IS NOT NULL", columnName, querybuilder.ClickHouseIdentifier(key.Name)))
case schema.ColumnTypeEnumString,
schema.ColumnTypeEnumUInt64,
schema.ColumnTypeEnumUInt32,
@@ -329,8 +329,8 @@ func (m *fieldMapper) resolveColumnExprs(
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
} else {
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
exprs = append(exprs, fmt.Sprintf("%s[%s]", columnName, querybuilder.ClickHouseStringLiteral(key.Name)))
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, %s)", columnName, querybuilder.ClickHouseStringLiteral(key.Name)))
}
default:
return nil, nil, nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "value type %s is not supported for map column type %s", valueType, column.Type)