Compare commits

...

6 Commits

Author SHA1 Message Date
Piyush Singariya
4bac933968 chore: remove unused var 2026-06-02 19:28:20 +05:30
Piyush Singariya
8df70ed4be chore: minor changes 2026-06-02 19:27:25 +05:30
Piyush Singariya
806f1d0062 fix: direct use function 2026-06-02 19:23:25 +05:30
Piyush Singariya
19d7d5b2ad Merge branch 'fts-logs' into fts-3 2026-06-02 18:58:20 +05:30
Piyush Singariya
8861f4523c fix: new GetColumns fieldmapper method 2026-06-02 18:25:59 +05:30
Piyush Singariya
7f97249de3 fix: create top level function 2026-06-02 17:59:24 +05:30
18 changed files with 115 additions and 142 deletions

View File

@@ -104,3 +104,4 @@ func (c *conditionBuilder) ConditionFor(
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
}

View File

@@ -120,6 +120,7 @@ func newProvider(
logAggExprRewriter,
telemetrylogs.DefaultFullTextColumn,
telemetrylogs.GetBodyJSONKey,
logConditionBuilder.ConditionForSearch,
flagger,
)

View File

@@ -88,6 +88,7 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
logAggExprRewriter,
telemetrylogs.DefaultFullTextColumn,
telemetrylogs.GetBodyJSONKey,
logConditionBuilder.ConditionForSearch,
fl,
)

View File

@@ -12,14 +12,6 @@ const (
// FullTextSearchDefaultWarning is emitted when a search() function call is used.
FullTextSearchDefaultWarning = "Full text searches across all fields and will be slow and expensive. Consider using specific field to search e.g. <context>.<field_key>:<type>"
// FTSInternalKey is the sentinel Name on TelemetryFieldKey instances that represent
// wildcard map searches (all attribute/resource keys+values). The unconventional value
// prevents collision with any real field name a user could type.
FTSInternalKey = "_X_INTERNAL_FTS_KEY"
// SearchFunctionName is the grammar function name for full-text search.
SearchFunctionName = "search"
// FTSMaxWindowNs is the maximum allowed time range for a search() query (6 hours).
FTSMaxWindowNs = uint64(6 * 60 * 60 * 1_000_000_000)
)

View File

@@ -47,7 +47,7 @@ type filterExpressionVisitor struct {
keysWithWarnings map[string]bool
startNs uint64
endNs uint64
ftsFieldKeys []*telemetrytypes.TelemetryFieldKey
ftsCondition qbtypes.FTSConditionFunc
}
type FilterExprVisitorOpts struct {
@@ -59,6 +59,7 @@ type FilterExprVisitorOpts struct {
Builder *sqlbuilder.SelectBuilder
FreeTextColumn *telemetrytypes.TelemetryFieldKey
JsonKeyToKey qbtypes.JsonKeyToFieldFunc
FTSCondition qbtypes.FTSConditionFunc
BodyJSONEnabled bool
SkipResourceFilter bool
SkipFreeTextFilter bool
@@ -68,9 +69,6 @@ type FilterExprVisitorOpts struct {
Variables map[string]qbtypes.VariableItem
StartNs uint64
EndNs uint64
// FTSFieldKeys enables search() for this query context. nil disables search()
// (traces, metrics, and non-log callers leave this nil).
FTSFieldKeys []*telemetrytypes.TelemetryFieldKey
}
// newFilterExpressionVisitor creates a new filterExpressionVisitor.
@@ -84,6 +82,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
builder: opts.Builder,
freeTextColumn: opts.FreeTextColumn,
jsonKeyToKey: opts.JsonKeyToKey,
ftsCondition: opts.FTSCondition,
bodyJSONEnabled: opts.BodyJSONEnabled,
skipResourceFilter: opts.SkipResourceFilter,
skipFreeTextFilter: opts.SkipFreeTextFilter,
@@ -94,7 +93,6 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
keysWithWarnings: make(map[string]bool),
startNs: opts.StartNs,
endNs: opts.EndNs,
ftsFieldKeys: opts.FTSFieldKeys,
}
}
@@ -750,9 +748,9 @@ func (v *filterExpressionVisitor) VisitSearchCall(ctx *grammar.SearchCallContext
if v.skipFunctionCalls || v.skipFullTextSearch {
return SkipConditionLiteral
}
// ftsFieldKeys == nil means search() is not enabled for this signal/query type.
// Only log statement builders set FTSFieldKeys; traces/metrics leave it nil.
if len(v.ftsFieldKeys) == 0 {
// ftsCondition nil means search() is not enabled for this signal.
// Only log statement builders set these; traces/metrics leave them nil.
if v.ftsCondition == nil {
v.errors = append(v.errors, "search() is only supported for log queries")
return ErrorConditionLiteral
}
@@ -782,22 +780,14 @@ func (v *filterExpressionVisitor) VisitSearchCall(ctx *grammar.SearchCallContext
}
formattedText := FormatFullTextSearch(searchText)
var ftsConds []string
for _, key := range v.ftsFieldKeys {
cond, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, qbtypes.FilterOperatorRegexp, formattedText, v.builder)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("search() could not build condition for field %q: %s", key.Text(), err.Error()))
return ErrorConditionLiteral
}
ftsConds = append(ftsConds, cond)
}
if len(ftsConds) == 0 {
v.errors = append(v.errors, "search() failed: no searchable fields could be queried")
cond, err := v.ftsCondition(v.context, formattedText, v.builder)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("search() could not build condition: %s", err.Error()))
return ErrorConditionLiteral
}
v.warnings = append(v.warnings, FullTextSearchDefaultWarning)
return v.builder.Or(ftsConds...)
return cond
}
// VisitFunctionCall handles function calls like has(), hasAny(), hasAll(), hasToken().

View File

@@ -755,7 +755,6 @@ func (b *conditionBuilder) ConditionFor(
_ any,
_ *sqlbuilder.SelectBuilder,
) (string, error) {
return fmt.Sprintf("%s_cond", key.Name), nil
}
@@ -1686,14 +1685,9 @@ func TestVisitComparison_UnknownKeys(t *testing.T) {
}
// TestVisitComparison_FullTextSearch covers Full Text Search — the explicit search()
// function that fans out across all ftsFieldKeys. FTSFieldKeys must be set for
// function that fans out across all FTSSet columns. FTSSet must be set for
// search() to be enabled; invalid param counts must error.
func TestVisitComparison_FullTextSearch(t *testing.T) {
bodyKey := &telemetrytypes.TelemetryFieldKey{
Name: "body",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
ftsOpts := FilterExprVisitorOpts{
Context: t.Context(),
FieldKeys: visitTestKeys,
@@ -1702,7 +1696,9 @@ func TestVisitComparison_FullTextSearch(t *testing.T) {
SkipFreeTextFilter: false,
SkipFunctionCalls: false,
IgnoreNotFoundKeys: false,
FTSFieldKeys: []*telemetrytypes.TelemetryFieldKey{bodyKey},
FTSCondition: func(_ context.Context, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return fmt.Sprintf("fts_cond"), nil
},
}
tests := []struct {

View File

@@ -198,3 +198,4 @@ func (c *conditionBuilder) ConditionFor(
return condition, nil
}

View File

@@ -16,10 +16,6 @@ import (
"github.com/huandu/go-sqlbuilder"
)
var (
ErrCodeInvalidFTSOperator = errors.MustNewCode("invalid_fts_operator")
)
type conditionBuilder struct {
fm qbtypes.FieldMapper
fl flagger.Flagger
@@ -29,71 +25,47 @@ func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionB
return &conditionBuilder{fm: fm, fl: fl}
}
// buildFTSCondition produces the WHERE fragment for a single FTS key by iterating
// over its resolved columns and emitting the right match expression per column type:
// - Map → arrayExists(x -> match(x, ?), mapKeys/mapValues(col))
// - JSON → match(LOWER(toString(col)), LOWER(?)) — only when useJSONBody is true
// - String/LowCardinality → match(LOWER(col), LOWER(?))
//
// Evolution entries (key.Evolutions) are applied first: selectEvolutionsForColumns
// picks the active column set for the query time range, and the ColumnName from each
// entry overrides col.Name so evolved table layouts use the right physical column.
// JSON columns are then skipped when useJSONBody is false.
func buildFTSCondition(
columns []*schema.Column,
key *telemetrytypes.TelemetryFieldKey,
tsStart, tsEnd uint64,
// ConditionForSearch builds the search condition for all FTS columns belonging
// to fieldContext. JSON columns are skipped when useJSONBody is disabled.
// Returns ("", nil) when no searchable columns exist for the context.
func (c *conditionBuilder) ConditionForSearch(
ctx context.Context,
value any,
useJSONBody bool,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
activeColumns := columns
var evolutionEntries []*telemetrytypes.EvolutionEntry
if len(key.Evolutions) > 0 {
var err error
activeColumns, evolutionEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
if err != nil {
return "", err
}
contexts := []telemetrytypes.FieldContext{
telemetrytypes.FieldContextLog,
telemetrytypes.FieldContextBody,
telemetrytypes.FieldContextAttribute,
telemetrytypes.FieldContextResource,
}
useJSONBody := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
var conditions []string
for i, col := range activeColumns {
if !useJSONBody && col.Type.GetType() == schema.ColumnTypeEnumJSON {
continue
}
colName := col.Name
if evolutionEntries != nil && evolutionEntries[i] != nil {
colName = evolutionEntries[i].ColumnName
}
switch col.Type.GetType() {
case schema.ColumnTypeEnumMap:
keysExpr := fmt.Sprintf("mapKeys(%s)", colName)
valsExpr := fmt.Sprintf("mapValues(%s)", colName)
if mc, ok := col.Type.(schema.MapColumnType); ok && mc.ValueType.GetType() != schema.ColumnTypeEnumString {
valsExpr = fmt.Sprintf("arrayMap(x -> toString(x), mapValues(%s))", colName)
for _, fieldContext := range contexts {
for _, col := range ftsColumns(ctx, fieldContext, useJSONBody) {
switch col.Type.GetType() {
case schema.ColumnTypeEnumMap:
keysExpr := fmt.Sprintf("mapKeys(%s)", col.Name)
valsExpr := fmt.Sprintf("mapValues(%s)", col.Name)
if mc, ok := col.Type.(schema.MapColumnType); ok && mc.ValueType.GetType() != schema.ColumnTypeEnumString {
valsExpr = fmt.Sprintf("arrayMap(x -> toString(x), mapValues(%s))", col.Name)
}
conditions = append(conditions, sb.Or(
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), keysExpr),
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), valsExpr),
))
case schema.ColumnTypeEnumJSON:
conditions = append(conditions, fmt.Sprintf(`match(LOWER(toString(%s)), LOWER(%s))`, col.Name, sb.Var(value)))
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumLowCardinality:
conditions = append(conditions, fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, col.Name, sb.Var(value)))
default:
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "FTS unsupported column type %s", col.Type)
}
conditions = append(conditions,
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), keysExpr),
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), valsExpr),
)
case schema.ColumnTypeEnumJSON:
conditions = append(conditions,
fmt.Sprintf(`match(LOWER(toString(%s)), LOWER(%s))`, colName, sb.Var(value)))
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumLowCardinality:
conditions = append(conditions,
fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, colName, sb.Var(value)))
}
}
if len(conditions) == 0 {
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "no FTS conditions built for columns")
}
if len(conditions) == 1 {
return conditions[0], nil
}
return sb.Or(conditions...), nil
}
@@ -105,22 +77,11 @@ func (c *conditionBuilder) conditionFor(
value any,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
if key.Name == querybuilder.FTSInternalKey && operator != qbtypes.FilterOperatorRegexp {
return "", errors.NewInternalf(ErrCodeInvalidFTSOperator, "only regexp operator is supported for fts")
}
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
if err != nil {
return "", err
}
// FTS is branched here
if key.Name == querybuilder.FTSInternalKey {
// TODO(Tushar): thread orgID here to evaluate correctly
useJSONBody := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
return buildFTSCondition(columns, key, startNs, endNs, value, useJSONBody, sb)
}
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
for _, column := range columns {
// TODO(Tushar): thread orgID here to evaluate correctly
@@ -365,12 +326,6 @@ func (c *conditionBuilder) ConditionFor(
return "", err
}
// FTS wildcard conditions are self-contained (arrayExists over full map);
// no additional EXISTS wrapper is needed.
if key.Name == querybuilder.FTSInternalKey {
return condition, nil
}
// Skip adding exists filter for intrinsic fields i.e. Table level log context fields
buildExistCondition := operator.AddDefaultExistsFilter()
switch key.FieldContext {

View File

@@ -4,7 +4,6 @@ import (
"fmt"
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
@@ -35,6 +34,7 @@ const (
LogsV2AttributesNumberColumn = "attributes_number"
LogsV2AttributesBoolColumn = "attributes_bool"
LogsV2ResourcesStringColumn = "resources_string"
LogsV2ResourceJSONColumn = "resource"
LogsV2ScopeStringColumn = "scope_string"
BodyV2ColumnPrefix = constants.BodyV2ColumnPrefix
@@ -126,21 +126,6 @@ var (
Direction: qbtypes.OrderDirectionDesc,
},
}
// defaultFTSFieldKeys is set of TelemetryFieldKey instances that
// search() fans out across. Intrinsic log columns use the normal conditionFor
// path; entries with Name==FTSInternalKey are short-circuited in conditionFor
// to emit arrayExists conditions over mapKeys/mapValues without arrayConcat.
defaultFTSFieldKeys = []*telemetrytypes.TelemetryFieldKey{
{Name: LogsV2SeverityTextColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: LogsV2TraceIDColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: LogsV2SpanIDColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextBody, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeBool},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
}
)
func bodyAliasExpression(bodyJSONEnabled bool) string {

View File

@@ -543,3 +543,35 @@ func (m *fieldMapper) buildArrayMap(currentNode *telemetrytypes.JSONAccessNode,
return fmt.Sprintf("arrayMap(%s->%s, %s)", currentNode.Alias(), nestedExpr, arrayExpr), nil
}
// GetColumns returns the physical columns for the given field context that
// search() fans out across. For FieldContextBody the active column set depends
// on the useJSONBody feature flag.
func ftsColumns(ctx context.Context, fieldContext telemetrytypes.FieldContext, useJSONBody bool) []*schema.Column {
switch fieldContext {
case telemetrytypes.FieldContextLog:
return []*schema.Column{
logsV2Columns[LogsV2SeverityTextColumn],
logsV2Columns[LogsV2TraceIDColumn],
logsV2Columns[LogsV2SpanIDColumn],
}
case telemetrytypes.FieldContextBody:
if useJSONBody {
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}
}
return []*schema.Column{logsV2Columns[LogsV2BodyColumn]}
case telemetrytypes.FieldContextAttribute:
return []*schema.Column{
logsV2Columns[LogsV2AttributesStringColumn],
logsV2Columns[LogsV2AttributesNumberColumn],
logsV2Columns[LogsV2AttributesBoolColumn],
}
case telemetrytypes.FieldContextResource:
return []*schema.Column{
logsV2Columns[LogsV2ResourcesStringColumn],
logsV2Columns[LogsV2ResourceJSONColumn],
}
default:
return nil
}
}

View File

@@ -1204,6 +1204,7 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)

View File

@@ -27,9 +27,9 @@ type logQueryStatementBuilder struct {
aggExprRewriter qbtypes.AggExprRewriter
fl flagger.Flagger
freeTextColumn *telemetrytypes.TelemetryFieldKey
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
ftsFieldKeys []*telemetrytypes.TelemetryFieldKey
freeTextColumn *telemetrytypes.TelemetryFieldKey
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
ftsConditionFunc qbtypes.FTSConditionFunc
}
var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*logQueryStatementBuilder)(nil)
@@ -42,6 +42,7 @@ func NewLogQueryStatementBuilder(
aggExprRewriter qbtypes.AggExprRewriter,
freeTextColumn *telemetrytypes.TelemetryFieldKey,
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
ftsConditionFunc qbtypes.FTSConditionFunc,
fl flagger.Flagger,
) *logQueryStatementBuilder {
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
@@ -66,9 +67,9 @@ func NewLogQueryStatementBuilder(
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
aggExprRewriter: aggExprRewriter,
fl: fl,
freeTextColumn: freeTextColumn,
jsonKeyToKey: jsonKeyToKey,
ftsFieldKeys: defaultFTSFieldKeys,
freeTextColumn: freeTextColumn,
jsonKeyToKey: jsonKeyToKey,
ftsConditionFunc: ftsConditionFunc,
}
}
@@ -665,7 +666,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
EndNs: end,
}
if enableFTS {
opts.FTSFieldKeys = b.ftsFieldKeys
opts.FTSCondition = b.ftsConditionFunc
}
// add filter expression

View File

@@ -212,6 +212,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -353,6 +354,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -483,6 +485,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -559,6 +562,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -654,6 +658,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -878,6 +883,7 @@ func TestAdjustKey(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -1023,6 +1029,7 @@ func TestStmtBuilderBodyField(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
@@ -1111,9 +1118,9 @@ func TestStmtBuilderTextSearch(t *testing.T) {
},
},
// ── Full Text Search ──────────────────────────────────────────────────────────
// search() fans out via ftsFieldKeys across all fields (severity_text, trace_id,
// span_id, body, attributes_*, resources_*). Uses a 2-hour window to stay under
// the 6-hour limit.
// search() fans out via ftsSupportedContexts (log, body, attribute, resource).
// Each context returns one OR-combined condition from ConditionForContext.
// Uses a 2-hour window to stay under the 6-hour limit.
{
name: "search_fans_out_to_all_columns",
requestType: qbtypes.RequestTypeRaw,
@@ -1126,7 +1133,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
startMs: 1705309200000,
endMs: 1705316400000,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
},
@@ -1143,7 +1150,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
startMs: 1705309200000,
endMs: 1705316400000,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE NOT ((match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string))))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE NOT (((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string))))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
},
@@ -1160,7 +1167,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
startMs: 1705309200000,
endMs: 1705316400000,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND severity_text = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND severity_text = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "ERROR", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
},
@@ -1199,6 +1206,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
cb.ConditionForSearch,
fl,
)
startMs := uint64(1747947419000)

View File

@@ -135,3 +135,4 @@ func (c *conditionBuilder) ConditionFor(
return fmt.Sprintf(expr, columns[0].Name, sb.Var(key.Name), cond), nil
}

View File

@@ -157,3 +157,4 @@ func (c *conditionBuilder) ConditionFor(
return condition, nil
}

View File

@@ -206,3 +206,4 @@ func (b *defaultConditionBuilder) ConditionFor(
}
return "", qbtypes.ErrUnsupportedOperator
}

View File

@@ -265,6 +265,7 @@ func (c *conditionBuilder) ConditionFor(
return condition, nil
}
func (c *conditionBuilder) isSpanScopeField(name string) bool {
keyName := strings.ToLower(name)
return keyName == SpanSearchScopeRoot || keyName == SpanSearchScopeEntryPoint

View File

@@ -19,6 +19,11 @@ var (
type JsonKeyToFieldFunc func(context.Context, *telemetrytypes.TelemetryFieldKey, FilterOperator, any) (string, any)
// FTSConditionFunc builds the search() SQL condition for all FTS columns belonging
// to fieldContext. Returns ("", nil) when no columns are searchable for the
// context (e.g. body JSON when useJSONBody is off). Pass nil to disable search().
type FTSConditionFunc func(ctx context.Context, value any, sb *sqlbuilder.SelectBuilder) (string, error)
// FieldMapper maps the telemetry field key to the table field name.
type FieldMapper interface {
// FieldFor returns the field name for the given key.