Compare commits

..

2 Commits

Author SHA1 Message Date
Piyush Singariya
8861f4523c fix: new GetColumns fieldmapper method 2026-06-02 18:25:59 +05:30
Piyush Singariya
7f97249de3 fix: create top level function 2026-06-02 17:59:24 +05:30
22 changed files with 165 additions and 131 deletions

View File

@@ -112,7 +112,7 @@ functionCall
;
/*
* Search call: search() function
* Search call: search(field) or search("query text")
* First-class rule so search-specific semantics can be extended independently.
*/
searchCall

View File

@@ -104,3 +104,7 @@ func (c *conditionBuilder) ConditionFor(
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
}
func (c *conditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}

View File

@@ -64,3 +64,7 @@ func (m *fieldMapper) ColumnExpressionFor(ctx context.Context, tsStart, tsEnd ui
}
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
}
func (m *fieldMapper) GetColumns(_ context.Context, _ telemetrytypes.FieldContext) []*schema.Column {
return nil
}

View File

@@ -38,8 +38,7 @@ type filterExpressionVisitor struct {
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
bodyJSONEnabled bool
skipResourceFilter bool
skipFreeTextFilter bool
skipFullTextSearch bool
skipFullTextFilter bool
skipFunctionCalls bool
ignoreNotFoundKeys bool
variables map[string]qbtypes.VariableItem
@@ -47,7 +46,7 @@ type filterExpressionVisitor struct {
keysWithWarnings map[string]bool
startNs uint64
endNs uint64
ftsFieldKeys []*telemetrytypes.TelemetryFieldKey
ftsContexts []telemetrytypes.FieldContext
}
type FilterExprVisitorOpts struct {
@@ -61,16 +60,15 @@ type FilterExprVisitorOpts struct {
JsonKeyToKey qbtypes.JsonKeyToFieldFunc
BodyJSONEnabled bool
SkipResourceFilter bool
SkipFreeTextFilter bool
SkipFullTextSearch bool
SkipFullTextFilter bool
SkipFunctionCalls bool
IgnoreNotFoundKeys bool
Variables map[string]qbtypes.VariableItem
StartNs uint64
EndNs uint64
// FTSFieldKeys enables search() for this query context. nil disables search()
// FTSContexts enables search() for this query context. nil disables search()
// (traces, metrics, and non-log callers leave this nil).
FTSFieldKeys []*telemetrytypes.TelemetryFieldKey
FTSContexts []telemetrytypes.FieldContext
}
// newFilterExpressionVisitor creates a new filterExpressionVisitor.
@@ -86,15 +84,14 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
jsonKeyToKey: opts.JsonKeyToKey,
bodyJSONEnabled: opts.BodyJSONEnabled,
skipResourceFilter: opts.SkipResourceFilter,
skipFreeTextFilter: opts.SkipFreeTextFilter,
skipFullTextSearch: opts.SkipFullTextSearch,
skipFullTextFilter: opts.SkipFullTextFilter,
skipFunctionCalls: opts.SkipFunctionCalls,
ignoreNotFoundKeys: opts.IgnoreNotFoundKeys,
variables: opts.Variables,
keysWithWarnings: make(map[string]bool),
startNs: opts.StartNs,
endNs: opts.EndNs,
ftsFieldKeys: opts.FTSFieldKeys,
ftsContexts: opts.FTSContexts,
}
}
@@ -345,12 +342,12 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
// Handle standalone key/value as a full text search term
if ctx.GetChildCount() == 1 {
if v.skipFreeTextFilter {
if v.skipFullTextFilter {
return SkipConditionLiteral
}
if v.freeTextColumn == nil {
v.errors = append(v.errors, "free text search is not supported")
v.errors = append(v.errors, "full text search is not supported")
return ErrorConditionLiteral
}
@@ -713,7 +710,7 @@ func (v *filterExpressionVisitor) VisitValueList(ctx *grammar.ValueListContext)
// VisitFullText handles standalone quoted strings for full-text search.
func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) any {
if v.skipFreeTextFilter {
if v.skipFullTextFilter {
// A skipped FT term must be treated as TrueConditionLiteral, not "".
// Returning "" would silently drop this branch from an OR, incorrectly
// excluding rows that could match the FT condition on the real table.
@@ -729,7 +726,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
}
if v.freeTextColumn == nil {
v.errors = append(v.errors, "free text search is not supported")
v.errors = append(v.errors, "full text search is not supported")
return ErrorConditionLiteral
}
cond, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, v.freeTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder)
@@ -747,12 +744,12 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
// VisitSearchCall handles the search() function call.
func (v *filterExpressionVisitor) VisitSearchCall(ctx *grammar.SearchCallContext) any {
if v.skipFunctionCalls || v.skipFullTextSearch {
if v.skipFunctionCalls {
return SkipConditionLiteral
}
// ftsFieldKeys == nil means search() is not enabled for this signal/query type.
// Only log statement builders set FTSFieldKeys; traces/metrics leave it nil.
if len(v.ftsFieldKeys) == 0 {
// ftsContexts == nil means search() is not enabled for this signal/query type.
// Only log statement builders set FTSContexts; traces/metrics leave it nil.
if len(v.ftsContexts) == 0 {
v.errors = append(v.errors, "search() is only supported for log queries")
return ErrorConditionLiteral
}
@@ -777,16 +774,16 @@ func (v *filterExpressionVisitor) VisitSearchCall(ctx *grammar.SearchCallContext
}
if v.endNs > 0 && v.startNs > 0 && (v.endNs-v.startNs) > FTSMaxWindowNs {
v.errors = append(v.errors, "search() is restricted to a maximum of 6-hour time window")
v.errors = append(v.errors, "full text search is restricted to a maximum of 6-hour time window")
return ErrorConditionLiteral
}
formattedText := FormatFullTextSearch(searchText)
var ftsConds []string
for _, key := range v.ftsFieldKeys {
cond, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, qbtypes.FilterOperatorRegexp, formattedText, v.builder)
for _, fieldCtx := range v.ftsContexts {
cond, err := v.conditionBuilder.ConditionForContext(v.context, fieldCtx, formattedText, v.builder)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("search() could not build condition for field %q: %s", key.Text(), err.Error()))
v.errors = append(v.errors, fmt.Sprintf("search() could not build condition for context %q: %s", fieldCtx.StringValue(), err.Error()))
return ErrorConditionLiteral
}
ftsConds = append(ftsConds, cond)

View File

@@ -744,6 +744,10 @@ func (b *resourceConditionBuilder) ConditionFor(
return fmt.Sprintf("%s_cond", key.Name), nil
}
func (b *resourceConditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}
type conditionBuilder struct{}
func (b *conditionBuilder) ConditionFor(
@@ -755,10 +759,13 @@ func (b *conditionBuilder) ConditionFor(
_ any,
_ *sqlbuilder.SelectBuilder,
) (string, error) {
return fmt.Sprintf("%s_cond", key.Name), nil
}
func (b *conditionBuilder) ConditionForContext(_ context.Context, fc telemetrytypes.FieldContext, _ any, sb *sqlbuilder.SelectBuilder) (string, error) {
return fmt.Sprintf("fts_%s_cond", fc.StringValue()), nil
}
// visitComparisonCase is a single test case for the TestVisitComparison_* family.
// Each case is run under two independent configurations:
//
@@ -798,7 +805,7 @@ func visitComparisonOpts(t *testing.T) (rsbOpts, sbOpts FilterExprVisitorOpts) {
ConditionBuilder: &resourceConditionBuilder{},
Variables: allVariable,
SkipResourceFilter: false,
SkipFreeTextFilter: true,
SkipFullTextFilter: true,
SkipFunctionCalls: true,
IgnoreNotFoundKeys: true,
}
@@ -808,7 +815,7 @@ func visitComparisonOpts(t *testing.T) (rsbOpts, sbOpts FilterExprVisitorOpts) {
ConditionBuilder: &conditionBuilder{},
Variables: allVariable,
SkipResourceFilter: true,
SkipFreeTextFilter: false,
SkipFullTextFilter: false,
SkipFunctionCalls: false,
IgnoreNotFoundKeys: false,
FreeTextColumn: bodyCol,
@@ -1686,23 +1693,18 @@ func TestVisitComparison_UnknownKeys(t *testing.T) {
}
// TestVisitComparison_FullTextSearch covers Full Text Search — the explicit search()
// function that fans out across all ftsFieldKeys. FTSFieldKeys must be set for
// function that fans out across all FTSSet columns. FTSSet must be set for
// search() to be enabled; invalid param counts must error.
func TestVisitComparison_FullTextSearch(t *testing.T) {
bodyKey := &telemetrytypes.TelemetryFieldKey{
Name: "body",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
ftsOpts := FilterExprVisitorOpts{
Context: t.Context(),
FieldKeys: visitTestKeys,
ConditionBuilder: &conditionBuilder{},
SkipResourceFilter: false,
SkipFreeTextFilter: false,
SkipFullTextFilter: false,
SkipFunctionCalls: false,
IgnoreNotFoundKeys: false,
FTSFieldKeys: []*telemetrytypes.TelemetryFieldKey{bodyKey},
FTSContexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextResource},
}
tests := []struct {

View File

@@ -198,3 +198,7 @@ func (c *conditionBuilder) ConditionFor(
return condition, nil
}
func (c *conditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}

View File

@@ -122,3 +122,7 @@ func (m *fieldMapper) ColumnExpressionFor(
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
}
func (m *fieldMapper) GetColumns(_ context.Context, _ telemetrytypes.FieldContext) []*schema.Column {
return nil
}

View File

@@ -16,10 +16,6 @@ import (
"github.com/huandu/go-sqlbuilder"
)
var (
ErrCodeInvalidFTSOperator = errors.MustNewCode("invalid_fts_operator")
)
type conditionBuilder struct {
fm qbtypes.FieldMapper
fl flagger.Flagger
@@ -29,72 +25,47 @@ func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionB
return &conditionBuilder{fm: fm, fl: fl}
}
// buildFTSCondition produces the WHERE fragment for a single FTS key by iterating
// over its resolved columns and emitting the right match expression per column type:
// - Map → arrayExists(x -> match(x, ?), mapKeys/mapValues(col))
// - JSON → match(LOWER(toString(col)), LOWER(?)) — only when useJSONBody is true
// - String/LowCardinality → match(LOWER(col), LOWER(?))
//
// Evolution entries (key.Evolutions) are applied first: selectEvolutionsForColumns
// picks the active column set for the query time range, and the ColumnName from each
// entry overrides col.Name so evolved table layouts use the right physical column.
// JSON columns are then skipped when useJSONBody is false.
func buildFTSCondition(
columns []*schema.Column,
key *telemetrytypes.TelemetryFieldKey,
tsStart, tsEnd uint64,
// ConditionForContext builds the search condition for all FTS columns belonging
// to fieldContext. JSON columns are skipped when useJSONBody is disabled.
// Returns ("", nil) when no searchable columns exist for the context.
func (c *conditionBuilder) ConditionForContext(
ctx context.Context,
fieldContext telemetrytypes.FieldContext,
value any,
useJSONBody bool,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
activeColumns := columns
var evolutionEntries []*telemetrytypes.EvolutionEntry
if len(key.Evolutions) > 0 {
var err error
activeColumns, evolutionEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
if err != nil {
return "", err
}
}
columns := c.fm.GetColumns(ctx, fieldContext)
var conditions []string
for i, col := range activeColumns {
if !useJSONBody && col.Type.GetType() == schema.ColumnTypeEnumJSON {
continue
}
colName := col.Name
if evolutionEntries != nil && evolutionEntries[i] != nil {
colName = evolutionEntries[i].ColumnName
}
for _, col := range columns {
switch col.Type.GetType() {
case schema.ColumnTypeEnumMap:
keysExpr := fmt.Sprintf("mapKeys(%s)", colName)
valsExpr := fmt.Sprintf("mapValues(%s)", colName)
keysExpr := fmt.Sprintf("mapKeys(%s)", col.Name)
valsExpr := fmt.Sprintf("mapValues(%s)", col.Name)
if mc, ok := col.Type.(schema.MapColumnType); ok && mc.ValueType.GetType() != schema.ColumnTypeEnumString {
valsExpr = fmt.Sprintf("arrayMap(x -> toString(x), mapValues(%s))", colName)
valsExpr = fmt.Sprintf("arrayMap(x -> toString(x), mapValues(%s))", col.Name)
}
conditions = append(conditions,
conditions = append(conditions, sb.Or(
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), keysExpr),
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), valsExpr),
)
))
case schema.ColumnTypeEnumJSON:
conditions = append(conditions,
fmt.Sprintf(`match(LOWER(toString(%s)), LOWER(%s))`, colName, sb.Var(value)))
conditions = append(conditions, fmt.Sprintf(`match(LOWER(toString(%s)), LOWER(%s))`, col.Name, sb.Var(value)))
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumLowCardinality:
conditions = append(conditions,
fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, colName, sb.Var(value)))
conditions = append(conditions, fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, col.Name, sb.Var(value)))
default:
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "FTS unsupported column type %s", col.Type)
}
}
if len(conditions) == 0 {
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "no FTS conditions built for columns")
}
if len(conditions) == 1 {
switch len(conditions) {
case 0:
return "", nil
case 1:
return conditions[0], nil
default:
return sb.Or(conditions...), nil
}
return sb.Or(conditions...), nil
}
func (c *conditionBuilder) conditionFor(
@@ -105,22 +76,11 @@ func (c *conditionBuilder) conditionFor(
value any,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
if key.Name == querybuilder.FTSInternalKey && operator != qbtypes.FilterOperatorRegexp {
return "", errors.NewInternalf(ErrCodeInvalidFTSOperator, "only regexp operator is supported for fts")
}
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
if err != nil {
return "", err
}
// FTS is branched here
if key.Name == querybuilder.FTSInternalKey {
// TODO(Tushar): thread orgID here to evaluate correctly
useJSONBody := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
return buildFTSCondition(columns, key, startNs, endNs, value, useJSONBody, sb)
}
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
for _, column := range columns {
// TODO(Tushar): thread orgID here to evaluate correctly
@@ -365,12 +325,6 @@ func (c *conditionBuilder) ConditionFor(
return "", err
}
// FTS wildcard conditions are self-contained (arrayExists over full map);
// no additional EXISTS wrapper is needed.
if key.Name == querybuilder.FTSInternalKey {
return condition, nil
}
// Skip adding exists filter for intrinsic fields i.e. Table level log context fields
buildExistCondition := operator.AddDefaultExistsFilter()
switch key.FieldContext {

View File

@@ -4,7 +4,6 @@ import (
"fmt"
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
@@ -35,6 +34,7 @@ const (
LogsV2AttributesNumberColumn = "attributes_number"
LogsV2AttributesBoolColumn = "attributes_bool"
LogsV2ResourcesStringColumn = "resources_string"
LogsV2ResourceJSONColumn = "resource"
LogsV2ScopeStringColumn = "scope_string"
BodyV2ColumnPrefix = constants.BodyV2ColumnPrefix
@@ -127,19 +127,14 @@ var (
},
}
// defaultFTSFieldKeys is set of TelemetryFieldKey instances that
// search() fans out across. Intrinsic log columns use the normal conditionFor
// path; entries with Name==FTSInternalKey are short-circuited in conditionFor
// to emit arrayExists conditions over mapKeys/mapValues without arrayConcat.
defaultFTSFieldKeys = []*telemetrytypes.TelemetryFieldKey{
{Name: LogsV2SeverityTextColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: LogsV2TraceIDColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: LogsV2SpanIDColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextBody, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeBool},
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
// ftsSupportedContexts is the ordered list of field contexts that search()
// fans out across. The field mapper's GetColumns is called for each to
// retrieve the physical columns.
ftsSupportedContexts = []telemetrytypes.FieldContext{
telemetrytypes.FieldContextLog,
telemetrytypes.FieldContextBody,
telemetrytypes.FieldContextAttribute,
telemetrytypes.FieldContextResource,
}
)

View File

@@ -543,3 +543,36 @@ func (m *fieldMapper) buildArrayMap(currentNode *telemetrytypes.JSONAccessNode,
return fmt.Sprintf("arrayMap(%s->%s, %s)", currentNode.Alias(), nestedExpr, arrayExpr), nil
}
// GetColumns returns the physical columns for the given field context that
// search() fans out across. For FieldContextBody the active column set depends
// on the useJSONBody feature flag.
func (m *fieldMapper) GetColumns(ctx context.Context, fieldContext telemetrytypes.FieldContext) []*schema.Column {
switch fieldContext {
case telemetrytypes.FieldContextLog:
return []*schema.Column{
logsV2Columns[LogsV2SeverityTextColumn],
logsV2Columns[LogsV2TraceIDColumn],
logsV2Columns[LogsV2SpanIDColumn],
}
case telemetrytypes.FieldContextBody:
// TODO(Tushar): thread orgID here to evaluate correctly
if m.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{})) {
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}
}
return []*schema.Column{logsV2Columns[LogsV2BodyColumn]}
case telemetrytypes.FieldContextAttribute:
return []*schema.Column{
logsV2Columns[LogsV2AttributesStringColumn],
logsV2Columns[LogsV2AttributesNumberColumn],
logsV2Columns[LogsV2AttributesBoolColumn],
}
case telemetrytypes.FieldContextResource:
return []*schema.Column{
logsV2Columns[LogsV2ResourcesStringColumn],
logsV2Columns[LogsV2ResourceJSONColumn],
}
default:
return nil
}
}

View File

@@ -29,7 +29,6 @@ type logQueryStatementBuilder struct {
freeTextColumn *telemetrytypes.TelemetryFieldKey
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
ftsFieldKeys []*telemetrytypes.TelemetryFieldKey
}
var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*logQueryStatementBuilder)(nil)
@@ -66,9 +65,8 @@ func NewLogQueryStatementBuilder(
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
aggExprRewriter: aggExprRewriter,
fl: fl,
freeTextColumn: freeTextColumn,
jsonKeyToKey: jsonKeyToKey,
ftsFieldKeys: defaultFTSFieldKeys,
freeTextColumn: freeTextColumn,
jsonKeyToKey: jsonKeyToKey,
}
}
@@ -665,7 +663,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
EndNs: end,
}
if enableFTS {
opts.FTSFieldKeys = b.ftsFieldKeys
opts.FTSContexts = ftsSupportedContexts
}
// add filter expression

View File

@@ -1111,9 +1111,9 @@ func TestStmtBuilderTextSearch(t *testing.T) {
},
},
// ── Full Text Search ──────────────────────────────────────────────────────────
// search() fans out via ftsFieldKeys across all fields (severity_text, trace_id,
// span_id, body, attributes_*, resources_*). Uses a 2-hour window to stay under
// the 6-hour limit.
// search() fans out via ftsSupportedContexts (log, body, attribute, resource).
// Each context returns one OR-combined condition from ConditionForContext.
// Uses a 2-hour window to stay under the 6-hour limit.
{
name: "search_fans_out_to_all_columns",
requestType: qbtypes.RequestTypeRaw,
@@ -1126,7 +1126,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
startMs: 1705309200000,
endMs: 1705316400000,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
},
@@ -1143,7 +1143,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
startMs: 1705309200000,
endMs: 1705316400000,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE NOT ((match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string))))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE NOT (((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string))))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
},
@@ -1160,7 +1160,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
startMs: 1705309200000,
endMs: 1705316400000,
expected: qbtypes.Statement{
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND severity_text = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND severity_text = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "ERROR", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
},

View File

@@ -135,3 +135,7 @@ func (c *conditionBuilder) ConditionFor(
return fmt.Sprintf(expr, columns[0].Name, sb.Var(key.Name), cond), nil
}
func (c *conditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}

View File

@@ -116,3 +116,7 @@ func (m *fieldMapper) ColumnExpressionFor(
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
}
func (m *fieldMapper) GetColumns(_ context.Context, _ telemetrytypes.FieldContext) []*schema.Column {
return nil
}

View File

@@ -157,3 +157,7 @@ func (c *conditionBuilder) ConditionFor(
return condition, nil
}
func (c *conditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}

View File

@@ -104,3 +104,7 @@ func (m *fieldMapper) ColumnExpressionFor(
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
}
func (m *fieldMapper) GetColumns(_ context.Context, _ telemetrytypes.FieldContext) []*schema.Column {
return nil
}

View File

@@ -206,3 +206,7 @@ func (b *defaultConditionBuilder) ConditionFor(
}
return "", qbtypes.ErrUnsupportedOperator
}
func (b *defaultConditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}

View File

@@ -74,3 +74,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
}
return fmt.Sprintf("%s AS `%s`", fieldExpression, key.Name), nil
}
func (m *defaultFieldMapper) GetColumns(_ context.Context, _ telemetrytypes.FieldContext) []*schema.Column {
return nil
}

View File

@@ -155,8 +155,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
BodyJSONEnabled: bodyJSONEnabled,
FreeTextColumn: b.fullTextColumn,
JsonKeyToKey: b.jsonKeyToKey,
SkipFreeTextFilter: true,
SkipFullTextSearch: true,
SkipFullTextFilter: true,
SkipFunctionCalls: true,
// there is no need for "key" not found error for resource filtering
IgnoreNotFoundKeys: true,

View File

@@ -265,6 +265,10 @@ func (c *conditionBuilder) ConditionFor(
return condition, nil
}
func (c *conditionBuilder) ConditionForContext(_ context.Context, _ telemetrytypes.FieldContext, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
return "", nil
}
func (c *conditionBuilder) isSpanScopeField(name string) bool {
keyName := strings.ToLower(name)
return keyName == SpanSearchScopeRoot || keyName == SpanSearchScopeEntryPoint

View File

@@ -364,3 +364,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
}
func (m *defaultFieldMapper) GetColumns(_ context.Context, _ telemetrytypes.FieldContext) []*schema.Column {
return nil
}

View File

@@ -27,12 +27,20 @@ type FieldMapper interface {
ColumnFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error)
// ColumnExpressionFor returns the column expression for the given key.
ColumnExpressionFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) (string, error)
// GetColumns returns the physical columns to fan out across for FTS search()
// for the given field context. Non-log signal implementations return nil.
GetColumns(ctx context.Context, fieldContext telemetrytypes.FieldContext) []*schema.Column
}
// ConditionBuilder builds the condition for the filter.
type ConditionBuilder interface {
// ConditionFor returns the condition for the given key, operator and value.
ConditionFor(ctx context.Context, startNs uint64, endNs uint64, key *telemetrytypes.TelemetryFieldKey, operator FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error)
// ConditionForContext builds the search expression for all FTS columns belonging
// to fieldContext. Returns ("", nil) when no searchable columns exist for the
// context (e.g. JSON body when useJSONBody is disabled). Non-log
// implementations may return ("", nil) unconditionally.
ConditionForContext(ctx context.Context, fieldContext telemetrytypes.FieldContext, value any, sb *sqlbuilder.SelectBuilder) (string, error)
}
type AggExprRewriter interface {