mirror of
https://github.com/SigNoz/signoz.git
synced 2026-06-02 15:10:34 +01:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4bac933968 | ||
|
|
8df70ed4be | ||
|
|
806f1d0062 | ||
|
|
19d7d5b2ad | ||
|
|
8861f4523c | ||
|
|
7f97249de3 |
@@ -104,3 +104,4 @@ func (c *conditionBuilder) ConditionFor(
|
||||
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
|
||||
}
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ func newProvider(
|
||||
logAggExprRewriter,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
telemetrylogs.GetBodyJSONKey,
|
||||
logConditionBuilder.ConditionForSearch,
|
||||
flagger,
|
||||
)
|
||||
|
||||
|
||||
@@ -88,6 +88,7 @@ func prepareQuerierForLogs(t *testing.T, telemetryStore telemetrystore.Telemetry
|
||||
logAggExprRewriter,
|
||||
telemetrylogs.DefaultFullTextColumn,
|
||||
telemetrylogs.GetBodyJSONKey,
|
||||
logConditionBuilder.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
|
||||
@@ -12,14 +12,6 @@ const (
|
||||
// FullTextSearchDefaultWarning is emitted when a search() function call is used.
|
||||
FullTextSearchDefaultWarning = "Full text searches across all fields and will be slow and expensive. Consider using specific field to search e.g. <context>.<field_key>:<type>"
|
||||
|
||||
// FTSInternalKey is the sentinel Name on TelemetryFieldKey instances that represent
|
||||
// wildcard map searches (all attribute/resource keys+values). The unconventional value
|
||||
// prevents collision with any real field name a user could type.
|
||||
FTSInternalKey = "_X_INTERNAL_FTS_KEY"
|
||||
|
||||
// SearchFunctionName is the grammar function name for full-text search.
|
||||
SearchFunctionName = "search"
|
||||
|
||||
// FTSMaxWindowNs is the maximum allowed time range for a search() query (6 hours).
|
||||
FTSMaxWindowNs = uint64(6 * 60 * 60 * 1_000_000_000)
|
||||
)
|
||||
|
||||
@@ -47,7 +47,7 @@ type filterExpressionVisitor struct {
|
||||
keysWithWarnings map[string]bool
|
||||
startNs uint64
|
||||
endNs uint64
|
||||
ftsFieldKeys []*telemetrytypes.TelemetryFieldKey
|
||||
ftsCondition qbtypes.FTSConditionFunc
|
||||
}
|
||||
|
||||
type FilterExprVisitorOpts struct {
|
||||
@@ -59,6 +59,7 @@ type FilterExprVisitorOpts struct {
|
||||
Builder *sqlbuilder.SelectBuilder
|
||||
FreeTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
JsonKeyToKey qbtypes.JsonKeyToFieldFunc
|
||||
FTSCondition qbtypes.FTSConditionFunc
|
||||
BodyJSONEnabled bool
|
||||
SkipResourceFilter bool
|
||||
SkipFreeTextFilter bool
|
||||
@@ -68,9 +69,6 @@ type FilterExprVisitorOpts struct {
|
||||
Variables map[string]qbtypes.VariableItem
|
||||
StartNs uint64
|
||||
EndNs uint64
|
||||
// FTSFieldKeys enables search() for this query context. nil disables search()
|
||||
// (traces, metrics, and non-log callers leave this nil).
|
||||
FTSFieldKeys []*telemetrytypes.TelemetryFieldKey
|
||||
}
|
||||
|
||||
// newFilterExpressionVisitor creates a new filterExpressionVisitor.
|
||||
@@ -84,6 +82,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
|
||||
builder: opts.Builder,
|
||||
freeTextColumn: opts.FreeTextColumn,
|
||||
jsonKeyToKey: opts.JsonKeyToKey,
|
||||
ftsCondition: opts.FTSCondition,
|
||||
bodyJSONEnabled: opts.BodyJSONEnabled,
|
||||
skipResourceFilter: opts.SkipResourceFilter,
|
||||
skipFreeTextFilter: opts.SkipFreeTextFilter,
|
||||
@@ -94,7 +93,6 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
|
||||
keysWithWarnings: make(map[string]bool),
|
||||
startNs: opts.StartNs,
|
||||
endNs: opts.EndNs,
|
||||
ftsFieldKeys: opts.FTSFieldKeys,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -750,9 +748,9 @@ func (v *filterExpressionVisitor) VisitSearchCall(ctx *grammar.SearchCallContext
|
||||
if v.skipFunctionCalls || v.skipFullTextSearch {
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
// ftsFieldKeys == nil means search() is not enabled for this signal/query type.
|
||||
// Only log statement builders set FTSFieldKeys; traces/metrics leave it nil.
|
||||
if len(v.ftsFieldKeys) == 0 {
|
||||
// ftsCondition nil means search() is not enabled for this signal.
|
||||
// Only log statement builders set these; traces/metrics leave them nil.
|
||||
if v.ftsCondition == nil {
|
||||
v.errors = append(v.errors, "search() is only supported for log queries")
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -782,22 +780,14 @@ func (v *filterExpressionVisitor) VisitSearchCall(ctx *grammar.SearchCallContext
|
||||
}
|
||||
|
||||
formattedText := FormatFullTextSearch(searchText)
|
||||
var ftsConds []string
|
||||
for _, key := range v.ftsFieldKeys {
|
||||
cond, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, qbtypes.FilterOperatorRegexp, formattedText, v.builder)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("search() could not build condition for field %q: %s", key.Text(), err.Error()))
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
ftsConds = append(ftsConds, cond)
|
||||
}
|
||||
if len(ftsConds) == 0 {
|
||||
v.errors = append(v.errors, "search() failed: no searchable fields could be queried")
|
||||
cond, err := v.ftsCondition(v.context, formattedText, v.builder)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("search() could not build condition: %s", err.Error()))
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
|
||||
v.warnings = append(v.warnings, FullTextSearchDefaultWarning)
|
||||
return v.builder.Or(ftsConds...)
|
||||
return cond
|
||||
}
|
||||
|
||||
// VisitFunctionCall handles function calls like has(), hasAny(), hasAll(), hasToken().
|
||||
|
||||
@@ -755,7 +755,6 @@ func (b *conditionBuilder) ConditionFor(
|
||||
_ any,
|
||||
_ *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
|
||||
return fmt.Sprintf("%s_cond", key.Name), nil
|
||||
}
|
||||
|
||||
@@ -1686,14 +1685,9 @@ func TestVisitComparison_UnknownKeys(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestVisitComparison_FullTextSearch covers Full Text Search — the explicit search()
|
||||
// function that fans out across all ftsFieldKeys. FTSFieldKeys must be set for
|
||||
// function that fans out across all FTSSet columns. FTSSet must be set for
|
||||
// search() to be enabled; invalid param counts must error.
|
||||
func TestVisitComparison_FullTextSearch(t *testing.T) {
|
||||
bodyKey := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "body",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
ftsOpts := FilterExprVisitorOpts{
|
||||
Context: t.Context(),
|
||||
FieldKeys: visitTestKeys,
|
||||
@@ -1702,7 +1696,9 @@ func TestVisitComparison_FullTextSearch(t *testing.T) {
|
||||
SkipFreeTextFilter: false,
|
||||
SkipFunctionCalls: false,
|
||||
IgnoreNotFoundKeys: false,
|
||||
FTSFieldKeys: []*telemetrytypes.TelemetryFieldKey{bodyKey},
|
||||
FTSCondition: func(_ context.Context, _ any, _ *sqlbuilder.SelectBuilder) (string, error) {
|
||||
return fmt.Sprintf("fts_cond"), nil
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
|
||||
@@ -198,3 +198,4 @@ func (c *conditionBuilder) ConditionFor(
|
||||
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -16,10 +16,6 @@ import (
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeInvalidFTSOperator = errors.MustNewCode("invalid_fts_operator")
|
||||
)
|
||||
|
||||
type conditionBuilder struct {
|
||||
fm qbtypes.FieldMapper
|
||||
fl flagger.Flagger
|
||||
@@ -29,71 +25,47 @@ func NewConditionBuilder(fm qbtypes.FieldMapper, fl flagger.Flagger) *conditionB
|
||||
return &conditionBuilder{fm: fm, fl: fl}
|
||||
}
|
||||
|
||||
// buildFTSCondition produces the WHERE fragment for a single FTS key by iterating
|
||||
// over its resolved columns and emitting the right match expression per column type:
|
||||
// - Map → arrayExists(x -> match(x, ?), mapKeys/mapValues(col))
|
||||
// - JSON → match(LOWER(toString(col)), LOWER(?)) — only when useJSONBody is true
|
||||
// - String/LowCardinality → match(LOWER(col), LOWER(?))
|
||||
//
|
||||
// Evolution entries (key.Evolutions) are applied first: selectEvolutionsForColumns
|
||||
// picks the active column set for the query time range, and the ColumnName from each
|
||||
// entry overrides col.Name so evolved table layouts use the right physical column.
|
||||
// JSON columns are then skipped when useJSONBody is false.
|
||||
func buildFTSCondition(
|
||||
columns []*schema.Column,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
tsStart, tsEnd uint64,
|
||||
// ConditionForSearch builds the search condition for all FTS columns belonging
|
||||
// to fieldContext. JSON columns are skipped when useJSONBody is disabled.
|
||||
// Returns ("", nil) when no searchable columns exist for the context.
|
||||
func (c *conditionBuilder) ConditionForSearch(
|
||||
ctx context.Context,
|
||||
value any,
|
||||
useJSONBody bool,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
activeColumns := columns
|
||||
var evolutionEntries []*telemetrytypes.EvolutionEntry
|
||||
|
||||
if len(key.Evolutions) > 0 {
|
||||
var err error
|
||||
activeColumns, evolutionEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
contexts := []telemetrytypes.FieldContext{
|
||||
telemetrytypes.FieldContextLog,
|
||||
telemetrytypes.FieldContextBody,
|
||||
telemetrytypes.FieldContextAttribute,
|
||||
telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
useJSONBody := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
|
||||
|
||||
var conditions []string
|
||||
for i, col := range activeColumns {
|
||||
if !useJSONBody && col.Type.GetType() == schema.ColumnTypeEnumJSON {
|
||||
continue
|
||||
}
|
||||
|
||||
colName := col.Name
|
||||
if evolutionEntries != nil && evolutionEntries[i] != nil {
|
||||
colName = evolutionEntries[i].ColumnName
|
||||
}
|
||||
|
||||
switch col.Type.GetType() {
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keysExpr := fmt.Sprintf("mapKeys(%s)", colName)
|
||||
valsExpr := fmt.Sprintf("mapValues(%s)", colName)
|
||||
if mc, ok := col.Type.(schema.MapColumnType); ok && mc.ValueType.GetType() != schema.ColumnTypeEnumString {
|
||||
valsExpr = fmt.Sprintf("arrayMap(x -> toString(x), mapValues(%s))", colName)
|
||||
for _, fieldContext := range contexts {
|
||||
for _, col := range ftsColumns(ctx, fieldContext, useJSONBody) {
|
||||
switch col.Type.GetType() {
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keysExpr := fmt.Sprintf("mapKeys(%s)", col.Name)
|
||||
valsExpr := fmt.Sprintf("mapValues(%s)", col.Name)
|
||||
if mc, ok := col.Type.(schema.MapColumnType); ok && mc.ValueType.GetType() != schema.ColumnTypeEnumString {
|
||||
valsExpr = fmt.Sprintf("arrayMap(x -> toString(x), mapValues(%s))", col.Name)
|
||||
}
|
||||
conditions = append(conditions, sb.Or(
|
||||
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), keysExpr),
|
||||
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), valsExpr),
|
||||
))
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
conditions = append(conditions, fmt.Sprintf(`match(LOWER(toString(%s)), LOWER(%s))`, col.Name, sb.Var(value)))
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumLowCardinality:
|
||||
conditions = append(conditions, fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, col.Name, sb.Var(value)))
|
||||
default:
|
||||
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "FTS unsupported column type %s", col.Type)
|
||||
}
|
||||
conditions = append(conditions,
|
||||
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), keysExpr),
|
||||
fmt.Sprintf(`arrayExists(x -> match(x, %s), %s)`, sb.Var(value), valsExpr),
|
||||
)
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
conditions = append(conditions,
|
||||
fmt.Sprintf(`match(LOWER(toString(%s)), LOWER(%s))`, colName, sb.Var(value)))
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumLowCardinality:
|
||||
conditions = append(conditions,
|
||||
fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, colName, sb.Var(value)))
|
||||
}
|
||||
}
|
||||
if len(conditions) == 0 {
|
||||
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "no FTS conditions built for columns")
|
||||
}
|
||||
if len(conditions) == 1 {
|
||||
return conditions[0], nil
|
||||
}
|
||||
|
||||
return sb.Or(conditions...), nil
|
||||
}
|
||||
|
||||
@@ -105,22 +77,11 @@ func (c *conditionBuilder) conditionFor(
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
if key.Name == querybuilder.FTSInternalKey && operator != qbtypes.FilterOperatorRegexp {
|
||||
return "", errors.NewInternalf(ErrCodeInvalidFTSOperator, "only regexp operator is supported for fts")
|
||||
}
|
||||
|
||||
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// FTS is branched here
|
||||
if key.Name == querybuilder.FTSInternalKey {
|
||||
// TODO(Tushar): thread orgID here to evaluate correctly
|
||||
useJSONBody := c.fl.BooleanOrEmpty(ctx, flagger.FeatureUseJSONBody, featuretypes.NewFlaggerEvaluationContext(valuer.UUID{}))
|
||||
return buildFTSCondition(columns, key, startNs, endNs, value, useJSONBody, sb)
|
||||
}
|
||||
|
||||
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
|
||||
for _, column := range columns {
|
||||
// TODO(Tushar): thread orgID here to evaluate correctly
|
||||
@@ -365,12 +326,6 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return "", err
|
||||
}
|
||||
|
||||
// FTS wildcard conditions are self-contained (arrayExists over full map);
|
||||
// no additional EXISTS wrapper is needed.
|
||||
if key.Name == querybuilder.FTSInternalKey {
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
// Skip adding exists filter for intrinsic fields i.e. Table level log context fields
|
||||
buildExistCondition := operator.AddDefaultExistsFilter()
|
||||
switch key.FieldContext {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz-otel-collector/constants"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
@@ -35,6 +34,7 @@ const (
|
||||
LogsV2AttributesNumberColumn = "attributes_number"
|
||||
LogsV2AttributesBoolColumn = "attributes_bool"
|
||||
LogsV2ResourcesStringColumn = "resources_string"
|
||||
LogsV2ResourceJSONColumn = "resource"
|
||||
LogsV2ScopeStringColumn = "scope_string"
|
||||
|
||||
BodyV2ColumnPrefix = constants.BodyV2ColumnPrefix
|
||||
@@ -126,21 +126,6 @@ var (
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
},
|
||||
}
|
||||
|
||||
// defaultFTSFieldKeys is set of TelemetryFieldKey instances that
|
||||
// search() fans out across. Intrinsic log columns use the normal conditionFor
|
||||
// path; entries with Name==FTSInternalKey are short-circuited in conditionFor
|
||||
// to emit arrayExists conditions over mapKeys/mapValues without arrayConcat.
|
||||
defaultFTSFieldKeys = []*telemetrytypes.TelemetryFieldKey{
|
||||
{Name: LogsV2SeverityTextColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: LogsV2TraceIDColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: LogsV2SpanIDColumn, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextLog, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextBody, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeBool},
|
||||
{Name: querybuilder.FTSInternalKey, Signal: telemetrytypes.SignalLogs, FieldContext: telemetrytypes.FieldContextResource, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
}
|
||||
)
|
||||
|
||||
func bodyAliasExpression(bodyJSONEnabled bool) string {
|
||||
|
||||
@@ -543,3 +543,35 @@ func (m *fieldMapper) buildArrayMap(currentNode *telemetrytypes.JSONAccessNode,
|
||||
|
||||
return fmt.Sprintf("arrayMap(%s->%s, %s)", currentNode.Alias(), nestedExpr, arrayExpr), nil
|
||||
}
|
||||
|
||||
// GetColumns returns the physical columns for the given field context that
|
||||
// search() fans out across. For FieldContextBody the active column set depends
|
||||
// on the useJSONBody feature flag.
|
||||
func ftsColumns(ctx context.Context, fieldContext telemetrytypes.FieldContext, useJSONBody bool) []*schema.Column {
|
||||
switch fieldContext {
|
||||
case telemetrytypes.FieldContextLog:
|
||||
return []*schema.Column{
|
||||
logsV2Columns[LogsV2SeverityTextColumn],
|
||||
logsV2Columns[LogsV2TraceIDColumn],
|
||||
logsV2Columns[LogsV2SpanIDColumn],
|
||||
}
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if useJSONBody {
|
||||
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}
|
||||
}
|
||||
return []*schema.Column{logsV2Columns[LogsV2BodyColumn]}
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
return []*schema.Column{
|
||||
logsV2Columns[LogsV2AttributesStringColumn],
|
||||
logsV2Columns[LogsV2AttributesNumberColumn],
|
||||
logsV2Columns[LogsV2AttributesBoolColumn],
|
||||
}
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return []*schema.Column{
|
||||
logsV2Columns[LogsV2ResourcesStringColumn],
|
||||
logsV2Columns[LogsV2ResourceJSONColumn],
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1204,6 +1204,7 @@ func buildJSONTestStatementBuilder(t *testing.T, addIndexes bool) (*logQueryStat
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
|
||||
@@ -27,9 +27,9 @@ type logQueryStatementBuilder struct {
|
||||
aggExprRewriter qbtypes.AggExprRewriter
|
||||
fl flagger.Flagger
|
||||
|
||||
freeTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
|
||||
ftsFieldKeys []*telemetrytypes.TelemetryFieldKey
|
||||
freeTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
|
||||
ftsConditionFunc qbtypes.FTSConditionFunc
|
||||
}
|
||||
|
||||
var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*logQueryStatementBuilder)(nil)
|
||||
@@ -42,6 +42,7 @@ func NewLogQueryStatementBuilder(
|
||||
aggExprRewriter qbtypes.AggExprRewriter,
|
||||
freeTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
|
||||
ftsConditionFunc qbtypes.FTSConditionFunc,
|
||||
fl flagger.Flagger,
|
||||
) *logQueryStatementBuilder {
|
||||
logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
|
||||
@@ -66,9 +67,9 @@ func NewLogQueryStatementBuilder(
|
||||
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
|
||||
aggExprRewriter: aggExprRewriter,
|
||||
fl: fl,
|
||||
freeTextColumn: freeTextColumn,
|
||||
jsonKeyToKey: jsonKeyToKey,
|
||||
ftsFieldKeys: defaultFTSFieldKeys,
|
||||
freeTextColumn: freeTextColumn,
|
||||
jsonKeyToKey: jsonKeyToKey,
|
||||
ftsConditionFunc: ftsConditionFunc,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -665,7 +666,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
EndNs: end,
|
||||
}
|
||||
if enableFTS {
|
||||
opts.FTSFieldKeys = b.ftsFieldKeys
|
||||
opts.FTSCondition = b.ftsConditionFunc
|
||||
}
|
||||
|
||||
// add filter expression
|
||||
|
||||
@@ -212,6 +212,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -353,6 +354,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -483,6 +485,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -559,6 +562,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -654,6 +658,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -878,6 +883,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -1023,6 +1029,7 @@ func TestStmtBuilderBodyField(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
|
||||
@@ -1111,9 +1118,9 @@ func TestStmtBuilderTextSearch(t *testing.T) {
|
||||
},
|
||||
},
|
||||
// ── Full Text Search ──────────────────────────────────────────────────────────
|
||||
// search() fans out via ftsFieldKeys across all fields (severity_text, trace_id,
|
||||
// span_id, body, attributes_*, resources_*). Uses a 2-hour window to stay under
|
||||
// the 6-hour limit.
|
||||
// search() fans out via ftsSupportedContexts (log, body, attribute, resource).
|
||||
// Each context returns one OR-combined condition from ConditionForContext.
|
||||
// Uses a 2-hour window to stay under the 6-hour limit.
|
||||
{
|
||||
name: "search_fans_out_to_all_columns",
|
||||
requestType: qbtypes.RequestTypeRaw,
|
||||
@@ -1126,7 +1133,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
|
||||
startMs: 1705309200000,
|
||||
endMs: 1705316400000,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
|
||||
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
|
||||
},
|
||||
@@ -1143,7 +1150,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
|
||||
startMs: 1705309200000,
|
||||
endMs: 1705316400000,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE NOT ((match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string))))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE NOT (((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string))))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "healthcheck", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
|
||||
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
|
||||
},
|
||||
@@ -1160,7 +1167,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
|
||||
startMs: 1705309200000,
|
||||
endMs: 1705316400000,
|
||||
expected: qbtypes.Statement{
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE ((match(severity_text, ?) OR match(trace_id, ?) OR match(span_id, ?) OR match(LOWER(body), LOWER(?)) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool)))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND severity_text = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Query: "SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE (((match(LOWER(severity_text), LOWER(?)) OR match(LOWER(trace_id), LOWER(?)) OR match(LOWER(span_id), LOWER(?))) OR match(LOWER(body), LOWER(?)) OR ((arrayExists(x -> match(x, ?), mapKeys(attributes_string)) OR arrayExists(x -> match(x, ?), mapValues(attributes_string))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_number)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_number)))) OR (arrayExists(x -> match(x, ?), mapKeys(attributes_bool)) OR arrayExists(x -> match(x, ?), arrayMap(x -> toString(x), mapValues(attributes_bool))))) OR (arrayExists(x -> match(x, ?), mapKeys(resources_string)) OR arrayExists(x -> match(x, ?), mapValues(resources_string)))) AND severity_text = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
|
||||
Args: []any{"error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "error", "ERROR", "1705309200000000000", uint64(1705307400), "1705316400000000000", uint64(1705316400), 10},
|
||||
Warnings: []string{querybuilder.FullTextSearchDefaultWarning},
|
||||
},
|
||||
@@ -1199,6 +1206,7 @@ func TestStmtBuilderTextSearch(t *testing.T) {
|
||||
aggExprRewriter,
|
||||
DefaultFullTextColumn,
|
||||
GetBodyJSONKey,
|
||||
cb.ConditionForSearch,
|
||||
fl,
|
||||
)
|
||||
startMs := uint64(1747947419000)
|
||||
|
||||
@@ -135,3 +135,4 @@ func (c *conditionBuilder) ConditionFor(
|
||||
|
||||
return fmt.Sprintf(expr, columns[0].Name, sb.Var(key.Name), cond), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -157,3 +157,4 @@ func (c *conditionBuilder) ConditionFor(
|
||||
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -206,3 +206,4 @@ func (b *defaultConditionBuilder) ConditionFor(
|
||||
}
|
||||
return "", qbtypes.ErrUnsupportedOperator
|
||||
}
|
||||
|
||||
|
||||
@@ -265,6 +265,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
|
||||
func (c *conditionBuilder) isSpanScopeField(name string) bool {
|
||||
keyName := strings.ToLower(name)
|
||||
return keyName == SpanSearchScopeRoot || keyName == SpanSearchScopeEntryPoint
|
||||
|
||||
@@ -19,6 +19,11 @@ var (
|
||||
|
||||
type JsonKeyToFieldFunc func(context.Context, *telemetrytypes.TelemetryFieldKey, FilterOperator, any) (string, any)
|
||||
|
||||
// FTSConditionFunc builds the search() SQL condition for all FTS columns belonging
|
||||
// to fieldContext. Returns ("", nil) when no columns are searchable for the
|
||||
// context (e.g. body JSON when useJSONBody is off). Pass nil to disable search().
|
||||
type FTSConditionFunc func(ctx context.Context, value any, sb *sqlbuilder.SelectBuilder) (string, error)
|
||||
|
||||
// FieldMapper maps the telemetry field key to the table field name.
|
||||
type FieldMapper interface {
|
||||
// FieldFor returns the field name for the given key.
|
||||
|
||||
Reference in New Issue
Block a user